1 | //===- llvm/CodeGen/SelectionDAGNodes.h - SelectionDAG Nodes ----*- C++ -*-===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file declares the SDNode class and derived classes, which are used to |
10 | // represent the nodes and operations present in a SelectionDAG. These nodes |
11 | // and operations are machine code level operations, with some similarities to |
12 | // the GCC RTL representation. |
13 | // |
14 | // Clients should include the SelectionDAG.h file instead of this file directly. |
15 | // |
16 | //===----------------------------------------------------------------------===// |
17 | |
18 | #ifndef LLVM_CODEGEN_SELECTIONDAGNODES_H |
19 | #define LLVM_CODEGEN_SELECTIONDAGNODES_H |
20 | |
21 | #include "llvm/ADT/APFloat.h" |
22 | #include "llvm/ADT/ArrayRef.h" |
23 | #include "llvm/ADT/BitVector.h" |
24 | #include "llvm/ADT/FoldingSet.h" |
25 | #include "llvm/ADT/GraphTraits.h" |
26 | #include "llvm/ADT/SmallPtrSet.h" |
27 | #include "llvm/ADT/SmallVector.h" |
28 | #include "llvm/ADT/ilist_node.h" |
29 | #include "llvm/ADT/iterator.h" |
30 | #include "llvm/ADT/iterator_range.h" |
31 | #include "llvm/CodeGen/ISDOpcodes.h" |
32 | #include "llvm/CodeGen/MachineMemOperand.h" |
33 | #include "llvm/CodeGen/Register.h" |
34 | #include "llvm/CodeGen/ValueTypes.h" |
35 | #include "llvm/CodeGenTypes/MachineValueType.h" |
36 | #include "llvm/IR/Constants.h" |
37 | #include "llvm/IR/DebugLoc.h" |
38 | #include "llvm/IR/Instruction.h" |
39 | #include "llvm/IR/Instructions.h" |
40 | #include "llvm/IR/Metadata.h" |
41 | #include "llvm/IR/Operator.h" |
42 | #include "llvm/Support/AlignOf.h" |
43 | #include "llvm/Support/AtomicOrdering.h" |
44 | #include "llvm/Support/Casting.h" |
45 | #include "llvm/Support/ErrorHandling.h" |
46 | #include "llvm/Support/TypeSize.h" |
47 | #include <algorithm> |
48 | #include <cassert> |
49 | #include <climits> |
50 | #include <cstddef> |
51 | #include <cstdint> |
52 | #include <cstring> |
53 | #include <iterator> |
54 | #include <string> |
55 | #include <tuple> |
56 | #include <utility> |
57 | |
58 | namespace llvm { |
59 | |
60 | class APInt; |
61 | class Constant; |
62 | class GlobalValue; |
63 | class MachineBasicBlock; |
64 | class MachineConstantPoolValue; |
65 | class MCSymbol; |
66 | class raw_ostream; |
67 | class SDNode; |
68 | class SelectionDAG; |
69 | class Type; |
70 | class Value; |
71 | |
72 | void checkForCycles(const SDNode *N, const SelectionDAG *DAG = nullptr, |
73 | bool force = false); |
74 | |
75 | /// This represents a list of ValueType's that has been intern'd by |
76 | /// a SelectionDAG. Instances of this simple value class are returned by |
77 | /// SelectionDAG::getVTList(...). |
78 | /// |
79 | struct SDVTList { |
80 | const EVT *VTs; |
81 | unsigned int NumVTs; |
82 | }; |
83 | |
84 | namespace ISD { |
85 | |
86 | /// Node predicates |
87 | |
88 | /// If N is a BUILD_VECTOR or SPLAT_VECTOR node whose elements are all the |
89 | /// same constant or undefined, return true and return the constant value in |
90 | /// \p SplatValue. |
91 | bool isConstantSplatVector(const SDNode *N, APInt &SplatValue); |
92 | |
93 | /// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where |
94 | /// all of the elements are ~0 or undef. If \p BuildVectorOnly is set to |
95 | /// true, it only checks BUILD_VECTOR. |
96 | bool isConstantSplatVectorAllOnes(const SDNode *N, |
97 | bool BuildVectorOnly = false); |
98 | |
99 | /// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where |
100 | /// all of the elements are 0 or undef. If \p BuildVectorOnly is set to true, it |
101 | /// only checks BUILD_VECTOR. |
102 | bool isConstantSplatVectorAllZeros(const SDNode *N, |
103 | bool BuildVectorOnly = false); |
104 | |
105 | /// Return true if the specified node is a BUILD_VECTOR where all of the |
106 | /// elements are ~0 or undef. |
107 | bool isBuildVectorAllOnes(const SDNode *N); |
108 | |
109 | /// Return true if the specified node is a BUILD_VECTOR where all of the |
110 | /// elements are 0 or undef. |
111 | bool isBuildVectorAllZeros(const SDNode *N); |
112 | |
113 | /// Return true if the specified node is a BUILD_VECTOR node of all |
114 | /// ConstantSDNode or undef. |
115 | bool isBuildVectorOfConstantSDNodes(const SDNode *N); |
116 | |
117 | /// Return true if the specified node is a BUILD_VECTOR node of all |
118 | /// ConstantFPSDNode or undef. |
119 | bool isBuildVectorOfConstantFPSDNodes(const SDNode *N); |
120 | |
121 | /// Returns true if the specified node is a vector where all elements can |
122 | /// be truncated to the specified element size without a loss in meaning. |
123 | bool isVectorShrinkable(const SDNode *N, unsigned NewEltSize, bool Signed); |
124 | |
125 | /// Return true if the node has at least one operand and all operands of the |
126 | /// specified node are ISD::UNDEF. |
127 | bool allOperandsUndef(const SDNode *N); |
128 | |
129 | /// Return true if the specified node is FREEZE(UNDEF). |
130 | bool isFreezeUndef(const SDNode *N); |
131 | |
132 | } // end namespace ISD |
133 | |
134 | //===----------------------------------------------------------------------===// |
135 | /// Unlike LLVM values, Selection DAG nodes may return multiple |
136 | /// values as the result of a computation. Many nodes return multiple values, |
137 | /// from loads (which define a token and a return value) to ADDC (which returns |
138 | /// a result and a carry value), to calls (which may return an arbitrary number |
139 | /// of values). |
140 | /// |
141 | /// As such, each use of a SelectionDAG computation must indicate the node that |
142 | /// computes it as well as which return value to use from that node. This pair |
143 | /// of information is represented with the SDValue value type. |
144 | /// |
145 | class SDValue { |
146 | friend struct DenseMapInfo<SDValue>; |
147 | |
148 | SDNode *Node = nullptr; // The node defining the value we are using. |
149 | unsigned ResNo = 0; // Which return value of the node we are using. |
150 | |
151 | public: |
152 | SDValue() = default; |
153 | SDValue(SDNode *node, unsigned resno); |
154 | |
155 | /// get the index which selects a specific result in the SDNode |
156 | unsigned getResNo() const { return ResNo; } |
157 | |
158 | /// get the SDNode which holds the desired result |
159 | SDNode *getNode() const { return Node; } |
160 | |
161 | /// set the SDNode |
162 | void setNode(SDNode *N) { Node = N; } |
163 | |
164 | inline SDNode *operator->() const { return Node; } |
165 | |
166 | bool operator==(const SDValue &O) const { |
167 | return Node == O.Node && ResNo == O.ResNo; |
168 | } |
169 | bool operator!=(const SDValue &O) const { |
170 | return !operator==(O); |
171 | } |
172 | bool operator<(const SDValue &O) const { |
173 | return std::tie(args: Node, args: ResNo) < std::tie(args: O.Node, args: O.ResNo); |
174 | } |
175 | explicit operator bool() const { |
176 | return Node != nullptr; |
177 | } |
178 | |
179 | SDValue getValue(unsigned R) const { |
180 | return SDValue(Node, R); |
181 | } |
182 | |
183 | /// Return true if this node is an operand of N. |
184 | bool isOperandOf(const SDNode *N) const; |
185 | |
186 | /// Return the ValueType of the referenced return value. |
187 | inline EVT getValueType() const; |
188 | |
189 | /// Return the simple ValueType of the referenced return value. |
190 | MVT getSimpleValueType() const { |
191 | return getValueType().getSimpleVT(); |
192 | } |
193 | |
194 | /// Returns the size of the value in bits. |
195 | /// |
196 | /// If the value type is a scalable vector type, the scalable property will |
197 | /// be set and the runtime size will be a positive integer multiple of the |
198 | /// base size. |
199 | TypeSize getValueSizeInBits() const { |
200 | return getValueType().getSizeInBits(); |
201 | } |
202 | |
203 | uint64_t getScalarValueSizeInBits() const { |
204 | return getValueType().getScalarType().getFixedSizeInBits(); |
205 | } |
206 | |
207 | // Forwarding methods - These forward to the corresponding methods in SDNode. |
208 | inline unsigned getOpcode() const; |
209 | inline unsigned getNumOperands() const; |
210 | inline const SDValue &getOperand(unsigned i) const; |
211 | inline uint64_t getConstantOperandVal(unsigned i) const; |
212 | inline const APInt &getConstantOperandAPInt(unsigned i) const; |
213 | inline bool isTargetMemoryOpcode() const; |
214 | inline bool isTargetOpcode() const; |
215 | inline bool isMachineOpcode() const; |
216 | inline bool isUndef() const; |
217 | inline unsigned getMachineOpcode() const; |
218 | inline const DebugLoc &getDebugLoc() const; |
219 | inline void dump() const; |
220 | inline void dump(const SelectionDAG *G) const; |
221 | inline void dumpr() const; |
222 | inline void dumpr(const SelectionDAG *G) const; |
223 | |
224 | /// Return true if this operand (which must be a chain) reaches the |
225 | /// specified operand without crossing any side-effecting instructions. |
226 | /// In practice, this looks through token factors and non-volatile loads. |
227 | /// In order to remain efficient, this only |
228 | /// looks a couple of nodes in, it does not do an exhaustive search. |
229 | bool reachesChainWithoutSideEffects(SDValue Dest, |
230 | unsigned Depth = 2) const; |
231 | |
232 | /// Return true if there are no nodes using value ResNo of Node. |
233 | inline bool use_empty() const; |
234 | |
235 | /// Return true if there is exactly one node using value ResNo of Node. |
236 | inline bool hasOneUse() const; |
237 | }; |
238 | |
239 | template<> struct DenseMapInfo<SDValue> { |
240 | static inline SDValue getEmptyKey() { |
241 | SDValue V; |
242 | V.ResNo = -1U; |
243 | return V; |
244 | } |
245 | |
246 | static inline SDValue getTombstoneKey() { |
247 | SDValue V; |
248 | V.ResNo = -2U; |
249 | return V; |
250 | } |
251 | |
252 | static unsigned getHashValue(const SDValue &Val) { |
253 | return ((unsigned)((uintptr_t)Val.getNode() >> 4) ^ |
254 | (unsigned)((uintptr_t)Val.getNode() >> 9)) + Val.getResNo(); |
255 | } |
256 | |
257 | static bool isEqual(const SDValue &LHS, const SDValue &RHS) { |
258 | return LHS == RHS; |
259 | } |
260 | }; |
261 | |
262 | /// Allow casting operators to work directly on |
263 | /// SDValues as if they were SDNode*'s. |
264 | template<> struct simplify_type<SDValue> { |
265 | using SimpleType = SDNode *; |
266 | |
267 | static SimpleType getSimplifiedValue(SDValue &Val) { |
268 | return Val.getNode(); |
269 | } |
270 | }; |
271 | template<> struct simplify_type<const SDValue> { |
272 | using SimpleType = /*const*/ SDNode *; |
273 | |
274 | static SimpleType getSimplifiedValue(const SDValue &Val) { |
275 | return Val.getNode(); |
276 | } |
277 | }; |
278 | |
279 | /// Represents a use of a SDNode. This class holds an SDValue, |
280 | /// which records the SDNode being used and the result number, a |
281 | /// pointer to the SDNode using the value, and Next and Prev pointers, |
282 | /// which link together all the uses of an SDNode. |
283 | /// |
284 | class SDUse { |
285 | /// Val - The value being used. |
286 | SDValue Val; |
287 | /// User - The user of this value. |
288 | SDNode *User = nullptr; |
289 | /// Prev, Next - Pointers to the uses list of the SDNode referred by |
290 | /// this operand. |
291 | SDUse **Prev = nullptr; |
292 | SDUse *Next = nullptr; |
293 | |
294 | public: |
295 | SDUse() = default; |
296 | SDUse(const SDUse &U) = delete; |
297 | SDUse &operator=(const SDUse &) = delete; |
298 | |
299 | /// Normally SDUse will just implicitly convert to an SDValue that it holds. |
300 | operator const SDValue&() const { return Val; } |
301 | |
302 | /// If implicit conversion to SDValue doesn't work, the get() method returns |
303 | /// the SDValue. |
304 | const SDValue &get() const { return Val; } |
305 | |
306 | /// This returns the SDNode that contains this Use. |
307 | SDNode *getUser() { return User; } |
308 | const SDNode *getUser() const { return User; } |
309 | |
310 | /// Get the next SDUse in the use list. |
311 | SDUse *getNext() const { return Next; } |
312 | |
313 | /// Convenience function for get().getNode(). |
314 | SDNode *getNode() const { return Val.getNode(); } |
315 | /// Convenience function for get().getResNo(). |
316 | unsigned getResNo() const { return Val.getResNo(); } |
317 | /// Convenience function for get().getValueType(). |
318 | EVT getValueType() const { return Val.getValueType(); } |
319 | |
320 | /// Convenience function for get().operator== |
321 | bool operator==(const SDValue &V) const { |
322 | return Val == V; |
323 | } |
324 | |
325 | /// Convenience function for get().operator!= |
326 | bool operator!=(const SDValue &V) const { |
327 | return Val != V; |
328 | } |
329 | |
330 | /// Convenience function for get().operator< |
331 | bool operator<(const SDValue &V) const { |
332 | return Val < V; |
333 | } |
334 | |
335 | private: |
336 | friend class SelectionDAG; |
337 | friend class SDNode; |
338 | // TODO: unfriend HandleSDNode once we fix its operand handling. |
339 | friend class HandleSDNode; |
340 | |
341 | void setUser(SDNode *p) { User = p; } |
342 | |
343 | /// Remove this use from its existing use list, assign it the |
344 | /// given value, and add it to the new value's node's use list. |
345 | inline void set(const SDValue &V); |
346 | /// Like set, but only supports initializing a newly-allocated |
347 | /// SDUse with a non-null value. |
348 | inline void setInitial(const SDValue &V); |
349 | /// Like set, but only sets the Node portion of the value, |
350 | /// leaving the ResNo portion unmodified. |
351 | inline void setNode(SDNode *N); |
352 | |
353 | void addToList(SDUse **List) { |
354 | Next = *List; |
355 | if (Next) Next->Prev = &Next; |
356 | Prev = List; |
357 | *List = this; |
358 | } |
359 | |
360 | void removeFromList() { |
361 | *Prev = Next; |
362 | if (Next) Next->Prev = Prev; |
363 | } |
364 | }; |
365 | |
366 | /// simplify_type specializations - Allow casting operators to work directly on |
367 | /// SDValues as if they were SDNode*'s. |
368 | template<> struct simplify_type<SDUse> { |
369 | using SimpleType = SDNode *; |
370 | |
371 | static SimpleType getSimplifiedValue(SDUse &Val) { |
372 | return Val.getNode(); |
373 | } |
374 | }; |
375 | |
376 | /// These are IR-level optimization flags that may be propagated to SDNodes. |
377 | /// TODO: This data structure should be shared by the IR optimizer and the |
378 | /// the backend. |
379 | struct SDNodeFlags { |
380 | private: |
381 | bool NoUnsignedWrap : 1; |
382 | bool NoSignedWrap : 1; |
383 | bool Exact : 1; |
384 | bool Disjoint : 1; |
385 | bool NonNeg : 1; |
386 | bool NoNaNs : 1; |
387 | bool NoInfs : 1; |
388 | bool NoSignedZeros : 1; |
389 | bool AllowReciprocal : 1; |
390 | bool AllowContract : 1; |
391 | bool ApproximateFuncs : 1; |
392 | bool AllowReassociation : 1; |
393 | |
394 | // We assume instructions do not raise floating-point exceptions by default, |
395 | // and only those marked explicitly may do so. We could choose to represent |
396 | // this via a positive "FPExcept" flags like on the MI level, but having a |
397 | // negative "NoFPExcept" flag here makes the flag intersection logic more |
398 | // straightforward. |
399 | bool NoFPExcept : 1; |
400 | // Instructions with attached 'unpredictable' metadata on IR level. |
401 | bool Unpredictable : 1; |
402 | |
403 | public: |
404 | /// Default constructor turns off all optimization flags. |
405 | SDNodeFlags() |
406 | : NoUnsignedWrap(false), NoSignedWrap(false), Exact(false), |
407 | Disjoint(false), NonNeg(false), NoNaNs(false), NoInfs(false), |
408 | NoSignedZeros(false), AllowReciprocal(false), AllowContract(false), |
409 | ApproximateFuncs(false), AllowReassociation(false), NoFPExcept(false), |
410 | Unpredictable(false) {} |
411 | |
412 | /// Propagate the fast-math-flags from an IR FPMathOperator. |
413 | void copyFMF(const FPMathOperator &FPMO) { |
414 | setNoNaNs(FPMO.hasNoNaNs()); |
415 | setNoInfs(FPMO.hasNoInfs()); |
416 | setNoSignedZeros(FPMO.hasNoSignedZeros()); |
417 | setAllowReciprocal(FPMO.hasAllowReciprocal()); |
418 | setAllowContract(FPMO.hasAllowContract()); |
419 | setApproximateFuncs(FPMO.hasApproxFunc()); |
420 | setAllowReassociation(FPMO.hasAllowReassoc()); |
421 | } |
422 | |
423 | // These are mutators for each flag. |
424 | void setNoUnsignedWrap(bool b) { NoUnsignedWrap = b; } |
425 | void setNoSignedWrap(bool b) { NoSignedWrap = b; } |
426 | void setExact(bool b) { Exact = b; } |
427 | void setDisjoint(bool b) { Disjoint = b; } |
428 | void setNonNeg(bool b) { NonNeg = b; } |
429 | void setNoNaNs(bool b) { NoNaNs = b; } |
430 | void setNoInfs(bool b) { NoInfs = b; } |
431 | void setNoSignedZeros(bool b) { NoSignedZeros = b; } |
432 | void setAllowReciprocal(bool b) { AllowReciprocal = b; } |
433 | void setAllowContract(bool b) { AllowContract = b; } |
434 | void setApproximateFuncs(bool b) { ApproximateFuncs = b; } |
435 | void setAllowReassociation(bool b) { AllowReassociation = b; } |
436 | void setNoFPExcept(bool b) { NoFPExcept = b; } |
437 | void setUnpredictable(bool b) { Unpredictable = b; } |
438 | |
439 | // These are accessors for each flag. |
440 | bool hasNoUnsignedWrap() const { return NoUnsignedWrap; } |
441 | bool hasNoSignedWrap() const { return NoSignedWrap; } |
442 | bool hasExact() const { return Exact; } |
443 | bool hasDisjoint() const { return Disjoint; } |
444 | bool hasNonNeg() const { return NonNeg; } |
445 | bool hasNoNaNs() const { return NoNaNs; } |
446 | bool hasNoInfs() const { return NoInfs; } |
447 | bool hasNoSignedZeros() const { return NoSignedZeros; } |
448 | bool hasAllowReciprocal() const { return AllowReciprocal; } |
449 | bool hasAllowContract() const { return AllowContract; } |
450 | bool hasApproximateFuncs() const { return ApproximateFuncs; } |
451 | bool hasAllowReassociation() const { return AllowReassociation; } |
452 | bool hasNoFPExcept() const { return NoFPExcept; } |
453 | bool hasUnpredictable() const { return Unpredictable; } |
454 | |
455 | /// Clear any flags in this flag set that aren't also set in Flags. All |
456 | /// flags will be cleared if Flags are undefined. |
457 | void intersectWith(const SDNodeFlags Flags) { |
458 | NoUnsignedWrap &= Flags.NoUnsignedWrap; |
459 | NoSignedWrap &= Flags.NoSignedWrap; |
460 | Exact &= Flags.Exact; |
461 | Disjoint &= Flags.Disjoint; |
462 | NonNeg &= Flags.NonNeg; |
463 | NoNaNs &= Flags.NoNaNs; |
464 | NoInfs &= Flags.NoInfs; |
465 | NoSignedZeros &= Flags.NoSignedZeros; |
466 | AllowReciprocal &= Flags.AllowReciprocal; |
467 | AllowContract &= Flags.AllowContract; |
468 | ApproximateFuncs &= Flags.ApproximateFuncs; |
469 | AllowReassociation &= Flags.AllowReassociation; |
470 | NoFPExcept &= Flags.NoFPExcept; |
471 | Unpredictable &= Flags.Unpredictable; |
472 | } |
473 | }; |
474 | |
475 | /// Represents one node in the SelectionDAG. |
476 | /// |
477 | class SDNode : public FoldingSetNode, public ilist_node<SDNode> { |
478 | private: |
479 | /// The operation that this node performs. |
480 | int32_t NodeType; |
481 | |
482 | public: |
483 | /// Unique and persistent id per SDNode in the DAG. Used for debug printing. |
484 | /// We do not place that under `#if LLVM_ENABLE_ABI_BREAKING_CHECKS` |
485 | /// intentionally because it adds unneeded complexity without noticeable |
486 | /// benefits (see discussion with @thakis in D120714). |
487 | uint16_t PersistentId = 0xffff; |
488 | |
489 | protected: |
490 | // We define a set of mini-helper classes to help us interpret the bits in our |
491 | // SubclassData. These are designed to fit within a uint16_t so they pack |
492 | // with PersistentId. |
493 | |
494 | #if defined(_AIX) && (!defined(__GNUC__) || defined(__clang__)) |
495 | // Except for GCC; by default, AIX compilers store bit-fields in 4-byte words |
496 | // and give the `pack` pragma push semantics. |
497 | #define BEGIN_TWO_BYTE_PACK() _Pragma("pack(2)") |
498 | #define END_TWO_BYTE_PACK() _Pragma("pack(pop)") |
499 | #else |
500 | #define BEGIN_TWO_BYTE_PACK() |
501 | #define END_TWO_BYTE_PACK() |
502 | #endif |
503 | |
504 | BEGIN_TWO_BYTE_PACK() |
505 | class SDNodeBitfields { |
506 | friend class SDNode; |
507 | friend class MemIntrinsicSDNode; |
508 | friend class MemSDNode; |
509 | friend class SelectionDAG; |
510 | |
511 | uint16_t HasDebugValue : 1; |
512 | uint16_t IsMemIntrinsic : 1; |
513 | uint16_t IsDivergent : 1; |
514 | }; |
515 | enum { NumSDNodeBits = 3 }; |
516 | |
517 | class ConstantSDNodeBitfields { |
518 | friend class ConstantSDNode; |
519 | |
520 | uint16_t : NumSDNodeBits; |
521 | |
522 | uint16_t IsOpaque : 1; |
523 | }; |
524 | |
525 | class MemSDNodeBitfields { |
526 | friend class MemSDNode; |
527 | friend class MemIntrinsicSDNode; |
528 | friend class AtomicSDNode; |
529 | |
530 | uint16_t : NumSDNodeBits; |
531 | |
532 | uint16_t IsVolatile : 1; |
533 | uint16_t IsNonTemporal : 1; |
534 | uint16_t IsDereferenceable : 1; |
535 | uint16_t IsInvariant : 1; |
536 | }; |
537 | enum { NumMemSDNodeBits = NumSDNodeBits + 4 }; |
538 | |
539 | class LSBaseSDNodeBitfields { |
540 | friend class LSBaseSDNode; |
541 | friend class VPBaseLoadStoreSDNode; |
542 | friend class MaskedLoadStoreSDNode; |
543 | friend class MaskedGatherScatterSDNode; |
544 | friend class VPGatherScatterSDNode; |
545 | |
546 | uint16_t : NumMemSDNodeBits; |
547 | |
548 | // This storage is shared between disparate class hierarchies to hold an |
549 | // enumeration specific to the class hierarchy in use. |
550 | // LSBaseSDNode => enum ISD::MemIndexedMode |
551 | // VPLoadStoreBaseSDNode => enum ISD::MemIndexedMode |
552 | // MaskedLoadStoreBaseSDNode => enum ISD::MemIndexedMode |
553 | // VPGatherScatterSDNode => enum ISD::MemIndexType |
554 | // MaskedGatherScatterSDNode => enum ISD::MemIndexType |
555 | uint16_t AddressingMode : 3; |
556 | }; |
557 | enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 }; |
558 | |
559 | class LoadSDNodeBitfields { |
560 | friend class LoadSDNode; |
561 | friend class VPLoadSDNode; |
562 | friend class VPStridedLoadSDNode; |
563 | friend class MaskedLoadSDNode; |
564 | friend class MaskedGatherSDNode; |
565 | friend class VPGatherSDNode; |
566 | |
567 | uint16_t : NumLSBaseSDNodeBits; |
568 | |
569 | uint16_t ExtTy : 2; // enum ISD::LoadExtType |
570 | uint16_t IsExpanding : 1; |
571 | }; |
572 | |
573 | class StoreSDNodeBitfields { |
574 | friend class StoreSDNode; |
575 | friend class VPStoreSDNode; |
576 | friend class VPStridedStoreSDNode; |
577 | friend class MaskedStoreSDNode; |
578 | friend class MaskedScatterSDNode; |
579 | friend class VPScatterSDNode; |
580 | |
581 | uint16_t : NumLSBaseSDNodeBits; |
582 | |
583 | uint16_t IsTruncating : 1; |
584 | uint16_t IsCompressing : 1; |
585 | }; |
586 | |
587 | union { |
588 | char RawSDNodeBits[sizeof(uint16_t)]; |
589 | SDNodeBitfields SDNodeBits; |
590 | ConstantSDNodeBitfields ConstantSDNodeBits; |
591 | MemSDNodeBitfields MemSDNodeBits; |
592 | LSBaseSDNodeBitfields LSBaseSDNodeBits; |
593 | LoadSDNodeBitfields LoadSDNodeBits; |
594 | StoreSDNodeBitfields StoreSDNodeBits; |
595 | }; |
596 | END_TWO_BYTE_PACK() |
597 | #undef BEGIN_TWO_BYTE_PACK |
598 | #undef END_TWO_BYTE_PACK |
599 | |
600 | // RawSDNodeBits must cover the entirety of the union. This means that all of |
601 | // the union's members must have size <= RawSDNodeBits. We write the RHS as |
602 | // "2" instead of sizeof(RawSDNodeBits) because MSVC can't handle the latter. |
603 | static_assert(sizeof(SDNodeBitfields) <= 2, "field too wide" ); |
604 | static_assert(sizeof(ConstantSDNodeBitfields) <= 2, "field too wide" ); |
605 | static_assert(sizeof(MemSDNodeBitfields) <= 2, "field too wide" ); |
606 | static_assert(sizeof(LSBaseSDNodeBitfields) <= 2, "field too wide" ); |
607 | static_assert(sizeof(LoadSDNodeBitfields) <= 2, "field too wide" ); |
608 | static_assert(sizeof(StoreSDNodeBitfields) <= 2, "field too wide" ); |
609 | |
610 | private: |
611 | friend class SelectionDAG; |
612 | // TODO: unfriend HandleSDNode once we fix its operand handling. |
613 | friend class HandleSDNode; |
614 | |
615 | /// Unique id per SDNode in the DAG. |
616 | int NodeId = -1; |
617 | |
618 | /// The values that are used by this operation. |
619 | SDUse *OperandList = nullptr; |
620 | |
621 | /// The types of the values this node defines. SDNode's may |
622 | /// define multiple values simultaneously. |
623 | const EVT *ValueList; |
624 | |
625 | /// List of uses for this SDNode. |
626 | SDUse *UseList = nullptr; |
627 | |
628 | /// The number of entries in the Operand/Value list. |
629 | unsigned short NumOperands = 0; |
630 | unsigned short NumValues; |
631 | |
632 | // The ordering of the SDNodes. It roughly corresponds to the ordering of the |
633 | // original LLVM instructions. |
634 | // This is used for turning off scheduling, because we'll forgo |
635 | // the normal scheduling algorithms and output the instructions according to |
636 | // this ordering. |
637 | unsigned IROrder; |
638 | |
639 | /// Source line information. |
640 | DebugLoc debugLoc; |
641 | |
642 | /// Return a pointer to the specified value type. |
643 | static const EVT *getValueTypeList(EVT VT); |
644 | |
645 | SDNodeFlags Flags; |
646 | |
647 | uint32_t CFIType = 0; |
648 | |
649 | public: |
650 | //===--------------------------------------------------------------------===// |
651 | // Accessors |
652 | // |
653 | |
654 | /// Return the SelectionDAG opcode value for this node. For |
655 | /// pre-isel nodes (those for which isMachineOpcode returns false), these |
656 | /// are the opcode values in the ISD and <target>ISD namespaces. For |
657 | /// post-isel opcodes, see getMachineOpcode. |
658 | unsigned getOpcode() const { return (unsigned)NodeType; } |
659 | |
660 | /// Test if this node has a target-specific opcode (in the |
661 | /// \<target\>ISD namespace). |
662 | bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; } |
663 | |
664 | /// Test if this node has a target-specific opcode that may raise |
665 | /// FP exceptions (in the \<target\>ISD namespace and greater than |
666 | /// FIRST_TARGET_STRICTFP_OPCODE). Note that all target memory |
667 | /// opcode are currently automatically considered to possibly raise |
668 | /// FP exceptions as well. |
669 | bool isTargetStrictFPOpcode() const { |
670 | return NodeType >= ISD::FIRST_TARGET_STRICTFP_OPCODE; |
671 | } |
672 | |
673 | /// Test if this node has a target-specific |
674 | /// memory-referencing opcode (in the \<target\>ISD namespace and |
675 | /// greater than FIRST_TARGET_MEMORY_OPCODE). |
676 | bool isTargetMemoryOpcode() const { |
677 | return NodeType >= ISD::FIRST_TARGET_MEMORY_OPCODE; |
678 | } |
679 | |
680 | /// Return true if the type of the node type undefined. |
681 | bool isUndef() const { return NodeType == ISD::UNDEF; } |
682 | |
683 | /// Test if this node is a memory intrinsic (with valid pointer information). |
684 | /// INTRINSIC_W_CHAIN and INTRINSIC_VOID nodes are sometimes created for |
685 | /// non-memory intrinsics (with chains) that are not really instances of |
686 | /// MemSDNode. For such nodes, we need some extra state to determine the |
687 | /// proper classof relationship. |
688 | bool isMemIntrinsic() const { |
689 | return (NodeType == ISD::INTRINSIC_W_CHAIN || |
690 | NodeType == ISD::INTRINSIC_VOID) && |
691 | SDNodeBits.IsMemIntrinsic; |
692 | } |
693 | |
694 | /// Test if this node is a strict floating point pseudo-op. |
695 | bool isStrictFPOpcode() { |
696 | switch (NodeType) { |
697 | default: |
698 | return false; |
699 | case ISD::STRICT_FP16_TO_FP: |
700 | case ISD::STRICT_FP_TO_FP16: |
701 | #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ |
702 | case ISD::STRICT_##DAGN: |
703 | #include "llvm/IR/ConstrainedOps.def" |
704 | return true; |
705 | } |
706 | } |
707 | |
708 | /// Test if this node is a vector predication operation. |
709 | bool isVPOpcode() const { return ISD::isVPOpcode(Opcode: getOpcode()); } |
710 | |
711 | /// Test if this node has a post-isel opcode, directly |
712 | /// corresponding to a MachineInstr opcode. |
713 | bool isMachineOpcode() const { return NodeType < 0; } |
714 | |
715 | /// This may only be called if isMachineOpcode returns |
716 | /// true. It returns the MachineInstr opcode value that the node's opcode |
717 | /// corresponds to. |
718 | unsigned getMachineOpcode() const { |
719 | assert(isMachineOpcode() && "Not a MachineInstr opcode!" ); |
720 | return ~NodeType; |
721 | } |
722 | |
723 | bool getHasDebugValue() const { return SDNodeBits.HasDebugValue; } |
724 | void setHasDebugValue(bool b) { SDNodeBits.HasDebugValue = b; } |
725 | |
726 | bool isDivergent() const { return SDNodeBits.IsDivergent; } |
727 | |
728 | /// Return true if there are no uses of this node. |
729 | bool use_empty() const { return UseList == nullptr; } |
730 | |
731 | /// Return true if there is exactly one use of this node. |
732 | bool hasOneUse() const { return hasSingleElement(C: uses()); } |
733 | |
734 | /// Return the number of uses of this node. This method takes |
735 | /// time proportional to the number of uses. |
736 | size_t use_size() const { return std::distance(first: use_begin(), last: use_end()); } |
737 | |
738 | /// Return the unique node id. |
739 | int getNodeId() const { return NodeId; } |
740 | |
741 | /// Set unique node id. |
742 | void setNodeId(int Id) { NodeId = Id; } |
743 | |
744 | /// Return the node ordering. |
745 | unsigned getIROrder() const { return IROrder; } |
746 | |
747 | /// Set the node ordering. |
748 | void setIROrder(unsigned Order) { IROrder = Order; } |
749 | |
750 | /// Return the source location info. |
751 | const DebugLoc &getDebugLoc() const { return debugLoc; } |
752 | |
753 | /// Set source location info. Try to avoid this, putting |
754 | /// it in the constructor is preferable. |
755 | void setDebugLoc(DebugLoc dl) { debugLoc = std::move(dl); } |
756 | |
757 | /// This class provides iterator support for SDUse |
758 | /// operands that use a specific SDNode. |
759 | class use_iterator { |
760 | friend class SDNode; |
761 | |
762 | SDUse *Op = nullptr; |
763 | |
764 | explicit use_iterator(SDUse *op) : Op(op) {} |
765 | |
766 | public: |
767 | using iterator_category = std::forward_iterator_tag; |
768 | using value_type = SDUse; |
769 | using difference_type = std::ptrdiff_t; |
770 | using pointer = value_type *; |
771 | using reference = value_type &; |
772 | |
773 | use_iterator() = default; |
774 | use_iterator(const use_iterator &I) = default; |
775 | use_iterator &operator=(const use_iterator &) = default; |
776 | |
777 | bool operator==(const use_iterator &x) const { return Op == x.Op; } |
778 | bool operator!=(const use_iterator &x) const { |
779 | return !operator==(x); |
780 | } |
781 | |
782 | /// Return true if this iterator is at the end of uses list. |
783 | bool atEnd() const { return Op == nullptr; } |
784 | |
785 | // Iterator traversal: forward iteration only. |
786 | use_iterator &operator++() { // Preincrement |
787 | assert(Op && "Cannot increment end iterator!" ); |
788 | Op = Op->getNext(); |
789 | return *this; |
790 | } |
791 | |
792 | use_iterator operator++(int) { // Postincrement |
793 | use_iterator tmp = *this; ++*this; return tmp; |
794 | } |
795 | |
796 | /// Retrieve a pointer to the current user node. |
797 | SDNode *operator*() const { |
798 | assert(Op && "Cannot dereference end iterator!" ); |
799 | return Op->getUser(); |
800 | } |
801 | |
802 | SDNode *operator->() const { return operator*(); } |
803 | |
804 | SDUse &getUse() const { return *Op; } |
805 | |
806 | /// Retrieve the operand # of this use in its user. |
807 | unsigned getOperandNo() const { |
808 | assert(Op && "Cannot dereference end iterator!" ); |
809 | return (unsigned)(Op - Op->getUser()->OperandList); |
810 | } |
811 | }; |
812 | |
813 | /// Provide iteration support to walk over all uses of an SDNode. |
814 | use_iterator use_begin() const { |
815 | return use_iterator(UseList); |
816 | } |
817 | |
818 | static use_iterator use_end() { return use_iterator(nullptr); } |
819 | |
820 | inline iterator_range<use_iterator> uses() { |
821 | return make_range(x: use_begin(), y: use_end()); |
822 | } |
823 | inline iterator_range<use_iterator> uses() const { |
824 | return make_range(x: use_begin(), y: use_end()); |
825 | } |
826 | |
827 | /// Return true if there are exactly NUSES uses of the indicated value. |
828 | /// This method ignores uses of other values defined by this operation. |
829 | bool hasNUsesOfValue(unsigned NUses, unsigned Value) const; |
830 | |
831 | /// Return true if there are any use of the indicated value. |
832 | /// This method ignores uses of other values defined by this operation. |
833 | bool hasAnyUseOfValue(unsigned Value) const; |
834 | |
835 | /// Return true if this node is the only use of N. |
836 | bool isOnlyUserOf(const SDNode *N) const; |
837 | |
838 | /// Return true if this node is an operand of N. |
839 | bool isOperandOf(const SDNode *N) const; |
840 | |
841 | /// Return true if this node is a predecessor of N. |
842 | /// NOTE: Implemented on top of hasPredecessor and every bit as |
843 | /// expensive. Use carefully. |
844 | bool isPredecessorOf(const SDNode *N) const { |
845 | return N->hasPredecessor(N: this); |
846 | } |
847 | |
848 | /// Return true if N is a predecessor of this node. |
849 | /// N is either an operand of this node, or can be reached by recursively |
850 | /// traversing up the operands. |
851 | /// NOTE: This is an expensive method. Use it carefully. |
852 | bool hasPredecessor(const SDNode *N) const; |
853 | |
854 | /// Returns true if N is a predecessor of any node in Worklist. This |
855 | /// helper keeps Visited and Worklist sets externally to allow unions |
856 | /// searches to be performed in parallel, caching of results across |
857 | /// queries and incremental addition to Worklist. Stops early if N is |
858 | /// found but will resume. Remember to clear Visited and Worklists |
859 | /// if DAG changes. MaxSteps gives a maximum number of nodes to visit before |
860 | /// giving up. The TopologicalPrune flag signals that positive NodeIds are |
861 | /// topologically ordered (Operands have strictly smaller node id) and search |
862 | /// can be pruned leveraging this. |
863 | static bool hasPredecessorHelper(const SDNode *N, |
864 | SmallPtrSetImpl<const SDNode *> &Visited, |
865 | SmallVectorImpl<const SDNode *> &Worklist, |
866 | unsigned int MaxSteps = 0, |
867 | bool TopologicalPrune = false) { |
868 | SmallVector<const SDNode *, 8> DeferredNodes; |
869 | if (Visited.count(Ptr: N)) |
870 | return true; |
871 | |
872 | // Node Id's are assigned in three places: As a topological |
873 | // ordering (> 0), during legalization (results in values set to |
874 | // 0), new nodes (set to -1). If N has a topolgical id then we |
875 | // know that all nodes with ids smaller than it cannot be |
876 | // successors and we need not check them. Filter out all node |
877 | // that can't be matches. We add them to the worklist before exit |
878 | // in case of multiple calls. Note that during selection the topological id |
879 | // may be violated if a node's predecessor is selected before it. We mark |
880 | // this at selection negating the id of unselected successors and |
881 | // restricting topological pruning to positive ids. |
882 | |
883 | int NId = N->getNodeId(); |
884 | // If we Invalidated the Id, reconstruct original NId. |
885 | if (NId < -1) |
886 | NId = -(NId + 1); |
887 | |
888 | bool Found = false; |
889 | while (!Worklist.empty()) { |
890 | const SDNode *M = Worklist.pop_back_val(); |
891 | int MId = M->getNodeId(); |
892 | if (TopologicalPrune && M->getOpcode() != ISD::TokenFactor && (NId > 0) && |
893 | (MId > 0) && (MId < NId)) { |
894 | DeferredNodes.push_back(Elt: M); |
895 | continue; |
896 | } |
897 | for (const SDValue &OpV : M->op_values()) { |
898 | SDNode *Op = OpV.getNode(); |
899 | if (Visited.insert(Ptr: Op).second) |
900 | Worklist.push_back(Elt: Op); |
901 | if (Op == N) |
902 | Found = true; |
903 | } |
904 | if (Found) |
905 | break; |
906 | if (MaxSteps != 0 && Visited.size() >= MaxSteps) |
907 | break; |
908 | } |
909 | // Push deferred nodes back on worklist. |
910 | Worklist.append(in_start: DeferredNodes.begin(), in_end: DeferredNodes.end()); |
911 | // If we bailed early, conservatively return found. |
912 | if (MaxSteps != 0 && Visited.size() >= MaxSteps) |
913 | return true; |
914 | return Found; |
915 | } |
916 | |
917 | /// Return true if all the users of N are contained in Nodes. |
918 | /// NOTE: Requires at least one match, but doesn't require them all. |
919 | static bool areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N); |
920 | |
921 | /// Return the number of values used by this operation. |
922 | unsigned getNumOperands() const { return NumOperands; } |
923 | |
924 | /// Return the maximum number of operands that a SDNode can hold. |
925 | static constexpr size_t getMaxNumOperands() { |
926 | return std::numeric_limits<decltype(SDNode::NumOperands)>::max(); |
927 | } |
928 | |
929 | /// Helper method returns the integer value of a ConstantSDNode operand. |
930 | inline uint64_t getConstantOperandVal(unsigned Num) const; |
931 | |
932 | /// Helper method returns the zero-extended integer value of a ConstantSDNode. |
933 | inline uint64_t getAsZExtVal() const; |
934 | |
935 | /// Helper method returns the APInt of a ConstantSDNode operand. |
936 | inline const APInt &getConstantOperandAPInt(unsigned Num) const; |
937 | |
938 | /// Helper method returns the APInt value of a ConstantSDNode. |
939 | inline const APInt &getAsAPIntVal() const; |
940 | |
941 | const SDValue &getOperand(unsigned Num) const { |
942 | assert(Num < NumOperands && "Invalid child # of SDNode!" ); |
943 | return OperandList[Num]; |
944 | } |
945 | |
946 | using op_iterator = SDUse *; |
947 | |
948 | op_iterator op_begin() const { return OperandList; } |
949 | op_iterator op_end() const { return OperandList+NumOperands; } |
950 | ArrayRef<SDUse> ops() const { return ArrayRef(op_begin(), op_end()); } |
951 | |
952 | /// Iterator for directly iterating over the operand SDValue's. |
953 | struct value_op_iterator |
954 | : iterator_adaptor_base<value_op_iterator, op_iterator, |
955 | std::random_access_iterator_tag, SDValue, |
956 | ptrdiff_t, value_op_iterator *, |
957 | value_op_iterator *> { |
958 | explicit value_op_iterator(SDUse *U = nullptr) |
959 | : iterator_adaptor_base(U) {} |
960 | |
961 | const SDValue &operator*() const { return I->get(); } |
962 | }; |
963 | |
964 | iterator_range<value_op_iterator> op_values() const { |
965 | return make_range(x: value_op_iterator(op_begin()), |
966 | y: value_op_iterator(op_end())); |
967 | } |
968 | |
969 | SDVTList getVTList() const { |
970 | SDVTList X = { .VTs: ValueList, .NumVTs: NumValues }; |
971 | return X; |
972 | } |
973 | |
974 | /// If this node has a glue operand, return the node |
975 | /// to which the glue operand points. Otherwise return NULL. |
976 | SDNode *getGluedNode() const { |
977 | if (getNumOperands() != 0 && |
978 | getOperand(getNumOperands()-1).getValueType() == MVT::Glue) |
979 | return getOperand(Num: getNumOperands()-1).getNode(); |
980 | return nullptr; |
981 | } |
982 | |
983 | /// If this node has a glue value with a user, return |
984 | /// the user (there is at most one). Otherwise return NULL. |
985 | SDNode *getGluedUser() const { |
986 | for (use_iterator UI = use_begin(), UE = use_end(); UI != UE; ++UI) |
987 | if (UI.getUse().get().getValueType() == MVT::Glue) |
988 | return *UI; |
989 | return nullptr; |
990 | } |
991 | |
992 | SDNodeFlags getFlags() const { return Flags; } |
993 | void setFlags(SDNodeFlags NewFlags) { Flags = NewFlags; } |
994 | |
995 | /// Clear any flags in this node that aren't also set in Flags. |
996 | /// If Flags is not in a defined state then this has no effect. |
997 | void intersectFlagsWith(const SDNodeFlags Flags); |
998 | |
999 | void setCFIType(uint32_t Type) { CFIType = Type; } |
1000 | uint32_t getCFIType() const { return CFIType; } |
1001 | |
1002 | /// Return the number of values defined/returned by this operator. |
1003 | unsigned getNumValues() const { return NumValues; } |
1004 | |
1005 | /// Return the type of a specified result. |
1006 | EVT getValueType(unsigned ResNo) const { |
1007 | assert(ResNo < NumValues && "Illegal result number!" ); |
1008 | return ValueList[ResNo]; |
1009 | } |
1010 | |
1011 | /// Return the type of a specified result as a simple type. |
1012 | MVT getSimpleValueType(unsigned ResNo) const { |
1013 | return getValueType(ResNo).getSimpleVT(); |
1014 | } |
1015 | |
1016 | /// Returns MVT::getSizeInBits(getValueType(ResNo)). |
1017 | /// |
1018 | /// If the value type is a scalable vector type, the scalable property will |
1019 | /// be set and the runtime size will be a positive integer multiple of the |
1020 | /// base size. |
1021 | TypeSize getValueSizeInBits(unsigned ResNo) const { |
1022 | return getValueType(ResNo).getSizeInBits(); |
1023 | } |
1024 | |
1025 | using value_iterator = const EVT *; |
1026 | |
1027 | value_iterator value_begin() const { return ValueList; } |
1028 | value_iterator value_end() const { return ValueList+NumValues; } |
1029 | iterator_range<value_iterator> values() const { |
1030 | return llvm::make_range(x: value_begin(), y: value_end()); |
1031 | } |
1032 | |
1033 | /// Return the opcode of this operation for printing. |
1034 | std::string getOperationName(const SelectionDAG *G = nullptr) const; |
1035 | static const char* getIndexedModeName(ISD::MemIndexedMode AM); |
1036 | void print_types(raw_ostream &OS, const SelectionDAG *G) const; |
1037 | void print_details(raw_ostream &OS, const SelectionDAG *G) const; |
1038 | void print(raw_ostream &OS, const SelectionDAG *G = nullptr) const; |
1039 | void printr(raw_ostream &OS, const SelectionDAG *G = nullptr) const; |
1040 | |
1041 | /// Print a SelectionDAG node and all children down to |
1042 | /// the leaves. The given SelectionDAG allows target-specific nodes |
1043 | /// to be printed in human-readable form. Unlike printr, this will |
1044 | /// print the whole DAG, including children that appear multiple |
1045 | /// times. |
1046 | /// |
1047 | void printrFull(raw_ostream &O, const SelectionDAG *G = nullptr) const; |
1048 | |
1049 | /// Print a SelectionDAG node and children up to |
1050 | /// depth "depth." The given SelectionDAG allows target-specific |
1051 | /// nodes to be printed in human-readable form. Unlike printr, this |
1052 | /// will print children that appear multiple times wherever they are |
1053 | /// used. |
1054 | /// |
1055 | void printrWithDepth(raw_ostream &O, const SelectionDAG *G = nullptr, |
1056 | unsigned depth = 100) const; |
1057 | |
1058 | /// Dump this node, for debugging. |
1059 | void dump() const; |
1060 | |
1061 | /// Dump (recursively) this node and its use-def subgraph. |
1062 | void dumpr() const; |
1063 | |
1064 | /// Dump this node, for debugging. |
1065 | /// The given SelectionDAG allows target-specific nodes to be printed |
1066 | /// in human-readable form. |
1067 | void dump(const SelectionDAG *G) const; |
1068 | |
1069 | /// Dump (recursively) this node and its use-def subgraph. |
1070 | /// The given SelectionDAG allows target-specific nodes to be printed |
1071 | /// in human-readable form. |
1072 | void dumpr(const SelectionDAG *G) const; |
1073 | |
1074 | /// printrFull to dbgs(). The given SelectionDAG allows |
1075 | /// target-specific nodes to be printed in human-readable form. |
1076 | /// Unlike dumpr, this will print the whole DAG, including children |
1077 | /// that appear multiple times. |
1078 | void dumprFull(const SelectionDAG *G = nullptr) const; |
1079 | |
1080 | /// printrWithDepth to dbgs(). The given |
1081 | /// SelectionDAG allows target-specific nodes to be printed in |
1082 | /// human-readable form. Unlike dumpr, this will print children |
1083 | /// that appear multiple times wherever they are used. |
1084 | /// |
1085 | void dumprWithDepth(const SelectionDAG *G = nullptr, |
1086 | unsigned depth = 100) const; |
1087 | |
1088 | /// Gather unique data for the node. |
1089 | void Profile(FoldingSetNodeID &ID) const; |
1090 | |
1091 | /// This method should only be used by the SDUse class. |
1092 | void addUse(SDUse &U) { U.addToList(List: &UseList); } |
1093 | |
1094 | protected: |
1095 | static SDVTList getSDVTList(EVT VT) { |
1096 | SDVTList Ret = { .VTs: getValueTypeList(VT), .NumVTs: 1 }; |
1097 | return Ret; |
1098 | } |
1099 | |
1100 | /// Create an SDNode. |
1101 | /// |
1102 | /// SDNodes are created without any operands, and never own the operand |
1103 | /// storage. To add operands, see SelectionDAG::createOperands. |
1104 | SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs) |
1105 | : NodeType(Opc), ValueList(VTs.VTs), NumValues(VTs.NumVTs), |
1106 | IROrder(Order), debugLoc(std::move(dl)) { |
1107 | memset(s: &RawSDNodeBits, c: 0, n: sizeof(RawSDNodeBits)); |
1108 | assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor" ); |
1109 | assert(NumValues == VTs.NumVTs && |
1110 | "NumValues wasn't wide enough for its operands!" ); |
1111 | } |
1112 | |
1113 | /// Release the operands and set this node to have zero operands. |
1114 | void DropOperands(); |
1115 | }; |
1116 | |
1117 | /// Wrapper class for IR location info (IR ordering and DebugLoc) to be passed |
1118 | /// into SDNode creation functions. |
1119 | /// When an SDNode is created from the DAGBuilder, the DebugLoc is extracted |
1120 | /// from the original Instruction, and IROrder is the ordinal position of |
1121 | /// the instruction. |
1122 | /// When an SDNode is created after the DAG is being built, both DebugLoc and |
1123 | /// the IROrder are propagated from the original SDNode. |
1124 | /// So SDLoc class provides two constructors besides the default one, one to |
1125 | /// be used by the DAGBuilder, the other to be used by others. |
1126 | class SDLoc { |
1127 | private: |
1128 | DebugLoc DL; |
1129 | int IROrder = 0; |
1130 | |
1131 | public: |
1132 | SDLoc() = default; |
1133 | SDLoc(const SDNode *N) : DL(N->getDebugLoc()), IROrder(N->getIROrder()) {} |
1134 | SDLoc(const SDValue V) : SDLoc(V.getNode()) {} |
1135 | SDLoc(const Instruction *I, int Order) : IROrder(Order) { |
1136 | assert(Order >= 0 && "bad IROrder" ); |
1137 | if (I) |
1138 | DL = I->getDebugLoc(); |
1139 | } |
1140 | |
1141 | unsigned getIROrder() const { return IROrder; } |
1142 | const DebugLoc &getDebugLoc() const { return DL; } |
1143 | }; |
1144 | |
1145 | // Define inline functions from the SDValue class. |
1146 | |
1147 | inline SDValue::SDValue(SDNode *node, unsigned resno) |
1148 | : Node(node), ResNo(resno) { |
1149 | // Explicitly check for !ResNo to avoid use-after-free, because there are |
1150 | // callers that use SDValue(N, 0) with a deleted N to indicate successful |
1151 | // combines. |
1152 | assert((!Node || !ResNo || ResNo < Node->getNumValues()) && |
1153 | "Invalid result number for the given node!" ); |
1154 | assert(ResNo < -2U && "Cannot use result numbers reserved for DenseMaps." ); |
1155 | } |
1156 | |
1157 | inline unsigned SDValue::getOpcode() const { |
1158 | return Node->getOpcode(); |
1159 | } |
1160 | |
1161 | inline EVT SDValue::getValueType() const { |
1162 | return Node->getValueType(ResNo); |
1163 | } |
1164 | |
1165 | inline unsigned SDValue::getNumOperands() const { |
1166 | return Node->getNumOperands(); |
1167 | } |
1168 | |
1169 | inline const SDValue &SDValue::getOperand(unsigned i) const { |
1170 | return Node->getOperand(Num: i); |
1171 | } |
1172 | |
1173 | inline uint64_t SDValue::getConstantOperandVal(unsigned i) const { |
1174 | return Node->getConstantOperandVal(Num: i); |
1175 | } |
1176 | |
1177 | inline const APInt &SDValue::getConstantOperandAPInt(unsigned i) const { |
1178 | return Node->getConstantOperandAPInt(Num: i); |
1179 | } |
1180 | |
1181 | inline bool SDValue::isTargetOpcode() const { |
1182 | return Node->isTargetOpcode(); |
1183 | } |
1184 | |
1185 | inline bool SDValue::isTargetMemoryOpcode() const { |
1186 | return Node->isTargetMemoryOpcode(); |
1187 | } |
1188 | |
1189 | inline bool SDValue::isMachineOpcode() const { |
1190 | return Node->isMachineOpcode(); |
1191 | } |
1192 | |
1193 | inline unsigned SDValue::getMachineOpcode() const { |
1194 | return Node->getMachineOpcode(); |
1195 | } |
1196 | |
1197 | inline bool SDValue::isUndef() const { |
1198 | return Node->isUndef(); |
1199 | } |
1200 | |
1201 | inline bool SDValue::use_empty() const { |
1202 | return !Node->hasAnyUseOfValue(Value: ResNo); |
1203 | } |
1204 | |
1205 | inline bool SDValue::hasOneUse() const { |
1206 | return Node->hasNUsesOfValue(NUses: 1, Value: ResNo); |
1207 | } |
1208 | |
1209 | inline const DebugLoc &SDValue::getDebugLoc() const { |
1210 | return Node->getDebugLoc(); |
1211 | } |
1212 | |
1213 | inline void SDValue::dump() const { |
1214 | return Node->dump(); |
1215 | } |
1216 | |
1217 | inline void SDValue::dump(const SelectionDAG *G) const { |
1218 | return Node->dump(G); |
1219 | } |
1220 | |
1221 | inline void SDValue::dumpr() const { |
1222 | return Node->dumpr(); |
1223 | } |
1224 | |
1225 | inline void SDValue::dumpr(const SelectionDAG *G) const { |
1226 | return Node->dumpr(G); |
1227 | } |
1228 | |
1229 | // Define inline functions from the SDUse class. |
1230 | |
1231 | inline void SDUse::set(const SDValue &V) { |
1232 | if (Val.getNode()) removeFromList(); |
1233 | Val = V; |
1234 | if (V.getNode()) |
1235 | V->addUse(U&: *this); |
1236 | } |
1237 | |
1238 | inline void SDUse::setInitial(const SDValue &V) { |
1239 | Val = V; |
1240 | V->addUse(U&: *this); |
1241 | } |
1242 | |
1243 | inline void SDUse::setNode(SDNode *N) { |
1244 | if (Val.getNode()) removeFromList(); |
1245 | Val.setNode(N); |
1246 | if (N) N->addUse(U&: *this); |
1247 | } |
1248 | |
1249 | /// This class is used to form a handle around another node that |
1250 | /// is persistent and is updated across invocations of replaceAllUsesWith on its |
1251 | /// operand. This node should be directly created by end-users and not added to |
1252 | /// the AllNodes list. |
1253 | class HandleSDNode : public SDNode { |
1254 | SDUse Op; |
1255 | |
1256 | public: |
1257 | explicit HandleSDNode(SDValue X) |
1258 | : SDNode(ISD::HANDLENODE, 0, DebugLoc(), getSDVTList(MVT::Other)) { |
1259 | // HandleSDNodes are never inserted into the DAG, so they won't be |
1260 | // auto-numbered. Use ID 65535 as a sentinel. |
1261 | PersistentId = 0xffff; |
1262 | |
1263 | // Manually set up the operand list. This node type is special in that it's |
1264 | // always stack allocated and SelectionDAG does not manage its operands. |
1265 | // TODO: This should either (a) not be in the SDNode hierarchy, or (b) not |
1266 | // be so special. |
1267 | Op.setUser(this); |
1268 | Op.setInitial(X); |
1269 | NumOperands = 1; |
1270 | OperandList = &Op; |
1271 | } |
1272 | ~HandleSDNode(); |
1273 | |
1274 | const SDValue &getValue() const { return Op; } |
1275 | }; |
1276 | |
1277 | class AddrSpaceCastSDNode : public SDNode { |
1278 | private: |
1279 | unsigned SrcAddrSpace; |
1280 | unsigned DestAddrSpace; |
1281 | |
1282 | public: |
1283 | AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, EVT VT, |
1284 | unsigned SrcAS, unsigned DestAS); |
1285 | |
1286 | unsigned getSrcAddressSpace() const { return SrcAddrSpace; } |
1287 | unsigned getDestAddressSpace() const { return DestAddrSpace; } |
1288 | |
1289 | static bool classof(const SDNode *N) { |
1290 | return N->getOpcode() == ISD::ADDRSPACECAST; |
1291 | } |
1292 | }; |
1293 | |
1294 | /// This is an abstract virtual class for memory operations. |
1295 | class MemSDNode : public SDNode { |
1296 | private: |
1297 | // VT of in-memory value. |
1298 | EVT MemoryVT; |
1299 | |
1300 | protected: |
1301 | /// Memory reference information. |
1302 | MachineMemOperand *MMO; |
1303 | |
1304 | public: |
1305 | MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs, |
1306 | EVT memvt, MachineMemOperand *MMO); |
1307 | |
1308 | bool readMem() const { return MMO->isLoad(); } |
1309 | bool writeMem() const { return MMO->isStore(); } |
1310 | |
1311 | /// Returns alignment and volatility of the memory access |
1312 | Align getOriginalAlign() const { return MMO->getBaseAlign(); } |
1313 | Align getAlign() const { return MMO->getAlign(); } |
1314 | |
1315 | /// Return the SubclassData value, without HasDebugValue. This contains an |
1316 | /// encoding of the volatile flag, as well as bits used by subclasses. This |
1317 | /// function should only be used to compute a FoldingSetNodeID value. |
1318 | /// The HasDebugValue bit is masked out because CSE map needs to match |
1319 | /// nodes with debug info with nodes without debug info. Same is about |
1320 | /// isDivergent bit. |
1321 | unsigned getRawSubclassData() const { |
1322 | uint16_t Data; |
1323 | union { |
1324 | char RawSDNodeBits[sizeof(uint16_t)]; |
1325 | SDNodeBitfields SDNodeBits; |
1326 | }; |
1327 | memcpy(dest: &RawSDNodeBits, src: &this->RawSDNodeBits, n: sizeof(this->RawSDNodeBits)); |
1328 | SDNodeBits.HasDebugValue = 0; |
1329 | SDNodeBits.IsDivergent = false; |
1330 | memcpy(dest: &Data, src: &RawSDNodeBits, n: sizeof(RawSDNodeBits)); |
1331 | return Data; |
1332 | } |
1333 | |
1334 | bool isVolatile() const { return MemSDNodeBits.IsVolatile; } |
1335 | bool isNonTemporal() const { return MemSDNodeBits.IsNonTemporal; } |
1336 | bool isDereferenceable() const { return MemSDNodeBits.IsDereferenceable; } |
1337 | bool isInvariant() const { return MemSDNodeBits.IsInvariant; } |
1338 | |
1339 | // Returns the offset from the location of the access. |
1340 | int64_t getSrcValueOffset() const { return MMO->getOffset(); } |
1341 | |
1342 | /// Returns the AA info that describes the dereference. |
1343 | AAMDNodes getAAInfo() const { return MMO->getAAInfo(); } |
1344 | |
1345 | /// Returns the Ranges that describes the dereference. |
1346 | const MDNode *getRanges() const { return MMO->getRanges(); } |
1347 | |
1348 | /// Returns the synchronization scope ID for this memory operation. |
1349 | SyncScope::ID getSyncScopeID() const { return MMO->getSyncScopeID(); } |
1350 | |
1351 | /// Return the atomic ordering requirements for this memory operation. For |
1352 | /// cmpxchg atomic operations, return the atomic ordering requirements when |
1353 | /// store occurs. |
1354 | AtomicOrdering getSuccessOrdering() const { |
1355 | return MMO->getSuccessOrdering(); |
1356 | } |
1357 | |
1358 | /// Return a single atomic ordering that is at least as strong as both the |
1359 | /// success and failure orderings for an atomic operation. (For operations |
1360 | /// other than cmpxchg, this is equivalent to getSuccessOrdering().) |
1361 | AtomicOrdering getMergedOrdering() const { return MMO->getMergedOrdering(); } |
1362 | |
1363 | /// Return true if the memory operation ordering is Unordered or higher. |
1364 | bool isAtomic() const { return MMO->isAtomic(); } |
1365 | |
1366 | /// Returns true if the memory operation doesn't imply any ordering |
1367 | /// constraints on surrounding memory operations beyond the normal memory |
1368 | /// aliasing rules. |
1369 | bool isUnordered() const { return MMO->isUnordered(); } |
1370 | |
1371 | /// Returns true if the memory operation is neither atomic or volatile. |
1372 | bool isSimple() const { return !isAtomic() && !isVolatile(); } |
1373 | |
1374 | /// Return the type of the in-memory value. |
1375 | EVT getMemoryVT() const { return MemoryVT; } |
1376 | |
1377 | /// Return a MachineMemOperand object describing the memory |
1378 | /// reference performed by operation. |
1379 | MachineMemOperand *getMemOperand() const { return MMO; } |
1380 | |
1381 | const MachinePointerInfo &getPointerInfo() const { |
1382 | return MMO->getPointerInfo(); |
1383 | } |
1384 | |
1385 | /// Return the address space for the associated pointer |
1386 | unsigned getAddressSpace() const { |
1387 | return getPointerInfo().getAddrSpace(); |
1388 | } |
1389 | |
1390 | /// Update this MemSDNode's MachineMemOperand information |
1391 | /// to reflect the alignment of NewMMO, if it has a greater alignment. |
1392 | /// This must only be used when the new alignment applies to all users of |
1393 | /// this MachineMemOperand. |
1394 | void refineAlignment(const MachineMemOperand *NewMMO) { |
1395 | MMO->refineAlignment(MMO: NewMMO); |
1396 | } |
1397 | |
1398 | const SDValue &getChain() const { return getOperand(Num: 0); } |
1399 | |
1400 | const SDValue &getBasePtr() const { |
1401 | switch (getOpcode()) { |
1402 | case ISD::STORE: |
1403 | case ISD::ATOMIC_STORE: |
1404 | case ISD::VP_STORE: |
1405 | case ISD::MSTORE: |
1406 | case ISD::VP_SCATTER: |
1407 | case ISD::EXPERIMENTAL_VP_STRIDED_STORE: |
1408 | return getOperand(Num: 2); |
1409 | case ISD::MGATHER: |
1410 | case ISD::MSCATTER: |
1411 | return getOperand(Num: 3); |
1412 | default: |
1413 | return getOperand(Num: 1); |
1414 | } |
1415 | } |
1416 | |
1417 | // Methods to support isa and dyn_cast |
1418 | static bool classof(const SDNode *N) { |
1419 | // For some targets, we lower some target intrinsics to a MemIntrinsicNode |
1420 | // with either an intrinsic or a target opcode. |
1421 | switch (N->getOpcode()) { |
1422 | case ISD::LOAD: |
1423 | case ISD::STORE: |
1424 | case ISD::PREFETCH: |
1425 | case ISD::ATOMIC_CMP_SWAP: |
1426 | case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: |
1427 | case ISD::ATOMIC_SWAP: |
1428 | case ISD::ATOMIC_LOAD_ADD: |
1429 | case ISD::ATOMIC_LOAD_SUB: |
1430 | case ISD::ATOMIC_LOAD_AND: |
1431 | case ISD::ATOMIC_LOAD_CLR: |
1432 | case ISD::ATOMIC_LOAD_OR: |
1433 | case ISD::ATOMIC_LOAD_XOR: |
1434 | case ISD::ATOMIC_LOAD_NAND: |
1435 | case ISD::ATOMIC_LOAD_MIN: |
1436 | case ISD::ATOMIC_LOAD_MAX: |
1437 | case ISD::ATOMIC_LOAD_UMIN: |
1438 | case ISD::ATOMIC_LOAD_UMAX: |
1439 | case ISD::ATOMIC_LOAD_FADD: |
1440 | case ISD::ATOMIC_LOAD_FSUB: |
1441 | case ISD::ATOMIC_LOAD_FMAX: |
1442 | case ISD::ATOMIC_LOAD_FMIN: |
1443 | case ISD::ATOMIC_LOAD_UINC_WRAP: |
1444 | case ISD::ATOMIC_LOAD_UDEC_WRAP: |
1445 | case ISD::ATOMIC_LOAD: |
1446 | case ISD::ATOMIC_STORE: |
1447 | case ISD::MLOAD: |
1448 | case ISD::MSTORE: |
1449 | case ISD::MGATHER: |
1450 | case ISD::MSCATTER: |
1451 | case ISD::VP_LOAD: |
1452 | case ISD::VP_STORE: |
1453 | case ISD::VP_GATHER: |
1454 | case ISD::VP_SCATTER: |
1455 | case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: |
1456 | case ISD::EXPERIMENTAL_VP_STRIDED_STORE: |
1457 | case ISD::GET_FPENV_MEM: |
1458 | case ISD::SET_FPENV_MEM: |
1459 | return true; |
1460 | default: |
1461 | return N->isMemIntrinsic() || N->isTargetMemoryOpcode(); |
1462 | } |
1463 | } |
1464 | }; |
1465 | |
1466 | /// This is an SDNode representing atomic operations. |
1467 | class AtomicSDNode : public MemSDNode { |
1468 | public: |
1469 | AtomicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTL, |
1470 | EVT MemVT, MachineMemOperand *MMO) |
1471 | : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) { |
1472 | assert(((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || |
1473 | MMO->isAtomic()) && "then why are we using an AtomicSDNode?" ); |
1474 | } |
1475 | |
1476 | const SDValue &getBasePtr() const { |
1477 | return getOpcode() == ISD::ATOMIC_STORE ? getOperand(Num: 2) : getOperand(Num: 1); |
1478 | } |
1479 | const SDValue &getVal() const { |
1480 | return getOpcode() == ISD::ATOMIC_STORE ? getOperand(Num: 1) : getOperand(Num: 2); |
1481 | } |
1482 | |
1483 | /// Returns true if this SDNode represents cmpxchg atomic operation, false |
1484 | /// otherwise. |
1485 | bool isCompareAndSwap() const { |
1486 | unsigned Op = getOpcode(); |
1487 | return Op == ISD::ATOMIC_CMP_SWAP || |
1488 | Op == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS; |
1489 | } |
1490 | |
1491 | /// For cmpxchg atomic operations, return the atomic ordering requirements |
1492 | /// when store does not occur. |
1493 | AtomicOrdering getFailureOrdering() const { |
1494 | assert(isCompareAndSwap() && "Must be cmpxchg operation" ); |
1495 | return MMO->getFailureOrdering(); |
1496 | } |
1497 | |
1498 | // Methods to support isa and dyn_cast |
1499 | static bool classof(const SDNode *N) { |
1500 | return N->getOpcode() == ISD::ATOMIC_CMP_SWAP || |
1501 | N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS || |
1502 | N->getOpcode() == ISD::ATOMIC_SWAP || |
1503 | N->getOpcode() == ISD::ATOMIC_LOAD_ADD || |
1504 | N->getOpcode() == ISD::ATOMIC_LOAD_SUB || |
1505 | N->getOpcode() == ISD::ATOMIC_LOAD_AND || |
1506 | N->getOpcode() == ISD::ATOMIC_LOAD_CLR || |
1507 | N->getOpcode() == ISD::ATOMIC_LOAD_OR || |
1508 | N->getOpcode() == ISD::ATOMIC_LOAD_XOR || |
1509 | N->getOpcode() == ISD::ATOMIC_LOAD_NAND || |
1510 | N->getOpcode() == ISD::ATOMIC_LOAD_MIN || |
1511 | N->getOpcode() == ISD::ATOMIC_LOAD_MAX || |
1512 | N->getOpcode() == ISD::ATOMIC_LOAD_UMIN || |
1513 | N->getOpcode() == ISD::ATOMIC_LOAD_UMAX || |
1514 | N->getOpcode() == ISD::ATOMIC_LOAD_FADD || |
1515 | N->getOpcode() == ISD::ATOMIC_LOAD_FSUB || |
1516 | N->getOpcode() == ISD::ATOMIC_LOAD_FMAX || |
1517 | N->getOpcode() == ISD::ATOMIC_LOAD_FMIN || |
1518 | N->getOpcode() == ISD::ATOMIC_LOAD_UINC_WRAP || |
1519 | N->getOpcode() == ISD::ATOMIC_LOAD_UDEC_WRAP || |
1520 | N->getOpcode() == ISD::ATOMIC_LOAD || |
1521 | N->getOpcode() == ISD::ATOMIC_STORE; |
1522 | } |
1523 | }; |
1524 | |
1525 | /// This SDNode is used for target intrinsics that touch |
1526 | /// memory and need an associated MachineMemOperand. Its opcode may be |
1527 | /// INTRINSIC_VOID, INTRINSIC_W_CHAIN, PREFETCH, or a target-specific opcode |
1528 | /// with a value not less than FIRST_TARGET_MEMORY_OPCODE. |
1529 | class MemIntrinsicSDNode : public MemSDNode { |
1530 | public: |
1531 | MemIntrinsicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, |
1532 | SDVTList VTs, EVT MemoryVT, MachineMemOperand *MMO) |
1533 | : MemSDNode(Opc, Order, dl, VTs, MemoryVT, MMO) { |
1534 | SDNodeBits.IsMemIntrinsic = true; |
1535 | } |
1536 | |
1537 | // Methods to support isa and dyn_cast |
1538 | static bool classof(const SDNode *N) { |
1539 | // We lower some target intrinsics to their target opcode |
1540 | // early a node with a target opcode can be of this class |
1541 | return N->isMemIntrinsic() || |
1542 | N->getOpcode() == ISD::PREFETCH || |
1543 | N->isTargetMemoryOpcode(); |
1544 | } |
1545 | }; |
1546 | |
1547 | /// This SDNode is used to implement the code generator |
1548 | /// support for the llvm IR shufflevector instruction. It combines elements |
1549 | /// from two input vectors into a new input vector, with the selection and |
1550 | /// ordering of elements determined by an array of integers, referred to as |
1551 | /// the shuffle mask. For input vectors of width N, mask indices of 0..N-1 |
1552 | /// refer to elements from the LHS input, and indices from N to 2N-1 the RHS. |
1553 | /// An index of -1 is treated as undef, such that the code generator may put |
1554 | /// any value in the corresponding element of the result. |
1555 | class ShuffleVectorSDNode : public SDNode { |
1556 | // The memory for Mask is owned by the SelectionDAG's OperandAllocator, and |
1557 | // is freed when the SelectionDAG object is destroyed. |
1558 | const int *Mask; |
1559 | |
1560 | protected: |
1561 | friend class SelectionDAG; |
1562 | |
1563 | ShuffleVectorSDNode(EVT VT, unsigned Order, const DebugLoc &dl, const int *M) |
1564 | : SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) {} |
1565 | |
1566 | public: |
1567 | ArrayRef<int> getMask() const { |
1568 | EVT VT = getValueType(ResNo: 0); |
1569 | return ArrayRef(Mask, VT.getVectorNumElements()); |
1570 | } |
1571 | |
1572 | int getMaskElt(unsigned Idx) const { |
1573 | assert(Idx < getValueType(0).getVectorNumElements() && "Idx out of range!" ); |
1574 | return Mask[Idx]; |
1575 | } |
1576 | |
1577 | bool isSplat() const { return isSplatMask(Mask, VT: getValueType(ResNo: 0)); } |
1578 | |
1579 | int getSplatIndex() const { |
1580 | assert(isSplat() && "Cannot get splat index for non-splat!" ); |
1581 | EVT VT = getValueType(ResNo: 0); |
1582 | for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) |
1583 | if (Mask[i] >= 0) |
1584 | return Mask[i]; |
1585 | |
1586 | // We can choose any index value here and be correct because all elements |
1587 | // are undefined. Return 0 for better potential for callers to simplify. |
1588 | return 0; |
1589 | } |
1590 | |
1591 | static bool isSplatMask(const int *Mask, EVT VT); |
1592 | |
1593 | /// Change values in a shuffle permute mask assuming |
1594 | /// the two vector operands have swapped position. |
1595 | static void commuteMask(MutableArrayRef<int> Mask) { |
1596 | unsigned NumElems = Mask.size(); |
1597 | for (unsigned i = 0; i != NumElems; ++i) { |
1598 | int idx = Mask[i]; |
1599 | if (idx < 0) |
1600 | continue; |
1601 | else if (idx < (int)NumElems) |
1602 | Mask[i] = idx + NumElems; |
1603 | else |
1604 | Mask[i] = idx - NumElems; |
1605 | } |
1606 | } |
1607 | |
1608 | static bool classof(const SDNode *N) { |
1609 | return N->getOpcode() == ISD::VECTOR_SHUFFLE; |
1610 | } |
1611 | }; |
1612 | |
1613 | class ConstantSDNode : public SDNode { |
1614 | friend class SelectionDAG; |
1615 | |
1616 | const ConstantInt *Value; |
1617 | |
1618 | ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, EVT VT) |
1619 | : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, 0, DebugLoc(), |
1620 | getSDVTList(VT)), |
1621 | Value(val) { |
1622 | ConstantSDNodeBits.IsOpaque = isOpaque; |
1623 | } |
1624 | |
1625 | public: |
1626 | const ConstantInt *getConstantIntValue() const { return Value; } |
1627 | const APInt &getAPIntValue() const { return Value->getValue(); } |
1628 | uint64_t getZExtValue() const { return Value->getZExtValue(); } |
1629 | int64_t getSExtValue() const { return Value->getSExtValue(); } |
1630 | uint64_t getLimitedValue(uint64_t Limit = UINT64_MAX) { |
1631 | return Value->getLimitedValue(Limit); |
1632 | } |
1633 | MaybeAlign getMaybeAlignValue() const { return Value->getMaybeAlignValue(); } |
1634 | Align getAlignValue() const { return Value->getAlignValue(); } |
1635 | |
1636 | bool isOne() const { return Value->isOne(); } |
1637 | bool isZero() const { return Value->isZero(); } |
1638 | bool isAllOnes() const { return Value->isMinusOne(); } |
1639 | bool isMaxSignedValue() const { return Value->isMaxValue(IsSigned: true); } |
1640 | bool isMinSignedValue() const { return Value->isMinValue(IsSigned: true); } |
1641 | |
1642 | bool isOpaque() const { return ConstantSDNodeBits.IsOpaque; } |
1643 | |
1644 | static bool classof(const SDNode *N) { |
1645 | return N->getOpcode() == ISD::Constant || |
1646 | N->getOpcode() == ISD::TargetConstant; |
1647 | } |
1648 | }; |
1649 | |
1650 | uint64_t SDNode::getConstantOperandVal(unsigned Num) const { |
1651 | return cast<ConstantSDNode>(Val: getOperand(Num))->getZExtValue(); |
1652 | } |
1653 | |
1654 | uint64_t SDNode::getAsZExtVal() const { |
1655 | return cast<ConstantSDNode>(Val: this)->getZExtValue(); |
1656 | } |
1657 | |
1658 | const APInt &SDNode::getConstantOperandAPInt(unsigned Num) const { |
1659 | return cast<ConstantSDNode>(Val: getOperand(Num))->getAPIntValue(); |
1660 | } |
1661 | |
1662 | const APInt &SDNode::getAsAPIntVal() const { |
1663 | return cast<ConstantSDNode>(Val: this)->getAPIntValue(); |
1664 | } |
1665 | |
1666 | class ConstantFPSDNode : public SDNode { |
1667 | friend class SelectionDAG; |
1668 | |
1669 | const ConstantFP *Value; |
1670 | |
1671 | ConstantFPSDNode(bool isTarget, const ConstantFP *val, EVT VT) |
1672 | : SDNode(isTarget ? ISD::TargetConstantFP : ISD::ConstantFP, 0, |
1673 | DebugLoc(), getSDVTList(VT)), |
1674 | Value(val) {} |
1675 | |
1676 | public: |
1677 | const APFloat& getValueAPF() const { return Value->getValueAPF(); } |
1678 | const ConstantFP *getConstantFPValue() const { return Value; } |
1679 | |
1680 | /// Return true if the value is positive or negative zero. |
1681 | bool isZero() const { return Value->isZero(); } |
1682 | |
1683 | /// Return true if the value is a NaN. |
1684 | bool isNaN() const { return Value->isNaN(); } |
1685 | |
1686 | /// Return true if the value is an infinity |
1687 | bool isInfinity() const { return Value->isInfinity(); } |
1688 | |
1689 | /// Return true if the value is negative. |
1690 | bool isNegative() const { return Value->isNegative(); } |
1691 | |
1692 | /// We don't rely on operator== working on double values, as |
1693 | /// it returns true for things that are clearly not equal, like -0.0 and 0.0. |
1694 | /// As such, this method can be used to do an exact bit-for-bit comparison of |
1695 | /// two floating point values. |
1696 | |
1697 | /// We leave the version with the double argument here because it's just so |
1698 | /// convenient to write "2.0" and the like. Without this function we'd |
1699 | /// have to duplicate its logic everywhere it's called. |
1700 | bool isExactlyValue(double V) const { |
1701 | return Value->getValueAPF().isExactlyValue(V); |
1702 | } |
1703 | bool isExactlyValue(const APFloat& V) const; |
1704 | |
1705 | static bool isValueValidForType(EVT VT, const APFloat& Val); |
1706 | |
1707 | static bool classof(const SDNode *N) { |
1708 | return N->getOpcode() == ISD::ConstantFP || |
1709 | N->getOpcode() == ISD::TargetConstantFP; |
1710 | } |
1711 | }; |
1712 | |
1713 | /// Returns true if \p V is a constant integer zero. |
1714 | bool isNullConstant(SDValue V); |
1715 | |
1716 | /// Returns true if \p V is an FP constant with a value of positive zero. |
1717 | bool isNullFPConstant(SDValue V); |
1718 | |
1719 | /// Returns true if \p V is an integer constant with all bits set. |
1720 | bool isAllOnesConstant(SDValue V); |
1721 | |
1722 | /// Returns true if \p V is a constant integer one. |
1723 | bool isOneConstant(SDValue V); |
1724 | |
1725 | /// Returns true if \p V is a constant min signed integer value. |
1726 | bool isMinSignedConstant(SDValue V); |
1727 | |
1728 | /// Returns true if \p V is a neutral element of Opc with Flags. |
1729 | /// When OperandNo is 0, it checks that V is a left identity. Otherwise, it |
1730 | /// checks that V is a right identity. |
1731 | bool isNeutralConstant(unsigned Opc, SDNodeFlags Flags, SDValue V, |
1732 | unsigned OperandNo); |
1733 | |
1734 | /// Return the non-bitcasted source operand of \p V if it exists. |
1735 | /// If \p V is not a bitcasted value, it is returned as-is. |
1736 | SDValue peekThroughBitcasts(SDValue V); |
1737 | |
1738 | /// Return the non-bitcasted and one-use source operand of \p V if it exists. |
1739 | /// If \p V is not a bitcasted one-use value, it is returned as-is. |
1740 | SDValue peekThroughOneUseBitcasts(SDValue V); |
1741 | |
1742 | /// Return the non-extracted vector source operand of \p V if it exists. |
1743 | /// If \p V is not an extracted subvector, it is returned as-is. |
1744 | SDValue (SDValue V); |
1745 | |
1746 | /// Return the non-truncated source operand of \p V if it exists. |
1747 | /// If \p V is not a truncation, it is returned as-is. |
1748 | SDValue peekThroughTruncates(SDValue V); |
1749 | |
1750 | /// Returns true if \p V is a bitwise not operation. Assumes that an all ones |
1751 | /// constant is canonicalized to be operand 1. |
1752 | bool isBitwiseNot(SDValue V, bool AllowUndefs = false); |
1753 | |
1754 | /// If \p V is a bitwise not, returns the inverted operand. Otherwise returns |
1755 | /// an empty SDValue. Only bits set in \p Mask are required to be inverted, |
1756 | /// other bits may be arbitrary. |
1757 | SDValue getBitwiseNotOperand(SDValue V, SDValue Mask, bool AllowUndefs); |
1758 | |
1759 | /// Returns the SDNode if it is a constant splat BuildVector or constant int. |
1760 | ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false, |
1761 | bool AllowTruncation = false); |
1762 | |
1763 | /// Returns the SDNode if it is a demanded constant splat BuildVector or |
1764 | /// constant int. |
1765 | ConstantSDNode *isConstOrConstSplat(SDValue N, const APInt &DemandedElts, |
1766 | bool AllowUndefs = false, |
1767 | bool AllowTruncation = false); |
1768 | |
1769 | /// Returns the SDNode if it is a constant splat BuildVector or constant float. |
1770 | ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, bool AllowUndefs = false); |
1771 | |
1772 | /// Returns the SDNode if it is a demanded constant splat BuildVector or |
1773 | /// constant float. |
1774 | ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, const APInt &DemandedElts, |
1775 | bool AllowUndefs = false); |
1776 | |
1777 | /// Return true if the value is a constant 0 integer or a splatted vector of |
1778 | /// a constant 0 integer (with no undefs by default). |
1779 | /// Build vector implicit truncation is not an issue for null values. |
1780 | bool isNullOrNullSplat(SDValue V, bool AllowUndefs = false); |
1781 | |
1782 | /// Return true if the value is a constant 1 integer or a splatted vector of a |
1783 | /// constant 1 integer (with no undefs). |
1784 | /// Build vector implicit truncation is allowed, but the truncated bits need to |
1785 | /// be zero. |
1786 | bool isOneOrOneSplat(SDValue V, bool AllowUndefs = false); |
1787 | |
1788 | /// Return true if the value is a constant -1 integer or a splatted vector of a |
1789 | /// constant -1 integer (with no undefs). |
1790 | /// Does not permit build vector implicit truncation. |
1791 | bool isAllOnesOrAllOnesSplat(SDValue V, bool AllowUndefs = false); |
1792 | |
1793 | /// Return true if \p V is either a integer or FP constant. |
1794 | inline bool isIntOrFPConstant(SDValue V) { |
1795 | return isa<ConstantSDNode>(Val: V) || isa<ConstantFPSDNode>(Val: V); |
1796 | } |
1797 | |
1798 | class GlobalAddressSDNode : public SDNode { |
1799 | friend class SelectionDAG; |
1800 | |
1801 | const GlobalValue *TheGlobal; |
1802 | int64_t Offset; |
1803 | unsigned TargetFlags; |
1804 | |
1805 | GlobalAddressSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL, |
1806 | const GlobalValue *GA, EVT VT, int64_t o, |
1807 | unsigned TF); |
1808 | |
1809 | public: |
1810 | const GlobalValue *getGlobal() const { return TheGlobal; } |
1811 | int64_t getOffset() const { return Offset; } |
1812 | unsigned getTargetFlags() const { return TargetFlags; } |
1813 | // Return the address space this GlobalAddress belongs to. |
1814 | unsigned getAddressSpace() const; |
1815 | |
1816 | static bool classof(const SDNode *N) { |
1817 | return N->getOpcode() == ISD::GlobalAddress || |
1818 | N->getOpcode() == ISD::TargetGlobalAddress || |
1819 | N->getOpcode() == ISD::GlobalTLSAddress || |
1820 | N->getOpcode() == ISD::TargetGlobalTLSAddress; |
1821 | } |
1822 | }; |
1823 | |
1824 | class FrameIndexSDNode : public SDNode { |
1825 | friend class SelectionDAG; |
1826 | |
1827 | int FI; |
1828 | |
1829 | FrameIndexSDNode(int fi, EVT VT, bool isTarg) |
1830 | : SDNode(isTarg ? ISD::TargetFrameIndex : ISD::FrameIndex, |
1831 | 0, DebugLoc(), getSDVTList(VT)), FI(fi) { |
1832 | } |
1833 | |
1834 | public: |
1835 | int getIndex() const { return FI; } |
1836 | |
1837 | static bool classof(const SDNode *N) { |
1838 | return N->getOpcode() == ISD::FrameIndex || |
1839 | N->getOpcode() == ISD::TargetFrameIndex; |
1840 | } |
1841 | }; |
1842 | |
1843 | /// This SDNode is used for LIFETIME_START/LIFETIME_END values, which indicate |
1844 | /// the offet and size that are started/ended in the underlying FrameIndex. |
1845 | class LifetimeSDNode : public SDNode { |
1846 | friend class SelectionDAG; |
1847 | int64_t Size; |
1848 | int64_t Offset; // -1 if offset is unknown. |
1849 | |
1850 | LifetimeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, |
1851 | SDVTList VTs, int64_t Size, int64_t Offset) |
1852 | : SDNode(Opcode, Order, dl, VTs), Size(Size), Offset(Offset) {} |
1853 | public: |
1854 | int64_t getFrameIndex() const { |
1855 | return cast<FrameIndexSDNode>(Val: getOperand(Num: 1))->getIndex(); |
1856 | } |
1857 | |
1858 | bool hasOffset() const { return Offset >= 0; } |
1859 | int64_t getOffset() const { |
1860 | assert(hasOffset() && "offset is unknown" ); |
1861 | return Offset; |
1862 | } |
1863 | int64_t getSize() const { |
1864 | assert(hasOffset() && "offset is unknown" ); |
1865 | return Size; |
1866 | } |
1867 | |
1868 | // Methods to support isa and dyn_cast |
1869 | static bool classof(const SDNode *N) { |
1870 | return N->getOpcode() == ISD::LIFETIME_START || |
1871 | N->getOpcode() == ISD::LIFETIME_END; |
1872 | } |
1873 | }; |
1874 | |
1875 | /// This SDNode is used for PSEUDO_PROBE values, which are the function guid and |
1876 | /// the index of the basic block being probed. A pseudo probe serves as a place |
1877 | /// holder and will be removed at the end of compilation. It does not have any |
1878 | /// operand because we do not want the instruction selection to deal with any. |
1879 | class PseudoProbeSDNode : public SDNode { |
1880 | friend class SelectionDAG; |
1881 | uint64_t Guid; |
1882 | uint64_t Index; |
1883 | uint32_t Attributes; |
1884 | |
1885 | PseudoProbeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &Dl, |
1886 | SDVTList VTs, uint64_t Guid, uint64_t Index, uint32_t Attr) |
1887 | : SDNode(Opcode, Order, Dl, VTs), Guid(Guid), Index(Index), |
1888 | Attributes(Attr) {} |
1889 | |
1890 | public: |
1891 | uint64_t getGuid() const { return Guid; } |
1892 | uint64_t getIndex() const { return Index; } |
1893 | uint32_t getAttributes() const { return Attributes; } |
1894 | |
1895 | // Methods to support isa and dyn_cast |
1896 | static bool classof(const SDNode *N) { |
1897 | return N->getOpcode() == ISD::PSEUDO_PROBE; |
1898 | } |
1899 | }; |
1900 | |
1901 | class JumpTableSDNode : public SDNode { |
1902 | friend class SelectionDAG; |
1903 | |
1904 | int JTI; |
1905 | unsigned TargetFlags; |
1906 | |
1907 | JumpTableSDNode(int jti, EVT VT, bool isTarg, unsigned TF) |
1908 | : SDNode(isTarg ? ISD::TargetJumpTable : ISD::JumpTable, |
1909 | 0, DebugLoc(), getSDVTList(VT)), JTI(jti), TargetFlags(TF) { |
1910 | } |
1911 | |
1912 | public: |
1913 | int getIndex() const { return JTI; } |
1914 | unsigned getTargetFlags() const { return TargetFlags; } |
1915 | |
1916 | static bool classof(const SDNode *N) { |
1917 | return N->getOpcode() == ISD::JumpTable || |
1918 | N->getOpcode() == ISD::TargetJumpTable; |
1919 | } |
1920 | }; |
1921 | |
1922 | class ConstantPoolSDNode : public SDNode { |
1923 | friend class SelectionDAG; |
1924 | |
1925 | union { |
1926 | const Constant *ConstVal; |
1927 | MachineConstantPoolValue *MachineCPVal; |
1928 | } Val; |
1929 | int Offset; // It's a MachineConstantPoolValue if top bit is set. |
1930 | Align Alignment; // Minimum alignment requirement of CP. |
1931 | unsigned TargetFlags; |
1932 | |
1933 | ConstantPoolSDNode(bool isTarget, const Constant *c, EVT VT, int o, |
1934 | Align Alignment, unsigned TF) |
1935 | : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0, |
1936 | DebugLoc(), getSDVTList(VT)), |
1937 | Offset(o), Alignment(Alignment), TargetFlags(TF) { |
1938 | assert(Offset >= 0 && "Offset is too large" ); |
1939 | Val.ConstVal = c; |
1940 | } |
1941 | |
1942 | ConstantPoolSDNode(bool isTarget, MachineConstantPoolValue *v, EVT VT, int o, |
1943 | Align Alignment, unsigned TF) |
1944 | : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0, |
1945 | DebugLoc(), getSDVTList(VT)), |
1946 | Offset(o), Alignment(Alignment), TargetFlags(TF) { |
1947 | assert(Offset >= 0 && "Offset is too large" ); |
1948 | Val.MachineCPVal = v; |
1949 | Offset |= 1 << (sizeof(unsigned)*CHAR_BIT-1); |
1950 | } |
1951 | |
1952 | public: |
1953 | bool isMachineConstantPoolEntry() const { |
1954 | return Offset < 0; |
1955 | } |
1956 | |
1957 | const Constant *getConstVal() const { |
1958 | assert(!isMachineConstantPoolEntry() && "Wrong constantpool type" ); |
1959 | return Val.ConstVal; |
1960 | } |
1961 | |
1962 | MachineConstantPoolValue *getMachineCPVal() const { |
1963 | assert(isMachineConstantPoolEntry() && "Wrong constantpool type" ); |
1964 | return Val.MachineCPVal; |
1965 | } |
1966 | |
1967 | int getOffset() const { |
1968 | return Offset & ~(1 << (sizeof(unsigned)*CHAR_BIT-1)); |
1969 | } |
1970 | |
1971 | // Return the alignment of this constant pool object, which is either 0 (for |
1972 | // default alignment) or the desired value. |
1973 | Align getAlign() const { return Alignment; } |
1974 | unsigned getTargetFlags() const { return TargetFlags; } |
1975 | |
1976 | Type *getType() const; |
1977 | |
1978 | static bool classof(const SDNode *N) { |
1979 | return N->getOpcode() == ISD::ConstantPool || |
1980 | N->getOpcode() == ISD::TargetConstantPool; |
1981 | } |
1982 | }; |
1983 | |
1984 | /// Completely target-dependent object reference. |
1985 | class TargetIndexSDNode : public SDNode { |
1986 | friend class SelectionDAG; |
1987 | |
1988 | unsigned TargetFlags; |
1989 | int Index; |
1990 | int64_t Offset; |
1991 | |
1992 | public: |
1993 | TargetIndexSDNode(int Idx, EVT VT, int64_t Ofs, unsigned TF) |
1994 | : SDNode(ISD::TargetIndex, 0, DebugLoc(), getSDVTList(VT)), |
1995 | TargetFlags(TF), Index(Idx), Offset(Ofs) {} |
1996 | |
1997 | unsigned getTargetFlags() const { return TargetFlags; } |
1998 | int getIndex() const { return Index; } |
1999 | int64_t getOffset() const { return Offset; } |
2000 | |
2001 | static bool classof(const SDNode *N) { |
2002 | return N->getOpcode() == ISD::TargetIndex; |
2003 | } |
2004 | }; |
2005 | |
2006 | class BasicBlockSDNode : public SDNode { |
2007 | friend class SelectionDAG; |
2008 | |
2009 | MachineBasicBlock *MBB; |
2010 | |
2011 | /// Debug info is meaningful and potentially useful here, but we create |
2012 | /// blocks out of order when they're jumped to, which makes it a bit |
2013 | /// harder. Let's see if we need it first. |
2014 | explicit BasicBlockSDNode(MachineBasicBlock *mbb) |
2015 | : SDNode(ISD::BasicBlock, 0, DebugLoc(), getSDVTList(MVT::Other)), MBB(mbb) |
2016 | {} |
2017 | |
2018 | public: |
2019 | MachineBasicBlock *getBasicBlock() const { return MBB; } |
2020 | |
2021 | static bool classof(const SDNode *N) { |
2022 | return N->getOpcode() == ISD::BasicBlock; |
2023 | } |
2024 | }; |
2025 | |
2026 | /// A "pseudo-class" with methods for operating on BUILD_VECTORs. |
2027 | class BuildVectorSDNode : public SDNode { |
2028 | public: |
2029 | // These are constructed as SDNodes and then cast to BuildVectorSDNodes. |
2030 | explicit BuildVectorSDNode() = delete; |
2031 | |
2032 | /// Check if this is a constant splat, and if so, find the |
2033 | /// smallest element size that splats the vector. If MinSplatBits is |
2034 | /// nonzero, the element size must be at least that large. Note that the |
2035 | /// splat element may be the entire vector (i.e., a one element vector). |
2036 | /// Returns the splat element value in SplatValue. Any undefined bits in |
2037 | /// that value are zero, and the corresponding bits in the SplatUndef mask |
2038 | /// are set. The SplatBitSize value is set to the splat element size in |
2039 | /// bits. HasAnyUndefs is set to true if any bits in the vector are |
2040 | /// undefined. isBigEndian describes the endianness of the target. |
2041 | bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef, |
2042 | unsigned &SplatBitSize, bool &HasAnyUndefs, |
2043 | unsigned MinSplatBits = 0, |
2044 | bool isBigEndian = false) const; |
2045 | |
2046 | /// Returns the demanded splatted value or a null value if this is not a |
2047 | /// splat. |
2048 | /// |
2049 | /// The DemandedElts mask indicates the elements that must be in the splat. |
2050 | /// If passed a non-null UndefElements bitvector, it will resize it to match |
2051 | /// the vector width and set the bits where elements are undef. |
2052 | SDValue getSplatValue(const APInt &DemandedElts, |
2053 | BitVector *UndefElements = nullptr) const; |
2054 | |
2055 | /// Returns the splatted value or a null value if this is not a splat. |
2056 | /// |
2057 | /// If passed a non-null UndefElements bitvector, it will resize it to match |
2058 | /// the vector width and set the bits where elements are undef. |
2059 | SDValue getSplatValue(BitVector *UndefElements = nullptr) const; |
2060 | |
2061 | /// Find the shortest repeating sequence of values in the build vector. |
2062 | /// |
2063 | /// e.g. { u, X, u, X, u, u, X, u } -> { X } |
2064 | /// { X, Y, u, Y, u, u, X, u } -> { X, Y } |
2065 | /// |
2066 | /// Currently this must be a power-of-2 build vector. |
2067 | /// The DemandedElts mask indicates the elements that must be present, |
2068 | /// undemanded elements in Sequence may be null (SDValue()). If passed a |
2069 | /// non-null UndefElements bitvector, it will resize it to match the original |
2070 | /// vector width and set the bits where elements are undef. If result is |
2071 | /// false, Sequence will be empty. |
2072 | bool getRepeatedSequence(const APInt &DemandedElts, |
2073 | SmallVectorImpl<SDValue> &Sequence, |
2074 | BitVector *UndefElements = nullptr) const; |
2075 | |
2076 | /// Find the shortest repeating sequence of values in the build vector. |
2077 | /// |
2078 | /// e.g. { u, X, u, X, u, u, X, u } -> { X } |
2079 | /// { X, Y, u, Y, u, u, X, u } -> { X, Y } |
2080 | /// |
2081 | /// Currently this must be a power-of-2 build vector. |
2082 | /// If passed a non-null UndefElements bitvector, it will resize it to match |
2083 | /// the original vector width and set the bits where elements are undef. |
2084 | /// If result is false, Sequence will be empty. |
2085 | bool getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence, |
2086 | BitVector *UndefElements = nullptr) const; |
2087 | |
2088 | /// Returns the demanded splatted constant or null if this is not a constant |
2089 | /// splat. |
2090 | /// |
2091 | /// The DemandedElts mask indicates the elements that must be in the splat. |
2092 | /// If passed a non-null UndefElements bitvector, it will resize it to match |
2093 | /// the vector width and set the bits where elements are undef. |
2094 | ConstantSDNode * |
2095 | getConstantSplatNode(const APInt &DemandedElts, |
2096 | BitVector *UndefElements = nullptr) const; |
2097 | |
2098 | /// Returns the splatted constant or null if this is not a constant |
2099 | /// splat. |
2100 | /// |
2101 | /// If passed a non-null UndefElements bitvector, it will resize it to match |
2102 | /// the vector width and set the bits where elements are undef. |
2103 | ConstantSDNode * |
2104 | getConstantSplatNode(BitVector *UndefElements = nullptr) const; |
2105 | |
2106 | /// Returns the demanded splatted constant FP or null if this is not a |
2107 | /// constant FP splat. |
2108 | /// |
2109 | /// The DemandedElts mask indicates the elements that must be in the splat. |
2110 | /// If passed a non-null UndefElements bitvector, it will resize it to match |
2111 | /// the vector width and set the bits where elements are undef. |
2112 | ConstantFPSDNode * |
2113 | getConstantFPSplatNode(const APInt &DemandedElts, |
2114 | BitVector *UndefElements = nullptr) const; |
2115 | |
2116 | /// Returns the splatted constant FP or null if this is not a constant |
2117 | /// FP splat. |
2118 | /// |
2119 | /// If passed a non-null UndefElements bitvector, it will resize it to match |
2120 | /// the vector width and set the bits where elements are undef. |
2121 | ConstantFPSDNode * |
2122 | getConstantFPSplatNode(BitVector *UndefElements = nullptr) const; |
2123 | |
2124 | /// If this is a constant FP splat and the splatted constant FP is an |
2125 | /// exact power or 2, return the log base 2 integer value. Otherwise, |
2126 | /// return -1. |
2127 | /// |
2128 | /// The BitWidth specifies the necessary bit precision. |
2129 | int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements, |
2130 | uint32_t BitWidth) const; |
2131 | |
2132 | /// Extract the raw bit data from a build vector of Undef, Constant or |
2133 | /// ConstantFP node elements. Each raw bit element will be \p |
2134 | /// DstEltSizeInBits wide, undef elements are treated as zero, and entirely |
2135 | /// undefined elements are flagged in \p UndefElements. |
2136 | bool getConstantRawBits(bool IsLittleEndian, unsigned DstEltSizeInBits, |
2137 | SmallVectorImpl<APInt> &RawBitElements, |
2138 | BitVector &UndefElements) const; |
2139 | |
2140 | bool isConstant() const; |
2141 | |
2142 | /// If this BuildVector is constant and represents the numerical series |
2143 | /// "<a, a+n, a+2n, a+3n, ...>" where a is integer and n is a non-zero integer, |
2144 | /// the value "<a,n>" is returned. |
2145 | std::optional<std::pair<APInt, APInt>> isConstantSequence() const; |
2146 | |
2147 | /// Recast bit data \p SrcBitElements to \p DstEltSizeInBits wide elements. |
2148 | /// Undef elements are treated as zero, and entirely undefined elements are |
2149 | /// flagged in \p DstUndefElements. |
2150 | static void recastRawBits(bool IsLittleEndian, unsigned DstEltSizeInBits, |
2151 | SmallVectorImpl<APInt> &DstBitElements, |
2152 | ArrayRef<APInt> SrcBitElements, |
2153 | BitVector &DstUndefElements, |
2154 | const BitVector &SrcUndefElements); |
2155 | |
2156 | static bool classof(const SDNode *N) { |
2157 | return N->getOpcode() == ISD::BUILD_VECTOR; |
2158 | } |
2159 | }; |
2160 | |
2161 | /// An SDNode that holds an arbitrary LLVM IR Value. This is |
2162 | /// used when the SelectionDAG needs to make a simple reference to something |
2163 | /// in the LLVM IR representation. |
2164 | /// |
2165 | class SrcValueSDNode : public SDNode { |
2166 | friend class SelectionDAG; |
2167 | |
2168 | const Value *V; |
2169 | |
2170 | /// Create a SrcValue for a general value. |
2171 | explicit SrcValueSDNode(const Value *v) |
2172 | : SDNode(ISD::SRCVALUE, 0, DebugLoc(), getSDVTList(MVT::Other)), V(v) {} |
2173 | |
2174 | public: |
2175 | /// Return the contained Value. |
2176 | const Value *getValue() const { return V; } |
2177 | |
2178 | static bool classof(const SDNode *N) { |
2179 | return N->getOpcode() == ISD::SRCVALUE; |
2180 | } |
2181 | }; |
2182 | |
2183 | class MDNodeSDNode : public SDNode { |
2184 | friend class SelectionDAG; |
2185 | |
2186 | const MDNode *MD; |
2187 | |
2188 | explicit MDNodeSDNode(const MDNode *md) |
2189 | : SDNode(ISD::MDNODE_SDNODE, 0, DebugLoc(), getSDVTList(MVT::Other)), MD(md) |
2190 | {} |
2191 | |
2192 | public: |
2193 | const MDNode *getMD() const { return MD; } |
2194 | |
2195 | static bool classof(const SDNode *N) { |
2196 | return N->getOpcode() == ISD::MDNODE_SDNODE; |
2197 | } |
2198 | }; |
2199 | |
2200 | class RegisterSDNode : public SDNode { |
2201 | friend class SelectionDAG; |
2202 | |
2203 | Register Reg; |
2204 | |
2205 | RegisterSDNode(Register reg, EVT VT) |
2206 | : SDNode(ISD::Register, 0, DebugLoc(), getSDVTList(VT)), Reg(reg) {} |
2207 | |
2208 | public: |
2209 | Register getReg() const { return Reg; } |
2210 | |
2211 | static bool classof(const SDNode *N) { |
2212 | return N->getOpcode() == ISD::Register; |
2213 | } |
2214 | }; |
2215 | |
2216 | class RegisterMaskSDNode : public SDNode { |
2217 | friend class SelectionDAG; |
2218 | |
2219 | // The memory for RegMask is not owned by the node. |
2220 | const uint32_t *RegMask; |
2221 | |
2222 | RegisterMaskSDNode(const uint32_t *mask) |
2223 | : SDNode(ISD::RegisterMask, 0, DebugLoc(), getSDVTList(MVT::Untyped)), |
2224 | RegMask(mask) {} |
2225 | |
2226 | public: |
2227 | const uint32_t *getRegMask() const { return RegMask; } |
2228 | |
2229 | static bool classof(const SDNode *N) { |
2230 | return N->getOpcode() == ISD::RegisterMask; |
2231 | } |
2232 | }; |
2233 | |
2234 | class BlockAddressSDNode : public SDNode { |
2235 | friend class SelectionDAG; |
2236 | |
2237 | const BlockAddress *BA; |
2238 | int64_t Offset; |
2239 | unsigned TargetFlags; |
2240 | |
2241 | BlockAddressSDNode(unsigned NodeTy, EVT VT, const BlockAddress *ba, |
2242 | int64_t o, unsigned Flags) |
2243 | : SDNode(NodeTy, 0, DebugLoc(), getSDVTList(VT)), |
2244 | BA(ba), Offset(o), TargetFlags(Flags) {} |
2245 | |
2246 | public: |
2247 | const BlockAddress *getBlockAddress() const { return BA; } |
2248 | int64_t getOffset() const { return Offset; } |
2249 | unsigned getTargetFlags() const { return TargetFlags; } |
2250 | |
2251 | static bool classof(const SDNode *N) { |
2252 | return N->getOpcode() == ISD::BlockAddress || |
2253 | N->getOpcode() == ISD::TargetBlockAddress; |
2254 | } |
2255 | }; |
2256 | |
2257 | class LabelSDNode : public SDNode { |
2258 | friend class SelectionDAG; |
2259 | |
2260 | MCSymbol *Label; |
2261 | |
2262 | LabelSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, MCSymbol *L) |
2263 | : SDNode(Opcode, Order, dl, getSDVTList(MVT::Other)), Label(L) { |
2264 | assert(LabelSDNode::classof(this) && "not a label opcode" ); |
2265 | } |
2266 | |
2267 | public: |
2268 | MCSymbol *getLabel() const { return Label; } |
2269 | |
2270 | static bool classof(const SDNode *N) { |
2271 | return N->getOpcode() == ISD::EH_LABEL || |
2272 | N->getOpcode() == ISD::ANNOTATION_LABEL; |
2273 | } |
2274 | }; |
2275 | |
2276 | class ExternalSymbolSDNode : public SDNode { |
2277 | friend class SelectionDAG; |
2278 | |
2279 | const char *Symbol; |
2280 | unsigned TargetFlags; |
2281 | |
2282 | ExternalSymbolSDNode(bool isTarget, const char *Sym, unsigned TF, EVT VT) |
2283 | : SDNode(isTarget ? ISD::TargetExternalSymbol : ISD::ExternalSymbol, 0, |
2284 | DebugLoc(), getSDVTList(VT)), |
2285 | Symbol(Sym), TargetFlags(TF) {} |
2286 | |
2287 | public: |
2288 | const char *getSymbol() const { return Symbol; } |
2289 | unsigned getTargetFlags() const { return TargetFlags; } |
2290 | |
2291 | static bool classof(const SDNode *N) { |
2292 | return N->getOpcode() == ISD::ExternalSymbol || |
2293 | N->getOpcode() == ISD::TargetExternalSymbol; |
2294 | } |
2295 | }; |
2296 | |
2297 | class MCSymbolSDNode : public SDNode { |
2298 | friend class SelectionDAG; |
2299 | |
2300 | MCSymbol *Symbol; |
2301 | |
2302 | MCSymbolSDNode(MCSymbol *Symbol, EVT VT) |
2303 | : SDNode(ISD::MCSymbol, 0, DebugLoc(), getSDVTList(VT)), Symbol(Symbol) {} |
2304 | |
2305 | public: |
2306 | MCSymbol *getMCSymbol() const { return Symbol; } |
2307 | |
2308 | static bool classof(const SDNode *N) { |
2309 | return N->getOpcode() == ISD::MCSymbol; |
2310 | } |
2311 | }; |
2312 | |
2313 | class CondCodeSDNode : public SDNode { |
2314 | friend class SelectionDAG; |
2315 | |
2316 | ISD::CondCode Condition; |
2317 | |
2318 | explicit CondCodeSDNode(ISD::CondCode Cond) |
2319 | : SDNode(ISD::CONDCODE, 0, DebugLoc(), getSDVTList(MVT::Other)), |
2320 | Condition(Cond) {} |
2321 | |
2322 | public: |
2323 | ISD::CondCode get() const { return Condition; } |
2324 | |
2325 | static bool classof(const SDNode *N) { |
2326 | return N->getOpcode() == ISD::CONDCODE; |
2327 | } |
2328 | }; |
2329 | |
2330 | /// This class is used to represent EVT's, which are used |
2331 | /// to parameterize some operations. |
2332 | class VTSDNode : public SDNode { |
2333 | friend class SelectionDAG; |
2334 | |
2335 | EVT ValueType; |
2336 | |
2337 | explicit VTSDNode(EVT VT) |
2338 | : SDNode(ISD::VALUETYPE, 0, DebugLoc(), getSDVTList(MVT::Other)), |
2339 | ValueType(VT) {} |
2340 | |
2341 | public: |
2342 | EVT getVT() const { return ValueType; } |
2343 | |
2344 | static bool classof(const SDNode *N) { |
2345 | return N->getOpcode() == ISD::VALUETYPE; |
2346 | } |
2347 | }; |
2348 | |
2349 | /// Base class for LoadSDNode and StoreSDNode |
2350 | class LSBaseSDNode : public MemSDNode { |
2351 | public: |
2352 | LSBaseSDNode(ISD::NodeType NodeTy, unsigned Order, const DebugLoc &dl, |
2353 | SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT, |
2354 | MachineMemOperand *MMO) |
2355 | : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { |
2356 | LSBaseSDNodeBits.AddressingMode = AM; |
2357 | assert(getAddressingMode() == AM && "Value truncated" ); |
2358 | } |
2359 | |
2360 | const SDValue &getOffset() const { |
2361 | return getOperand(Num: getOpcode() == ISD::LOAD ? 2 : 3); |
2362 | } |
2363 | |
2364 | /// Return the addressing mode for this load or store: |
2365 | /// unindexed, pre-inc, pre-dec, post-inc, or post-dec. |
2366 | ISD::MemIndexedMode getAddressingMode() const { |
2367 | return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode); |
2368 | } |
2369 | |
2370 | /// Return true if this is a pre/post inc/dec load/store. |
2371 | bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; } |
2372 | |
2373 | /// Return true if this is NOT a pre/post inc/dec load/store. |
2374 | bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; } |
2375 | |
2376 | static bool classof(const SDNode *N) { |
2377 | return N->getOpcode() == ISD::LOAD || |
2378 | N->getOpcode() == ISD::STORE; |
2379 | } |
2380 | }; |
2381 | |
2382 | /// This class is used to represent ISD::LOAD nodes. |
2383 | class LoadSDNode : public LSBaseSDNode { |
2384 | friend class SelectionDAG; |
2385 | |
2386 | LoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, |
2387 | ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT MemVT, |
2388 | MachineMemOperand *MMO) |
2389 | : LSBaseSDNode(ISD::LOAD, Order, dl, VTs, AM, MemVT, MMO) { |
2390 | LoadSDNodeBits.ExtTy = ETy; |
2391 | assert(readMem() && "Load MachineMemOperand is not a load!" ); |
2392 | assert(!writeMem() && "Load MachineMemOperand is a store!" ); |
2393 | } |
2394 | |
2395 | public: |
2396 | /// Return whether this is a plain node, |
2397 | /// or one of the varieties of value-extending loads. |
2398 | ISD::LoadExtType getExtensionType() const { |
2399 | return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy); |
2400 | } |
2401 | |
2402 | const SDValue &getBasePtr() const { return getOperand(Num: 1); } |
2403 | const SDValue &getOffset() const { return getOperand(Num: 2); } |
2404 | |
2405 | static bool classof(const SDNode *N) { |
2406 | return N->getOpcode() == ISD::LOAD; |
2407 | } |
2408 | }; |
2409 | |
2410 | /// This class is used to represent ISD::STORE nodes. |
2411 | class StoreSDNode : public LSBaseSDNode { |
2412 | friend class SelectionDAG; |
2413 | |
2414 | StoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, |
2415 | ISD::MemIndexedMode AM, bool isTrunc, EVT MemVT, |
2416 | MachineMemOperand *MMO) |
2417 | : LSBaseSDNode(ISD::STORE, Order, dl, VTs, AM, MemVT, MMO) { |
2418 | StoreSDNodeBits.IsTruncating = isTrunc; |
2419 | assert(!readMem() && "Store MachineMemOperand is a load!" ); |
2420 | assert(writeMem() && "Store MachineMemOperand is not a store!" ); |
2421 | } |
2422 | |
2423 | public: |
2424 | /// Return true if the op does a truncation before store. |
2425 | /// For integers this is the same as doing a TRUNCATE and storing the result. |
2426 | /// For floats, it is the same as doing an FP_ROUND and storing the result. |
2427 | bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; } |
2428 | void setTruncatingStore(bool Truncating) { |
2429 | StoreSDNodeBits.IsTruncating = Truncating; |
2430 | } |
2431 | |
2432 | const SDValue &getValue() const { return getOperand(Num: 1); } |
2433 | const SDValue &getBasePtr() const { return getOperand(Num: 2); } |
2434 | const SDValue &getOffset() const { return getOperand(Num: 3); } |
2435 | |
2436 | static bool classof(const SDNode *N) { |
2437 | return N->getOpcode() == ISD::STORE; |
2438 | } |
2439 | }; |
2440 | |
2441 | /// This base class is used to represent VP_LOAD, VP_STORE, |
2442 | /// EXPERIMENTAL_VP_STRIDED_LOAD and EXPERIMENTAL_VP_STRIDED_STORE nodes |
2443 | class VPBaseLoadStoreSDNode : public MemSDNode { |
2444 | public: |
2445 | friend class SelectionDAG; |
2446 | |
2447 | VPBaseLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order, |
2448 | const DebugLoc &DL, SDVTList VTs, |
2449 | ISD::MemIndexedMode AM, EVT MemVT, |
2450 | MachineMemOperand *MMO) |
2451 | : MemSDNode(NodeTy, Order, DL, VTs, MemVT, MMO) { |
2452 | LSBaseSDNodeBits.AddressingMode = AM; |
2453 | assert(getAddressingMode() == AM && "Value truncated" ); |
2454 | } |
2455 | |
2456 | // VPStridedStoreSDNode (Chain, Data, Ptr, Offset, Stride, Mask, EVL) |
2457 | // VPStoreSDNode (Chain, Data, Ptr, Offset, Mask, EVL) |
2458 | // VPStridedLoadSDNode (Chain, Ptr, Offset, Stride, Mask, EVL) |
2459 | // VPLoadSDNode (Chain, Ptr, Offset, Mask, EVL) |
2460 | // Mask is a vector of i1 elements; |
2461 | // the type of EVL is TLI.getVPExplicitVectorLengthTy(). |
2462 | const SDValue &getOffset() const { |
2463 | return getOperand(Num: (getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD || |
2464 | getOpcode() == ISD::VP_LOAD) |
2465 | ? 2 |
2466 | : 3); |
2467 | } |
2468 | const SDValue &getBasePtr() const { |
2469 | return getOperand(Num: (getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD || |
2470 | getOpcode() == ISD::VP_LOAD) |
2471 | ? 1 |
2472 | : 2); |
2473 | } |
2474 | const SDValue &getMask() const { |
2475 | switch (getOpcode()) { |
2476 | default: |
2477 | llvm_unreachable("Invalid opcode" ); |
2478 | case ISD::VP_LOAD: |
2479 | return getOperand(Num: 3); |
2480 | case ISD::VP_STORE: |
2481 | case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: |
2482 | return getOperand(Num: 4); |
2483 | case ISD::EXPERIMENTAL_VP_STRIDED_STORE: |
2484 | return getOperand(Num: 5); |
2485 | } |
2486 | } |
2487 | const SDValue &getVectorLength() const { |
2488 | switch (getOpcode()) { |
2489 | default: |
2490 | llvm_unreachable("Invalid opcode" ); |
2491 | case ISD::VP_LOAD: |
2492 | return getOperand(Num: 4); |
2493 | case ISD::VP_STORE: |
2494 | case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: |
2495 | return getOperand(Num: 5); |
2496 | case ISD::EXPERIMENTAL_VP_STRIDED_STORE: |
2497 | return getOperand(Num: 6); |
2498 | } |
2499 | } |
2500 | |
2501 | /// Return the addressing mode for this load or store: |
2502 | /// unindexed, pre-inc, pre-dec, post-inc, or post-dec. |
2503 | ISD::MemIndexedMode getAddressingMode() const { |
2504 | return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode); |
2505 | } |
2506 | |
2507 | /// Return true if this is a pre/post inc/dec load/store. |
2508 | bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; } |
2509 | |
2510 | /// Return true if this is NOT a pre/post inc/dec load/store. |
2511 | bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; } |
2512 | |
2513 | static bool classof(const SDNode *N) { |
2514 | return N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD || |
2515 | N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE || |
2516 | N->getOpcode() == ISD::VP_LOAD || N->getOpcode() == ISD::VP_STORE; |
2517 | } |
2518 | }; |
2519 | |
2520 | /// This class is used to represent a VP_LOAD node |
2521 | class VPLoadSDNode : public VPBaseLoadStoreSDNode { |
2522 | public: |
2523 | friend class SelectionDAG; |
2524 | |
2525 | VPLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, |
2526 | ISD::MemIndexedMode AM, ISD::LoadExtType ETy, bool isExpanding, |
2527 | EVT MemVT, MachineMemOperand *MMO) |
2528 | : VPBaseLoadStoreSDNode(ISD::VP_LOAD, Order, dl, VTs, AM, MemVT, MMO) { |
2529 | LoadSDNodeBits.ExtTy = ETy; |
2530 | LoadSDNodeBits.IsExpanding = isExpanding; |
2531 | } |
2532 | |
2533 | ISD::LoadExtType getExtensionType() const { |
2534 | return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy); |
2535 | } |
2536 | |
2537 | const SDValue &getBasePtr() const { return getOperand(Num: 1); } |
2538 | const SDValue &getOffset() const { return getOperand(Num: 2); } |
2539 | const SDValue &getMask() const { return getOperand(Num: 3); } |
2540 | const SDValue &getVectorLength() const { return getOperand(Num: 4); } |
2541 | |
2542 | static bool classof(const SDNode *N) { |
2543 | return N->getOpcode() == ISD::VP_LOAD; |
2544 | } |
2545 | bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; } |
2546 | }; |
2547 | |
2548 | /// This class is used to represent an EXPERIMENTAL_VP_STRIDED_LOAD node. |
2549 | class VPStridedLoadSDNode : public VPBaseLoadStoreSDNode { |
2550 | public: |
2551 | friend class SelectionDAG; |
2552 | |
2553 | VPStridedLoadSDNode(unsigned Order, const DebugLoc &DL, SDVTList VTs, |
2554 | ISD::MemIndexedMode AM, ISD::LoadExtType ETy, |
2555 | bool IsExpanding, EVT MemVT, MachineMemOperand *MMO) |
2556 | : VPBaseLoadStoreSDNode(ISD::EXPERIMENTAL_VP_STRIDED_LOAD, Order, DL, VTs, |
2557 | AM, MemVT, MMO) { |
2558 | LoadSDNodeBits.ExtTy = ETy; |
2559 | LoadSDNodeBits.IsExpanding = IsExpanding; |
2560 | } |
2561 | |
2562 | ISD::LoadExtType getExtensionType() const { |
2563 | return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy); |
2564 | } |
2565 | |
2566 | const SDValue &getBasePtr() const { return getOperand(Num: 1); } |
2567 | const SDValue &getOffset() const { return getOperand(Num: 2); } |
2568 | const SDValue &getStride() const { return getOperand(Num: 3); } |
2569 | const SDValue &getMask() const { return getOperand(Num: 4); } |
2570 | const SDValue &getVectorLength() const { return getOperand(Num: 5); } |
2571 | |
2572 | static bool classof(const SDNode *N) { |
2573 | return N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD; |
2574 | } |
2575 | bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; } |
2576 | }; |
2577 | |
2578 | /// This class is used to represent a VP_STORE node |
2579 | class VPStoreSDNode : public VPBaseLoadStoreSDNode { |
2580 | public: |
2581 | friend class SelectionDAG; |
2582 | |
2583 | VPStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, |
2584 | ISD::MemIndexedMode AM, bool isTrunc, bool isCompressing, |
2585 | EVT MemVT, MachineMemOperand *MMO) |
2586 | : VPBaseLoadStoreSDNode(ISD::VP_STORE, Order, dl, VTs, AM, MemVT, MMO) { |
2587 | StoreSDNodeBits.IsTruncating = isTrunc; |
2588 | StoreSDNodeBits.IsCompressing = isCompressing; |
2589 | } |
2590 | |
2591 | /// Return true if this is a truncating store. |
2592 | /// For integers this is the same as doing a TRUNCATE and storing the result. |
2593 | /// For floats, it is the same as doing an FP_ROUND and storing the result. |
2594 | bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; } |
2595 | |
2596 | /// Returns true if the op does a compression to the vector before storing. |
2597 | /// The node contiguously stores the active elements (integers or floats) |
2598 | /// in src (those with their respective bit set in writemask k) to unaligned |
2599 | /// memory at base_addr. |
2600 | bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; } |
2601 | |
2602 | const SDValue &getValue() const { return getOperand(Num: 1); } |
2603 | const SDValue &getBasePtr() const { return getOperand(Num: 2); } |
2604 | const SDValue &getOffset() const { return getOperand(Num: 3); } |
2605 | const SDValue &getMask() const { return getOperand(Num: 4); } |
2606 | const SDValue &getVectorLength() const { return getOperand(Num: 5); } |
2607 | |
2608 | static bool classof(const SDNode *N) { |
2609 | return N->getOpcode() == ISD::VP_STORE; |
2610 | } |
2611 | }; |
2612 | |
2613 | /// This class is used to represent an EXPERIMENTAL_VP_STRIDED_STORE node. |
2614 | class VPStridedStoreSDNode : public VPBaseLoadStoreSDNode { |
2615 | public: |
2616 | friend class SelectionDAG; |
2617 | |
2618 | VPStridedStoreSDNode(unsigned Order, const DebugLoc &DL, SDVTList VTs, |
2619 | ISD::MemIndexedMode AM, bool IsTrunc, bool IsCompressing, |
2620 | EVT MemVT, MachineMemOperand *MMO) |
2621 | : VPBaseLoadStoreSDNode(ISD::EXPERIMENTAL_VP_STRIDED_STORE, Order, DL, |
2622 | VTs, AM, MemVT, MMO) { |
2623 | StoreSDNodeBits.IsTruncating = IsTrunc; |
2624 | StoreSDNodeBits.IsCompressing = IsCompressing; |
2625 | } |
2626 | |
2627 | /// Return true if this is a truncating store. |
2628 | /// For integers this is the same as doing a TRUNCATE and storing the result. |
2629 | /// For floats, it is the same as doing an FP_ROUND and storing the result. |
2630 | bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; } |
2631 | |
2632 | /// Returns true if the op does a compression to the vector before storing. |
2633 | /// The node contiguously stores the active elements (integers or floats) |
2634 | /// in src (those with their respective bit set in writemask k) to unaligned |
2635 | /// memory at base_addr. |
2636 | bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; } |
2637 | |
2638 | const SDValue &getValue() const { return getOperand(Num: 1); } |
2639 | const SDValue &getBasePtr() const { return getOperand(Num: 2); } |
2640 | const SDValue &getOffset() const { return getOperand(Num: 3); } |
2641 | const SDValue &getStride() const { return getOperand(Num: 4); } |
2642 | const SDValue &getMask() const { return getOperand(Num: 5); } |
2643 | const SDValue &getVectorLength() const { return getOperand(Num: 6); } |
2644 | |
2645 | static bool classof(const SDNode *N) { |
2646 | return N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE; |
2647 | } |
2648 | }; |
2649 | |
2650 | /// This base class is used to represent MLOAD and MSTORE nodes |
2651 | class MaskedLoadStoreSDNode : public MemSDNode { |
2652 | public: |
2653 | friend class SelectionDAG; |
2654 | |
2655 | MaskedLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order, |
2656 | const DebugLoc &dl, SDVTList VTs, |
2657 | ISD::MemIndexedMode AM, EVT MemVT, |
2658 | MachineMemOperand *MMO) |
2659 | : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { |
2660 | LSBaseSDNodeBits.AddressingMode = AM; |
2661 | assert(getAddressingMode() == AM && "Value truncated" ); |
2662 | } |
2663 | |
2664 | // MaskedLoadSDNode (Chain, ptr, offset, mask, passthru) |
2665 | // MaskedStoreSDNode (Chain, data, ptr, offset, mask) |
2666 | // Mask is a vector of i1 elements |
2667 | const SDValue &getOffset() const { |
2668 | return getOperand(Num: getOpcode() == ISD::MLOAD ? 2 : 3); |
2669 | } |
2670 | const SDValue &getMask() const { |
2671 | return getOperand(Num: getOpcode() == ISD::MLOAD ? 3 : 4); |
2672 | } |
2673 | |
2674 | /// Return the addressing mode for this load or store: |
2675 | /// unindexed, pre-inc, pre-dec, post-inc, or post-dec. |
2676 | ISD::MemIndexedMode getAddressingMode() const { |
2677 | return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode); |
2678 | } |
2679 | |
2680 | /// Return true if this is a pre/post inc/dec load/store. |
2681 | bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; } |
2682 | |
2683 | /// Return true if this is NOT a pre/post inc/dec load/store. |
2684 | bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; } |
2685 | |
2686 | static bool classof(const SDNode *N) { |
2687 | return N->getOpcode() == ISD::MLOAD || |
2688 | N->getOpcode() == ISD::MSTORE; |
2689 | } |
2690 | }; |
2691 | |
2692 | /// This class is used to represent an MLOAD node |
2693 | class MaskedLoadSDNode : public MaskedLoadStoreSDNode { |
2694 | public: |
2695 | friend class SelectionDAG; |
2696 | |
2697 | MaskedLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, |
2698 | ISD::MemIndexedMode AM, ISD::LoadExtType ETy, |
2699 | bool IsExpanding, EVT MemVT, MachineMemOperand *MMO) |
2700 | : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, VTs, AM, MemVT, MMO) { |
2701 | LoadSDNodeBits.ExtTy = ETy; |
2702 | LoadSDNodeBits.IsExpanding = IsExpanding; |
2703 | } |
2704 | |
2705 | ISD::LoadExtType getExtensionType() const { |
2706 | return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy); |
2707 | } |
2708 | |
2709 | const SDValue &getBasePtr() const { return getOperand(Num: 1); } |
2710 | const SDValue &getOffset() const { return getOperand(Num: 2); } |
2711 | const SDValue &getMask() const { return getOperand(Num: 3); } |
2712 | const SDValue &getPassThru() const { return getOperand(Num: 4); } |
2713 | |
2714 | static bool classof(const SDNode *N) { |
2715 | return N->getOpcode() == ISD::MLOAD; |
2716 | } |
2717 | |
2718 | bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; } |
2719 | }; |
2720 | |
2721 | /// This class is used to represent an MSTORE node |
2722 | class MaskedStoreSDNode : public MaskedLoadStoreSDNode { |
2723 | public: |
2724 | friend class SelectionDAG; |
2725 | |
2726 | MaskedStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, |
2727 | ISD::MemIndexedMode AM, bool isTrunc, bool isCompressing, |
2728 | EVT MemVT, MachineMemOperand *MMO) |
2729 | : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, VTs, AM, MemVT, MMO) { |
2730 | StoreSDNodeBits.IsTruncating = isTrunc; |
2731 | StoreSDNodeBits.IsCompressing = isCompressing; |
2732 | } |
2733 | |
2734 | /// Return true if the op does a truncation before store. |
2735 | /// For integers this is the same as doing a TRUNCATE and storing the result. |
2736 | /// For floats, it is the same as doing an FP_ROUND and storing the result. |
2737 | bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; } |
2738 | |
2739 | /// Returns true if the op does a compression to the vector before storing. |
2740 | /// The node contiguously stores the active elements (integers or floats) |
2741 | /// in src (those with their respective bit set in writemask k) to unaligned |
2742 | /// memory at base_addr. |
2743 | bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; } |
2744 | |
2745 | const SDValue &getValue() const { return getOperand(Num: 1); } |
2746 | const SDValue &getBasePtr() const { return getOperand(Num: 2); } |
2747 | const SDValue &getOffset() const { return getOperand(Num: 3); } |
2748 | const SDValue &getMask() const { return getOperand(Num: 4); } |
2749 | |
2750 | static bool classof(const SDNode *N) { |
2751 | return N->getOpcode() == ISD::MSTORE; |
2752 | } |
2753 | }; |
2754 | |
2755 | /// This is a base class used to represent |
2756 | /// VP_GATHER and VP_SCATTER nodes |
2757 | /// |
2758 | class VPGatherScatterSDNode : public MemSDNode { |
2759 | public: |
2760 | friend class SelectionDAG; |
2761 | |
2762 | VPGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order, |
2763 | const DebugLoc &dl, SDVTList VTs, EVT MemVT, |
2764 | MachineMemOperand *MMO, ISD::MemIndexType IndexType) |
2765 | : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { |
2766 | LSBaseSDNodeBits.AddressingMode = IndexType; |
2767 | assert(getIndexType() == IndexType && "Value truncated" ); |
2768 | } |
2769 | |
2770 | /// How is Index applied to BasePtr when computing addresses. |
2771 | ISD::MemIndexType getIndexType() const { |
2772 | return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode); |
2773 | } |
2774 | bool isIndexScaled() const { |
2775 | return !cast<ConstantSDNode>(Val: getScale())->isOne(); |
2776 | } |
2777 | bool isIndexSigned() const { return isIndexTypeSigned(IndexType: getIndexType()); } |
2778 | |
2779 | // In the both nodes address is Op1, mask is Op2: |
2780 | // VPGatherSDNode (Chain, base, index, scale, mask, vlen) |
2781 | // VPScatterSDNode (Chain, value, base, index, scale, mask, vlen) |
2782 | // Mask is a vector of i1 elements |
2783 | const SDValue &getBasePtr() const { |
2784 | return getOperand(Num: (getOpcode() == ISD::VP_GATHER) ? 1 : 2); |
2785 | } |
2786 | const SDValue &getIndex() const { |
2787 | return getOperand(Num: (getOpcode() == ISD::VP_GATHER) ? 2 : 3); |
2788 | } |
2789 | const SDValue &getScale() const { |
2790 | return getOperand(Num: (getOpcode() == ISD::VP_GATHER) ? 3 : 4); |
2791 | } |
2792 | const SDValue &getMask() const { |
2793 | return getOperand(Num: (getOpcode() == ISD::VP_GATHER) ? 4 : 5); |
2794 | } |
2795 | const SDValue &getVectorLength() const { |
2796 | return getOperand(Num: (getOpcode() == ISD::VP_GATHER) ? 5 : 6); |
2797 | } |
2798 | |
2799 | static bool classof(const SDNode *N) { |
2800 | return N->getOpcode() == ISD::VP_GATHER || |
2801 | N->getOpcode() == ISD::VP_SCATTER; |
2802 | } |
2803 | }; |
2804 | |
2805 | /// This class is used to represent an VP_GATHER node |
2806 | /// |
2807 | class VPGatherSDNode : public VPGatherScatterSDNode { |
2808 | public: |
2809 | friend class SelectionDAG; |
2810 | |
2811 | VPGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemVT, |
2812 | MachineMemOperand *MMO, ISD::MemIndexType IndexType) |
2813 | : VPGatherScatterSDNode(ISD::VP_GATHER, Order, dl, VTs, MemVT, MMO, |
2814 | IndexType) {} |
2815 | |
2816 | static bool classof(const SDNode *N) { |
2817 | return N->getOpcode() == ISD::VP_GATHER; |
2818 | } |
2819 | }; |
2820 | |
2821 | /// This class is used to represent an VP_SCATTER node |
2822 | /// |
2823 | class VPScatterSDNode : public VPGatherScatterSDNode { |
2824 | public: |
2825 | friend class SelectionDAG; |
2826 | |
2827 | VPScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemVT, |
2828 | MachineMemOperand *MMO, ISD::MemIndexType IndexType) |
2829 | : VPGatherScatterSDNode(ISD::VP_SCATTER, Order, dl, VTs, MemVT, MMO, |
2830 | IndexType) {} |
2831 | |
2832 | const SDValue &getValue() const { return getOperand(Num: 1); } |
2833 | |
2834 | static bool classof(const SDNode *N) { |
2835 | return N->getOpcode() == ISD::VP_SCATTER; |
2836 | } |
2837 | }; |
2838 | |
2839 | /// This is a base class used to represent |
2840 | /// MGATHER and MSCATTER nodes |
2841 | /// |
2842 | class MaskedGatherScatterSDNode : public MemSDNode { |
2843 | public: |
2844 | friend class SelectionDAG; |
2845 | |
2846 | MaskedGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order, |
2847 | const DebugLoc &dl, SDVTList VTs, EVT MemVT, |
2848 | MachineMemOperand *MMO, ISD::MemIndexType IndexType) |
2849 | : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { |
2850 | LSBaseSDNodeBits.AddressingMode = IndexType; |
2851 | assert(getIndexType() == IndexType && "Value truncated" ); |
2852 | } |
2853 | |
2854 | /// How is Index applied to BasePtr when computing addresses. |
2855 | ISD::MemIndexType getIndexType() const { |
2856 | return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode); |
2857 | } |
2858 | bool isIndexScaled() const { |
2859 | return !cast<ConstantSDNode>(Val: getScale())->isOne(); |
2860 | } |
2861 | bool isIndexSigned() const { return isIndexTypeSigned(IndexType: getIndexType()); } |
2862 | |
2863 | // In the both nodes address is Op1, mask is Op2: |
2864 | // MaskedGatherSDNode (Chain, passthru, mask, base, index, scale) |
2865 | // MaskedScatterSDNode (Chain, value, mask, base, index, scale) |
2866 | // Mask is a vector of i1 elements |
2867 | const SDValue &getBasePtr() const { return getOperand(Num: 3); } |
2868 | const SDValue &getIndex() const { return getOperand(Num: 4); } |
2869 | const SDValue &getMask() const { return getOperand(Num: 2); } |
2870 | const SDValue &getScale() const { return getOperand(Num: 5); } |
2871 | |
2872 | static bool classof(const SDNode *N) { |
2873 | return N->getOpcode() == ISD::MGATHER || |
2874 | N->getOpcode() == ISD::MSCATTER; |
2875 | } |
2876 | }; |
2877 | |
2878 | /// This class is used to represent an MGATHER node |
2879 | /// |
2880 | class MaskedGatherSDNode : public MaskedGatherScatterSDNode { |
2881 | public: |
2882 | friend class SelectionDAG; |
2883 | |
2884 | MaskedGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, |
2885 | EVT MemVT, MachineMemOperand *MMO, |
2886 | ISD::MemIndexType IndexType, ISD::LoadExtType ETy) |
2887 | : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, VTs, MemVT, MMO, |
2888 | IndexType) { |
2889 | LoadSDNodeBits.ExtTy = ETy; |
2890 | } |
2891 | |
2892 | const SDValue &getPassThru() const { return getOperand(Num: 1); } |
2893 | |
2894 | ISD::LoadExtType getExtensionType() const { |
2895 | return ISD::LoadExtType(LoadSDNodeBits.ExtTy); |
2896 | } |
2897 | |
2898 | static bool classof(const SDNode *N) { |
2899 | return N->getOpcode() == ISD::MGATHER; |
2900 | } |
2901 | }; |
2902 | |
2903 | /// This class is used to represent an MSCATTER node |
2904 | /// |
2905 | class MaskedScatterSDNode : public MaskedGatherScatterSDNode { |
2906 | public: |
2907 | friend class SelectionDAG; |
2908 | |
2909 | MaskedScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, |
2910 | EVT MemVT, MachineMemOperand *MMO, |
2911 | ISD::MemIndexType IndexType, bool IsTrunc) |
2912 | : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, VTs, MemVT, MMO, |
2913 | IndexType) { |
2914 | StoreSDNodeBits.IsTruncating = IsTrunc; |
2915 | } |
2916 | |
2917 | /// Return true if the op does a truncation before store. |
2918 | /// For integers this is the same as doing a TRUNCATE and storing the result. |
2919 | /// For floats, it is the same as doing an FP_ROUND and storing the result. |
2920 | bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; } |
2921 | |
2922 | const SDValue &getValue() const { return getOperand(Num: 1); } |
2923 | |
2924 | static bool classof(const SDNode *N) { |
2925 | return N->getOpcode() == ISD::MSCATTER; |
2926 | } |
2927 | }; |
2928 | |
2929 | class FPStateAccessSDNode : public MemSDNode { |
2930 | public: |
2931 | friend class SelectionDAG; |
2932 | |
2933 | FPStateAccessSDNode(unsigned NodeTy, unsigned Order, const DebugLoc &dl, |
2934 | SDVTList VTs, EVT MemVT, MachineMemOperand *MMO) |
2935 | : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { |
2936 | assert((NodeTy == ISD::GET_FPENV_MEM || NodeTy == ISD::SET_FPENV_MEM) && |
2937 | "Expected FP state access node" ); |
2938 | } |
2939 | |
2940 | static bool classof(const SDNode *N) { |
2941 | return N->getOpcode() == ISD::GET_FPENV_MEM || |
2942 | N->getOpcode() == ISD::SET_FPENV_MEM; |
2943 | } |
2944 | }; |
2945 | |
2946 | /// An SDNode that represents everything that will be needed |
2947 | /// to construct a MachineInstr. These nodes are created during the |
2948 | /// instruction selection proper phase. |
2949 | /// |
2950 | /// Note that the only supported way to set the `memoperands` is by calling the |
2951 | /// `SelectionDAG::setNodeMemRefs` function as the memory management happens |
2952 | /// inside the DAG rather than in the node. |
2953 | class MachineSDNode : public SDNode { |
2954 | private: |
2955 | friend class SelectionDAG; |
2956 | |
2957 | MachineSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL, SDVTList VTs) |
2958 | : SDNode(Opc, Order, DL, VTs) {} |
2959 | |
2960 | // We use a pointer union between a single `MachineMemOperand` pointer and |
2961 | // a pointer to an array of `MachineMemOperand` pointers. This is null when |
2962 | // the number of these is zero, the single pointer variant used when the |
2963 | // number is one, and the array is used for larger numbers. |
2964 | // |
2965 | // The array is allocated via the `SelectionDAG`'s allocator and so will |
2966 | // always live until the DAG is cleaned up and doesn't require ownership here. |
2967 | // |
2968 | // We can't use something simpler like `TinyPtrVector` here because `SDNode` |
2969 | // subclasses aren't managed in a conforming C++ manner. See the comments on |
2970 | // `SelectionDAG::MorphNodeTo` which details what all goes on, but the |
2971 | // constraint here is that these don't manage memory with their constructor or |
2972 | // destructor and can be initialized to a good state even if they start off |
2973 | // uninitialized. |
2974 | PointerUnion<MachineMemOperand *, MachineMemOperand **> MemRefs = {}; |
2975 | |
2976 | // Note that this could be folded into the above `MemRefs` member if doing so |
2977 | // is advantageous at some point. We don't need to store this in most cases. |
2978 | // However, at the moment this doesn't appear to make the allocation any |
2979 | // smaller and makes the code somewhat simpler to read. |
2980 | int NumMemRefs = 0; |
2981 | |
2982 | public: |
2983 | using mmo_iterator = ArrayRef<MachineMemOperand *>::const_iterator; |
2984 | |
2985 | ArrayRef<MachineMemOperand *> memoperands() const { |
2986 | // Special case the common cases. |
2987 | if (NumMemRefs == 0) |
2988 | return {}; |
2989 | if (NumMemRefs == 1) |
2990 | return ArrayRef(MemRefs.getAddrOfPtr1(), 1); |
2991 | |
2992 | // Otherwise we have an actual array. |
2993 | return ArrayRef(cast<MachineMemOperand **>(Val: MemRefs), NumMemRefs); |
2994 | } |
2995 | mmo_iterator memoperands_begin() const { return memoperands().begin(); } |
2996 | mmo_iterator memoperands_end() const { return memoperands().end(); } |
2997 | bool memoperands_empty() const { return memoperands().empty(); } |
2998 | |
2999 | /// Clear out the memory reference descriptor list. |
3000 | void clearMemRefs() { |
3001 | MemRefs = nullptr; |
3002 | NumMemRefs = 0; |
3003 | } |
3004 | |
3005 | static bool classof(const SDNode *N) { |
3006 | return N->isMachineOpcode(); |
3007 | } |
3008 | }; |
3009 | |
3010 | /// An SDNode that records if a register contains a value that is guaranteed to |
3011 | /// be aligned accordingly. |
3012 | class AssertAlignSDNode : public SDNode { |
3013 | Align Alignment; |
3014 | |
3015 | public: |
3016 | AssertAlignSDNode(unsigned Order, const DebugLoc &DL, EVT VT, Align A) |
3017 | : SDNode(ISD::AssertAlign, Order, DL, getSDVTList(VT)), Alignment(A) {} |
3018 | |
3019 | Align getAlign() const { return Alignment; } |
3020 | |
3021 | static bool classof(const SDNode *N) { |
3022 | return N->getOpcode() == ISD::AssertAlign; |
3023 | } |
3024 | }; |
3025 | |
3026 | class SDNodeIterator { |
3027 | const SDNode *Node; |
3028 | unsigned Operand; |
3029 | |
3030 | SDNodeIterator(const SDNode *N, unsigned Op) : Node(N), Operand(Op) {} |
3031 | |
3032 | public: |
3033 | using iterator_category = std::forward_iterator_tag; |
3034 | using value_type = SDNode; |
3035 | using difference_type = std::ptrdiff_t; |
3036 | using pointer = value_type *; |
3037 | using reference = value_type &; |
3038 | |
3039 | bool operator==(const SDNodeIterator& x) const { |
3040 | return Operand == x.Operand; |
3041 | } |
3042 | bool operator!=(const SDNodeIterator& x) const { return !operator==(x); } |
3043 | |
3044 | pointer operator*() const { |
3045 | return Node->getOperand(Num: Operand).getNode(); |
3046 | } |
3047 | pointer operator->() const { return operator*(); } |
3048 | |
3049 | SDNodeIterator& operator++() { // Preincrement |
3050 | ++Operand; |
3051 | return *this; |
3052 | } |
3053 | SDNodeIterator operator++(int) { // Postincrement |
3054 | SDNodeIterator tmp = *this; ++*this; return tmp; |
3055 | } |
3056 | size_t operator-(SDNodeIterator Other) const { |
3057 | assert(Node == Other.Node && |
3058 | "Cannot compare iterators of two different nodes!" ); |
3059 | return Operand - Other.Operand; |
3060 | } |
3061 | |
3062 | static SDNodeIterator begin(const SDNode *N) { return SDNodeIterator(N, 0); } |
3063 | static SDNodeIterator end (const SDNode *N) { |
3064 | return SDNodeIterator(N, N->getNumOperands()); |
3065 | } |
3066 | |
3067 | unsigned getOperand() const { return Operand; } |
3068 | const SDNode *getNode() const { return Node; } |
3069 | }; |
3070 | |
3071 | template <> struct GraphTraits<SDNode*> { |
3072 | using NodeRef = SDNode *; |
3073 | using ChildIteratorType = SDNodeIterator; |
3074 | |
3075 | static NodeRef getEntryNode(SDNode *N) { return N; } |
3076 | |
3077 | static ChildIteratorType child_begin(NodeRef N) { |
3078 | return SDNodeIterator::begin(N); |
3079 | } |
3080 | |
3081 | static ChildIteratorType child_end(NodeRef N) { |
3082 | return SDNodeIterator::end(N); |
3083 | } |
3084 | }; |
3085 | |
3086 | /// A representation of the largest SDNode, for use in sizeof(). |
3087 | /// |
3088 | /// This needs to be a union because the largest node differs on 32 bit systems |
3089 | /// with 4 and 8 byte pointer alignment, respectively. |
3090 | using LargestSDNode = AlignedCharArrayUnion<AtomicSDNode, TargetIndexSDNode, |
3091 | BlockAddressSDNode, |
3092 | GlobalAddressSDNode, |
3093 | PseudoProbeSDNode>; |
3094 | |
3095 | /// The SDNode class with the greatest alignment requirement. |
3096 | using MostAlignedSDNode = GlobalAddressSDNode; |
3097 | |
3098 | namespace ISD { |
3099 | |
3100 | /// Returns true if the specified node is a non-extending and unindexed load. |
3101 | inline bool isNormalLoad(const SDNode *N) { |
3102 | auto *Ld = dyn_cast<LoadSDNode>(Val: N); |
3103 | return Ld && Ld->getExtensionType() == ISD::NON_EXTLOAD && |
3104 | Ld->getAddressingMode() == ISD::UNINDEXED; |
3105 | } |
3106 | |
3107 | /// Returns true if the specified node is a non-extending load. |
3108 | inline bool isNON_EXTLoad(const SDNode *N) { |
3109 | auto *Ld = dyn_cast<LoadSDNode>(Val: N); |
3110 | return Ld && Ld->getExtensionType() == ISD::NON_EXTLOAD; |
3111 | } |
3112 | |
3113 | /// Returns true if the specified node is a EXTLOAD. |
3114 | inline bool isEXTLoad(const SDNode *N) { |
3115 | auto *Ld = dyn_cast<LoadSDNode>(Val: N); |
3116 | return Ld && Ld->getExtensionType() == ISD::EXTLOAD; |
3117 | } |
3118 | |
3119 | /// Returns true if the specified node is a SEXTLOAD. |
3120 | inline bool isSEXTLoad(const SDNode *N) { |
3121 | auto *Ld = dyn_cast<LoadSDNode>(Val: N); |
3122 | return Ld && Ld->getExtensionType() == ISD::SEXTLOAD; |
3123 | } |
3124 | |
3125 | /// Returns true if the specified node is a ZEXTLOAD. |
3126 | inline bool isZEXTLoad(const SDNode *N) { |
3127 | auto *Ld = dyn_cast<LoadSDNode>(Val: N); |
3128 | return Ld && Ld->getExtensionType() == ISD::ZEXTLOAD; |
3129 | } |
3130 | |
3131 | /// Returns true if the specified node is an unindexed load. |
3132 | inline bool isUNINDEXEDLoad(const SDNode *N) { |
3133 | auto *Ld = dyn_cast<LoadSDNode>(Val: N); |
3134 | return Ld && Ld->getAddressingMode() == ISD::UNINDEXED; |
3135 | } |
3136 | |
3137 | /// Returns true if the specified node is a non-truncating |
3138 | /// and unindexed store. |
3139 | inline bool isNormalStore(const SDNode *N) { |
3140 | auto *St = dyn_cast<StoreSDNode>(Val: N); |
3141 | return St && !St->isTruncatingStore() && |
3142 | St->getAddressingMode() == ISD::UNINDEXED; |
3143 | } |
3144 | |
3145 | /// Returns true if the specified node is an unindexed store. |
3146 | inline bool isUNINDEXEDStore(const SDNode *N) { |
3147 | auto *St = dyn_cast<StoreSDNode>(Val: N); |
3148 | return St && St->getAddressingMode() == ISD::UNINDEXED; |
3149 | } |
3150 | |
3151 | /// Attempt to match a unary predicate against a scalar/splat constant or |
3152 | /// every element of a constant BUILD_VECTOR. |
3153 | /// If AllowUndef is true, then UNDEF elements will pass nullptr to Match. |
3154 | template <typename ConstNodeType> |
3155 | bool matchUnaryPredicateImpl(SDValue Op, |
3156 | std::function<bool(ConstNodeType *)> Match, |
3157 | bool AllowUndefs = false); |
3158 | |
3159 | /// Hook for matching ConstantSDNode predicate |
3160 | inline bool matchUnaryPredicate(SDValue Op, |
3161 | std::function<bool(ConstantSDNode *)> Match, |
3162 | bool AllowUndefs = false) { |
3163 | return matchUnaryPredicateImpl<ConstantSDNode>(Op, Match, AllowUndefs); |
3164 | } |
3165 | |
3166 | /// Hook for matching ConstantFPSDNode predicate |
3167 | inline bool |
3168 | matchUnaryFpPredicate(SDValue Op, |
3169 | std::function<bool(ConstantFPSDNode *)> Match, |
3170 | bool AllowUndefs = false) { |
3171 | return matchUnaryPredicateImpl<ConstantFPSDNode>(Op, Match, AllowUndefs); |
3172 | } |
3173 | |
3174 | /// Attempt to match a binary predicate against a pair of scalar/splat |
3175 | /// constants or every element of a pair of constant BUILD_VECTORs. |
3176 | /// If AllowUndef is true, then UNDEF elements will pass nullptr to Match. |
3177 | /// If AllowTypeMismatch is true then RetType + ArgTypes don't need to match. |
3178 | bool matchBinaryPredicate( |
3179 | SDValue LHS, SDValue RHS, |
3180 | std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match, |
3181 | bool AllowUndefs = false, bool AllowTypeMismatch = false); |
3182 | |
3183 | /// Returns true if the specified value is the overflow result from one |
3184 | /// of the overflow intrinsic nodes. |
3185 | inline bool isOverflowIntrOpRes(SDValue Op) { |
3186 | unsigned Opc = Op.getOpcode(); |
3187 | return (Op.getResNo() == 1 && |
3188 | (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO || |
3189 | Opc == ISD::USUBO || Opc == ISD::SMULO || Opc == ISD::UMULO)); |
3190 | } |
3191 | |
3192 | } // end namespace ISD |
3193 | |
3194 | } // end namespace llvm |
3195 | |
3196 | #endif // LLVM_CODEGEN_SELECTIONDAGNODES_H |
3197 | |