1 | //===-- AMDGPUISelDAGToDAG.h - A dag to dag inst selector for AMDGPU ----===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //==-----------------------------------------------------------------------===// |
8 | // |
9 | /// \file |
10 | /// Defines an instruction selector for the AMDGPU target. |
11 | // |
12 | //===----------------------------------------------------------------------===// |
13 | |
14 | #ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUISELDAGTODAG_H |
15 | #define LLVM_LIB_TARGET_AMDGPU_AMDGPUISELDAGTODAG_H |
16 | |
17 | #include "GCNSubtarget.h" |
18 | #include "SIMachineFunctionInfo.h" |
19 | #include "SIModeRegisterDefaults.h" |
20 | #include "llvm/CodeGen/SelectionDAGISel.h" |
21 | #include "llvm/Target/TargetMachine.h" |
22 | |
23 | using namespace llvm; |
24 | |
25 | namespace { |
26 | |
27 | static inline bool isNullConstantOrUndef(SDValue V) { |
28 | return V.isUndef() || isNullConstant(V); |
29 | } |
30 | |
31 | static inline bool getConstantValue(SDValue N, uint32_t &Out) { |
32 | // This is only used for packed vectors, where using 0 for undef should |
33 | // always be good. |
34 | if (N.isUndef()) { |
35 | Out = 0; |
36 | return true; |
37 | } |
38 | |
39 | if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: N)) { |
40 | Out = C->getAPIntValue().getSExtValue(); |
41 | return true; |
42 | } |
43 | |
44 | if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: N)) { |
45 | Out = C->getValueAPF().bitcastToAPInt().getSExtValue(); |
46 | return true; |
47 | } |
48 | |
49 | return false; |
50 | } |
51 | |
52 | // TODO: Handle undef as zero |
53 | static inline SDNode *packConstantV2I16(const SDNode *N, SelectionDAG &DAG) { |
54 | assert(N->getOpcode() == ISD::BUILD_VECTOR && N->getNumOperands() == 2); |
55 | uint32_t LHSVal, RHSVal; |
56 | if (getConstantValue(N: N->getOperand(Num: 0), Out&: LHSVal) && |
57 | getConstantValue(N: N->getOperand(Num: 1), Out&: RHSVal)) { |
58 | SDLoc SL(N); |
59 | uint32_t K = (LHSVal & 0xffff) | (RHSVal << 16); |
60 | return DAG.getMachineNode(AMDGPU::S_MOV_B32, SL, N->getValueType(ResNo: 0), |
61 | DAG.getTargetConstant(K, SL, MVT::i32)); |
62 | } |
63 | |
64 | return nullptr; |
65 | } |
66 | |
67 | } // namespace |
68 | |
69 | /// AMDGPU specific code to select AMDGPU machine instructions for |
70 | /// SelectionDAG operations. |
71 | class AMDGPUDAGToDAGISel : public SelectionDAGISel { |
72 | // Subtarget - Keep a pointer to the AMDGPU Subtarget around so that we can |
73 | // make the right decision when generating code for different targets. |
74 | const GCNSubtarget *Subtarget; |
75 | |
76 | // Default FP mode for the current function. |
77 | SIModeRegisterDefaults Mode; |
78 | |
79 | bool EnableLateStructurizeCFG; |
80 | |
81 | // Instructions that will be lowered with a final instruction that zeros the |
82 | // high result bits. |
83 | bool fp16SrcZerosHighBits(unsigned Opc) const; |
84 | |
85 | public: |
86 | static char ID; |
87 | |
88 | AMDGPUDAGToDAGISel() = delete; |
89 | |
90 | explicit AMDGPUDAGToDAGISel(TargetMachine &TM, CodeGenOptLevel OptLevel); |
91 | ~AMDGPUDAGToDAGISel() override = default; |
92 | |
93 | void getAnalysisUsage(AnalysisUsage &AU) const override; |
94 | |
95 | bool matchLoadD16FromBuildVector(SDNode *N) const; |
96 | |
97 | bool runOnMachineFunction(MachineFunction &MF) override; |
98 | void PreprocessISelDAG() override; |
99 | void Select(SDNode *N) override; |
100 | StringRef getPassName() const override; |
101 | void PostprocessISelDAG() override; |
102 | |
103 | protected: |
104 | void SelectBuildVector(SDNode *N, unsigned RegClassID); |
105 | |
106 | private: |
107 | std::pair<SDValue, SDValue> foldFrameIndex(SDValue N) const; |
108 | |
109 | bool isInlineImmediate(const SDNode *N) const; |
110 | |
111 | bool isInlineImmediate(const APInt &Imm) const { |
112 | return Subtarget->getInstrInfo()->isInlineConstant(Imm); |
113 | } |
114 | |
115 | bool isInlineImmediate(const APFloat &Imm) const { |
116 | return Subtarget->getInstrInfo()->isInlineConstant(Imm); |
117 | } |
118 | |
119 | bool isVGPRImm(const SDNode *N) const; |
120 | bool isUniformLoad(const SDNode *N) const; |
121 | bool isUniformBr(const SDNode *N) const; |
122 | |
123 | // Returns true if ISD::AND SDNode `N`'s masking of the shift amount operand's |
124 | // `ShAmtBits` bits is unneeded. |
125 | bool isUnneededShiftMask(const SDNode *N, unsigned ShAmtBits) const; |
126 | |
127 | bool isBaseWithConstantOffset64(SDValue Addr, SDValue &LHS, |
128 | SDValue &RHS) const; |
129 | |
130 | MachineSDNode *buildSMovImm64(SDLoc &DL, uint64_t Val, EVT VT) const; |
131 | |
132 | SDNode *glueCopyToOp(SDNode *N, SDValue NewChain, SDValue Glue) const; |
133 | SDNode *glueCopyToM0(SDNode *N, SDValue Val) const; |
134 | SDNode *glueCopyToM0LDSInit(SDNode *N) const; |
135 | |
136 | const TargetRegisterClass *getOperandRegClass(SDNode *N, unsigned OpNo) const; |
137 | virtual bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, SDValue &Offset); |
138 | virtual bool SelectADDRIndirect(SDValue Addr, SDValue &Base, SDValue &Offset); |
139 | bool isDSOffsetLegal(SDValue Base, unsigned Offset) const; |
140 | bool isDSOffset2Legal(SDValue Base, unsigned Offset0, unsigned Offset1, |
141 | unsigned Size) const; |
142 | |
143 | bool isFlatScratchBaseLegal(SDValue Addr) const; |
144 | bool isFlatScratchBaseLegalSV(SDValue Addr) const; |
145 | bool isFlatScratchBaseLegalSVImm(SDValue Addr) const; |
146 | |
147 | bool SelectDS1Addr1Offset(SDValue Ptr, SDValue &Base, SDValue &Offset) const; |
148 | bool SelectDS64Bit4ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0, |
149 | SDValue &Offset1) const; |
150 | bool SelectDS128Bit8ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0, |
151 | SDValue &Offset1) const; |
152 | bool SelectDSReadWrite2(SDValue Ptr, SDValue &Base, SDValue &Offset0, |
153 | SDValue &Offset1, unsigned Size) const; |
154 | bool SelectMUBUF(SDValue Addr, SDValue &SRsrc, SDValue &VAddr, |
155 | SDValue &SOffset, SDValue &Offset, SDValue &Offen, |
156 | SDValue &Idxen, SDValue &Addr64) const; |
157 | bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, SDValue &VAddr, |
158 | SDValue &SOffset, SDValue &Offset) const; |
159 | bool SelectMUBUFScratchOffen(SDNode *Parent, SDValue Addr, SDValue &RSrc, |
160 | SDValue &VAddr, SDValue &SOffset, |
161 | SDValue &ImmOffset) const; |
162 | bool SelectMUBUFScratchOffset(SDNode *Parent, SDValue Addr, SDValue &SRsrc, |
163 | SDValue &Soffset, SDValue &Offset) const; |
164 | |
165 | bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset, |
166 | SDValue &Offset) const; |
167 | bool SelectBUFSOffset(SDValue Addr, SDValue &SOffset) const; |
168 | |
169 | bool SelectFlatOffsetImpl(SDNode *N, SDValue Addr, SDValue &VAddr, |
170 | SDValue &Offset, uint64_t FlatVariant) const; |
171 | bool SelectFlatOffset(SDNode *N, SDValue Addr, SDValue &VAddr, |
172 | SDValue &Offset) const; |
173 | bool SelectGlobalOffset(SDNode *N, SDValue Addr, SDValue &VAddr, |
174 | SDValue &Offset) const; |
175 | bool SelectScratchOffset(SDNode *N, SDValue Addr, SDValue &VAddr, |
176 | SDValue &Offset) const; |
177 | bool SelectGlobalSAddr(SDNode *N, SDValue Addr, SDValue &SAddr, |
178 | SDValue &VOffset, SDValue &Offset) const; |
179 | bool SelectScratchSAddr(SDNode *N, SDValue Addr, SDValue &SAddr, |
180 | SDValue &Offset) const; |
181 | bool checkFlatScratchSVSSwizzleBug(SDValue VAddr, SDValue SAddr, |
182 | uint64_t ImmOffset) const; |
183 | bool SelectScratchSVAddr(SDNode *N, SDValue Addr, SDValue &VAddr, |
184 | SDValue &SAddr, SDValue &Offset) const; |
185 | |
186 | bool SelectSMRDOffset(SDValue ByteOffsetNode, SDValue *SOffset, |
187 | SDValue *Offset, bool Imm32Only = false, |
188 | bool IsBuffer = false) const; |
189 | SDValue Expand32BitAddress(SDValue Addr) const; |
190 | bool SelectSMRDBaseOffset(SDValue Addr, SDValue &SBase, SDValue *SOffset, |
191 | SDValue *Offset, bool Imm32Only = false, |
192 | bool IsBuffer = false) const; |
193 | bool SelectSMRD(SDValue Addr, SDValue &SBase, SDValue *SOffset, |
194 | SDValue *Offset, bool Imm32Only = false) const; |
195 | bool SelectSMRDImm(SDValue Addr, SDValue &SBase, SDValue &Offset) const; |
196 | bool SelectSMRDImm32(SDValue Addr, SDValue &SBase, SDValue &Offset) const; |
197 | bool SelectSMRDSgpr(SDValue Addr, SDValue &SBase, SDValue &SOffset) const; |
198 | bool SelectSMRDSgprImm(SDValue Addr, SDValue &SBase, SDValue &SOffset, |
199 | SDValue &Offset) const; |
200 | bool SelectSMRDBufferImm(SDValue N, SDValue &Offset) const; |
201 | bool SelectSMRDBufferImm32(SDValue N, SDValue &Offset) const; |
202 | bool SelectSMRDBufferSgprImm(SDValue N, SDValue &SOffset, |
203 | SDValue &Offset) const; |
204 | bool SelectMOVRELOffset(SDValue Index, SDValue &Base, SDValue &Offset) const; |
205 | |
206 | bool SelectVOP3ModsImpl(SDValue In, SDValue &Src, unsigned &SrcMods, |
207 | bool IsCanonicalizing = true, |
208 | bool AllowAbs = true) const; |
209 | bool SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods) const; |
210 | bool SelectVOP3ModsNonCanonicalizing(SDValue In, SDValue &Src, |
211 | SDValue &SrcMods) const; |
212 | bool SelectVOP3BMods(SDValue In, SDValue &Src, SDValue &SrcMods) const; |
213 | bool SelectVOP3NoMods(SDValue In, SDValue &Src) const; |
214 | bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods, |
215 | SDValue &Clamp, SDValue &Omod) const; |
216 | bool SelectVOP3BMods0(SDValue In, SDValue &Src, SDValue &SrcMods, |
217 | SDValue &Clamp, SDValue &Omod) const; |
218 | bool SelectVOP3NoMods0(SDValue In, SDValue &Src, SDValue &SrcMods, |
219 | SDValue &Clamp, SDValue &Omod) const; |
220 | |
221 | bool SelectVINTERPModsImpl(SDValue In, SDValue &Src, SDValue &SrcMods, |
222 | bool OpSel) const; |
223 | bool SelectVINTERPMods(SDValue In, SDValue &Src, SDValue &SrcMods) const; |
224 | bool SelectVINTERPModsHi(SDValue In, SDValue &Src, SDValue &SrcMods) const; |
225 | |
226 | bool SelectVOP3OMods(SDValue In, SDValue &Src, SDValue &Clamp, |
227 | SDValue &Omod) const; |
228 | |
229 | bool SelectVOP3PMods(SDValue In, SDValue &Src, SDValue &SrcMods, |
230 | bool IsDOT = false) const; |
231 | bool SelectVOP3PModsDOT(SDValue In, SDValue &Src, SDValue &SrcMods) const; |
232 | |
233 | bool SelectVOP3PModsNeg(SDValue In, SDValue &Src) const; |
234 | bool SelectWMMAOpSelVOP3PMods(SDValue In, SDValue &Src) const; |
235 | |
236 | bool SelectWMMAModsF32NegAbs(SDValue In, SDValue &Src, |
237 | SDValue &SrcMods) const; |
238 | bool SelectWMMAModsF16Neg(SDValue In, SDValue &Src, SDValue &SrcMods) const; |
239 | bool SelectWMMAModsF16NegAbs(SDValue In, SDValue &Src, |
240 | SDValue &SrcMods) const; |
241 | bool SelectWMMAVISrc(SDValue In, SDValue &Src) const; |
242 | |
243 | bool SelectSWMMACIndex8(SDValue In, SDValue &Src, SDValue &IndexKey) const; |
244 | bool SelectSWMMACIndex16(SDValue In, SDValue &Src, SDValue &IndexKey) const; |
245 | |
246 | bool SelectVOP3OpSel(SDValue In, SDValue &Src, SDValue &SrcMods) const; |
247 | |
248 | bool SelectVOP3OpSelMods(SDValue In, SDValue &Src, SDValue &SrcMods) const; |
249 | bool SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src, |
250 | unsigned &Mods) const; |
251 | bool SelectVOP3PMadMixModsExt(SDValue In, SDValue &Src, |
252 | SDValue &SrcMods) const; |
253 | bool SelectVOP3PMadMixMods(SDValue In, SDValue &Src, SDValue &SrcMods) const; |
254 | |
255 | SDValue getHi16Elt(SDValue In) const; |
256 | |
257 | SDValue getMaterializedScalarImm32(int64_t Val, const SDLoc &DL) const; |
258 | |
259 | void SelectADD_SUB_I64(SDNode *N); |
260 | void SelectAddcSubb(SDNode *N); |
261 | void SelectUADDO_USUBO(SDNode *N); |
262 | void SelectDIV_SCALE(SDNode *N); |
263 | void SelectMAD_64_32(SDNode *N); |
264 | void SelectMUL_LOHI(SDNode *N); |
265 | void SelectFMA_W_CHAIN(SDNode *N); |
266 | void SelectFMUL_W_CHAIN(SDNode *N); |
267 | SDNode *getBFE32(bool IsSigned, const SDLoc &DL, SDValue Val, uint32_t Offset, |
268 | uint32_t Width); |
269 | void SelectS_BFEFromShifts(SDNode *N); |
270 | void SelectS_BFE(SDNode *N); |
271 | bool isCBranchSCC(const SDNode *N) const; |
272 | void SelectBRCOND(SDNode *N); |
273 | void SelectFMAD_FMA(SDNode *N); |
274 | void SelectFP_EXTEND(SDNode *N); |
275 | void SelectDSAppendConsume(SDNode *N, unsigned IntrID); |
276 | void SelectDSBvhStackIntrinsic(SDNode *N); |
277 | void SelectDS_GWS(SDNode *N, unsigned IntrID); |
278 | void SelectInterpP1F16(SDNode *N); |
279 | void SelectINTRINSIC_W_CHAIN(SDNode *N); |
280 | void SelectINTRINSIC_WO_CHAIN(SDNode *N); |
281 | void SelectINTRINSIC_VOID(SDNode *N); |
282 | void SelectWAVE_ADDRESS(SDNode *N); |
283 | void SelectSTACKRESTORE(SDNode *N); |
284 | |
285 | protected: |
286 | // Include the pieces autogenerated from the target description. |
287 | #include "AMDGPUGenDAGISel.inc" |
288 | }; |
289 | |
290 | #endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUISELDAGTODAG_H |
291 | |