1//===-- AMDGPUISelDAGToDAG.h - A dag to dag inst selector for AMDGPU ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//==-----------------------------------------------------------------------===//
8//
9/// \file
10/// Defines an instruction selector for the AMDGPU target.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUISELDAGTODAG_H
15#define LLVM_LIB_TARGET_AMDGPU_AMDGPUISELDAGTODAG_H
16
17#include "GCNSubtarget.h"
18#include "SIMachineFunctionInfo.h"
19#include "SIModeRegisterDefaults.h"
20#include "llvm/CodeGen/SelectionDAGISel.h"
21#include "llvm/Target/TargetMachine.h"
22
23using namespace llvm;
24
25namespace {
26
27static inline bool isNullConstantOrUndef(SDValue V) {
28 return V.isUndef() || isNullConstant(V);
29}
30
31static inline bool getConstantValue(SDValue N, uint32_t &Out) {
32 // This is only used for packed vectors, where using 0 for undef should
33 // always be good.
34 if (N.isUndef()) {
35 Out = 0;
36 return true;
37 }
38
39 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: N)) {
40 Out = C->getAPIntValue().getSExtValue();
41 return true;
42 }
43
44 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: N)) {
45 Out = C->getValueAPF().bitcastToAPInt().getSExtValue();
46 return true;
47 }
48
49 return false;
50}
51
52// TODO: Handle undef as zero
53static inline SDNode *packConstantV2I16(const SDNode *N, SelectionDAG &DAG) {
54 assert(N->getOpcode() == ISD::BUILD_VECTOR && N->getNumOperands() == 2);
55 uint32_t LHSVal, RHSVal;
56 if (getConstantValue(N: N->getOperand(Num: 0), Out&: LHSVal) &&
57 getConstantValue(N: N->getOperand(Num: 1), Out&: RHSVal)) {
58 SDLoc SL(N);
59 uint32_t K = (LHSVal & 0xffff) | (RHSVal << 16);
60 return DAG.getMachineNode(AMDGPU::S_MOV_B32, SL, N->getValueType(ResNo: 0),
61 DAG.getTargetConstant(K, SL, MVT::i32));
62 }
63
64 return nullptr;
65}
66
67} // namespace
68
69/// AMDGPU specific code to select AMDGPU machine instructions for
70/// SelectionDAG operations.
71class AMDGPUDAGToDAGISel : public SelectionDAGISel {
72 // Subtarget - Keep a pointer to the AMDGPU Subtarget around so that we can
73 // make the right decision when generating code for different targets.
74 const GCNSubtarget *Subtarget;
75
76 // Default FP mode for the current function.
77 SIModeRegisterDefaults Mode;
78
79 bool EnableLateStructurizeCFG;
80
81 // Instructions that will be lowered with a final instruction that zeros the
82 // high result bits.
83 bool fp16SrcZerosHighBits(unsigned Opc) const;
84
85public:
86 static char ID;
87
88 AMDGPUDAGToDAGISel() = delete;
89
90 explicit AMDGPUDAGToDAGISel(TargetMachine &TM, CodeGenOptLevel OptLevel);
91 ~AMDGPUDAGToDAGISel() override = default;
92
93 void getAnalysisUsage(AnalysisUsage &AU) const override;
94
95 bool matchLoadD16FromBuildVector(SDNode *N) const;
96
97 bool runOnMachineFunction(MachineFunction &MF) override;
98 void PreprocessISelDAG() override;
99 void Select(SDNode *N) override;
100 StringRef getPassName() const override;
101 void PostprocessISelDAG() override;
102
103protected:
104 void SelectBuildVector(SDNode *N, unsigned RegClassID);
105
106private:
107 std::pair<SDValue, SDValue> foldFrameIndex(SDValue N) const;
108
109 bool isInlineImmediate(const SDNode *N) const;
110
111 bool isInlineImmediate(const APInt &Imm) const {
112 return Subtarget->getInstrInfo()->isInlineConstant(Imm);
113 }
114
115 bool isInlineImmediate(const APFloat &Imm) const {
116 return Subtarget->getInstrInfo()->isInlineConstant(Imm);
117 }
118
119 bool isVGPRImm(const SDNode *N) const;
120 bool isUniformLoad(const SDNode *N) const;
121 bool isUniformBr(const SDNode *N) const;
122
123 // Returns true if ISD::AND SDNode `N`'s masking of the shift amount operand's
124 // `ShAmtBits` bits is unneeded.
125 bool isUnneededShiftMask(const SDNode *N, unsigned ShAmtBits) const;
126
127 bool isBaseWithConstantOffset64(SDValue Addr, SDValue &LHS,
128 SDValue &RHS) const;
129
130 MachineSDNode *buildSMovImm64(SDLoc &DL, uint64_t Val, EVT VT) const;
131
132 SDNode *glueCopyToOp(SDNode *N, SDValue NewChain, SDValue Glue) const;
133 SDNode *glueCopyToM0(SDNode *N, SDValue Val) const;
134 SDNode *glueCopyToM0LDSInit(SDNode *N) const;
135
136 const TargetRegisterClass *getOperandRegClass(SDNode *N, unsigned OpNo) const;
137 virtual bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, SDValue &Offset);
138 virtual bool SelectADDRIndirect(SDValue Addr, SDValue &Base, SDValue &Offset);
139 bool isDSOffsetLegal(SDValue Base, unsigned Offset) const;
140 bool isDSOffset2Legal(SDValue Base, unsigned Offset0, unsigned Offset1,
141 unsigned Size) const;
142
143 bool isFlatScratchBaseLegal(SDValue Addr) const;
144 bool isFlatScratchBaseLegalSV(SDValue Addr) const;
145 bool isFlatScratchBaseLegalSVImm(SDValue Addr) const;
146
147 bool SelectDS1Addr1Offset(SDValue Ptr, SDValue &Base, SDValue &Offset) const;
148 bool SelectDS64Bit4ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0,
149 SDValue &Offset1) const;
150 bool SelectDS128Bit8ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0,
151 SDValue &Offset1) const;
152 bool SelectDSReadWrite2(SDValue Ptr, SDValue &Base, SDValue &Offset0,
153 SDValue &Offset1, unsigned Size) const;
154 bool SelectMUBUF(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
155 SDValue &SOffset, SDValue &Offset, SDValue &Offen,
156 SDValue &Idxen, SDValue &Addr64) const;
157 bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
158 SDValue &SOffset, SDValue &Offset) const;
159 bool SelectMUBUFScratchOffen(SDNode *Parent, SDValue Addr, SDValue &RSrc,
160 SDValue &VAddr, SDValue &SOffset,
161 SDValue &ImmOffset) const;
162 bool SelectMUBUFScratchOffset(SDNode *Parent, SDValue Addr, SDValue &SRsrc,
163 SDValue &Soffset, SDValue &Offset) const;
164
165 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
166 SDValue &Offset) const;
167 bool SelectBUFSOffset(SDValue Addr, SDValue &SOffset) const;
168
169 bool SelectFlatOffsetImpl(SDNode *N, SDValue Addr, SDValue &VAddr,
170 SDValue &Offset, uint64_t FlatVariant) const;
171 bool SelectFlatOffset(SDNode *N, SDValue Addr, SDValue &VAddr,
172 SDValue &Offset) const;
173 bool SelectGlobalOffset(SDNode *N, SDValue Addr, SDValue &VAddr,
174 SDValue &Offset) const;
175 bool SelectScratchOffset(SDNode *N, SDValue Addr, SDValue &VAddr,
176 SDValue &Offset) const;
177 bool SelectGlobalSAddr(SDNode *N, SDValue Addr, SDValue &SAddr,
178 SDValue &VOffset, SDValue &Offset) const;
179 bool SelectScratchSAddr(SDNode *N, SDValue Addr, SDValue &SAddr,
180 SDValue &Offset) const;
181 bool checkFlatScratchSVSSwizzleBug(SDValue VAddr, SDValue SAddr,
182 uint64_t ImmOffset) const;
183 bool SelectScratchSVAddr(SDNode *N, SDValue Addr, SDValue &VAddr,
184 SDValue &SAddr, SDValue &Offset) const;
185
186 bool SelectSMRDOffset(SDValue ByteOffsetNode, SDValue *SOffset,
187 SDValue *Offset, bool Imm32Only = false,
188 bool IsBuffer = false) const;
189 SDValue Expand32BitAddress(SDValue Addr) const;
190 bool SelectSMRDBaseOffset(SDValue Addr, SDValue &SBase, SDValue *SOffset,
191 SDValue *Offset, bool Imm32Only = false,
192 bool IsBuffer = false) const;
193 bool SelectSMRD(SDValue Addr, SDValue &SBase, SDValue *SOffset,
194 SDValue *Offset, bool Imm32Only = false) const;
195 bool SelectSMRDImm(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
196 bool SelectSMRDImm32(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
197 bool SelectSMRDSgpr(SDValue Addr, SDValue &SBase, SDValue &SOffset) const;
198 bool SelectSMRDSgprImm(SDValue Addr, SDValue &SBase, SDValue &SOffset,
199 SDValue &Offset) const;
200 bool SelectSMRDBufferImm(SDValue N, SDValue &Offset) const;
201 bool SelectSMRDBufferImm32(SDValue N, SDValue &Offset) const;
202 bool SelectSMRDBufferSgprImm(SDValue N, SDValue &SOffset,
203 SDValue &Offset) const;
204 bool SelectMOVRELOffset(SDValue Index, SDValue &Base, SDValue &Offset) const;
205
206 bool SelectVOP3ModsImpl(SDValue In, SDValue &Src, unsigned &SrcMods,
207 bool IsCanonicalizing = true,
208 bool AllowAbs = true) const;
209 bool SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
210 bool SelectVOP3ModsNonCanonicalizing(SDValue In, SDValue &Src,
211 SDValue &SrcMods) const;
212 bool SelectVOP3BMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
213 bool SelectVOP3NoMods(SDValue In, SDValue &Src) const;
214 bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods,
215 SDValue &Clamp, SDValue &Omod) const;
216 bool SelectVOP3BMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
217 SDValue &Clamp, SDValue &Omod) const;
218 bool SelectVOP3NoMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
219 SDValue &Clamp, SDValue &Omod) const;
220
221 bool SelectVINTERPModsImpl(SDValue In, SDValue &Src, SDValue &SrcMods,
222 bool OpSel) const;
223 bool SelectVINTERPMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
224 bool SelectVINTERPModsHi(SDValue In, SDValue &Src, SDValue &SrcMods) const;
225
226 bool SelectVOP3OMods(SDValue In, SDValue &Src, SDValue &Clamp,
227 SDValue &Omod) const;
228
229 bool SelectVOP3PMods(SDValue In, SDValue &Src, SDValue &SrcMods,
230 bool IsDOT = false) const;
231 bool SelectVOP3PModsDOT(SDValue In, SDValue &Src, SDValue &SrcMods) const;
232
233 bool SelectVOP3PModsNeg(SDValue In, SDValue &Src) const;
234 bool SelectWMMAOpSelVOP3PMods(SDValue In, SDValue &Src) const;
235
236 bool SelectWMMAModsF32NegAbs(SDValue In, SDValue &Src,
237 SDValue &SrcMods) const;
238 bool SelectWMMAModsF16Neg(SDValue In, SDValue &Src, SDValue &SrcMods) const;
239 bool SelectWMMAModsF16NegAbs(SDValue In, SDValue &Src,
240 SDValue &SrcMods) const;
241 bool SelectWMMAVISrc(SDValue In, SDValue &Src) const;
242
243 bool SelectSWMMACIndex8(SDValue In, SDValue &Src, SDValue &IndexKey) const;
244 bool SelectSWMMACIndex16(SDValue In, SDValue &Src, SDValue &IndexKey) const;
245
246 bool SelectVOP3OpSel(SDValue In, SDValue &Src, SDValue &SrcMods) const;
247
248 bool SelectVOP3OpSelMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
249 bool SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src,
250 unsigned &Mods) const;
251 bool SelectVOP3PMadMixModsExt(SDValue In, SDValue &Src,
252 SDValue &SrcMods) const;
253 bool SelectVOP3PMadMixMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
254
255 SDValue getHi16Elt(SDValue In) const;
256
257 SDValue getMaterializedScalarImm32(int64_t Val, const SDLoc &DL) const;
258
259 void SelectADD_SUB_I64(SDNode *N);
260 void SelectAddcSubb(SDNode *N);
261 void SelectUADDO_USUBO(SDNode *N);
262 void SelectDIV_SCALE(SDNode *N);
263 void SelectMAD_64_32(SDNode *N);
264 void SelectMUL_LOHI(SDNode *N);
265 void SelectFMA_W_CHAIN(SDNode *N);
266 void SelectFMUL_W_CHAIN(SDNode *N);
267 SDNode *getBFE32(bool IsSigned, const SDLoc &DL, SDValue Val, uint32_t Offset,
268 uint32_t Width);
269 void SelectS_BFEFromShifts(SDNode *N);
270 void SelectS_BFE(SDNode *N);
271 bool isCBranchSCC(const SDNode *N) const;
272 void SelectBRCOND(SDNode *N);
273 void SelectFMAD_FMA(SDNode *N);
274 void SelectFP_EXTEND(SDNode *N);
275 void SelectDSAppendConsume(SDNode *N, unsigned IntrID);
276 void SelectDSBvhStackIntrinsic(SDNode *N);
277 void SelectDS_GWS(SDNode *N, unsigned IntrID);
278 void SelectInterpP1F16(SDNode *N);
279 void SelectINTRINSIC_W_CHAIN(SDNode *N);
280 void SelectINTRINSIC_WO_CHAIN(SDNode *N);
281 void SelectINTRINSIC_VOID(SDNode *N);
282 void SelectWAVE_ADDRESS(SDNode *N);
283 void SelectSTACKRESTORE(SDNode *N);
284
285protected:
286 // Include the pieces autogenerated from the target description.
287#include "AMDGPUGenDAGISel.inc"
288};
289
290#endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUISELDAGTODAG_H
291

source code of llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h