1 | //===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file implements the targeting of the InstructionSelector class for |
10 | // SPIRV. |
11 | // TODO: This should be generated by TableGen. |
12 | // |
13 | //===----------------------------------------------------------------------===// |
14 | |
15 | #include "MCTargetDesc/SPIRVBaseInfo.h" |
16 | #include "MCTargetDesc/SPIRVMCTargetDesc.h" |
17 | #include "SPIRV.h" |
18 | #include "SPIRVGlobalRegistry.h" |
19 | #include "SPIRVInstrInfo.h" |
20 | #include "SPIRVRegisterBankInfo.h" |
21 | #include "SPIRVRegisterInfo.h" |
22 | #include "SPIRVTargetMachine.h" |
23 | #include "SPIRVUtils.h" |
24 | #include "llvm/ADT/APFloat.h" |
25 | #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h" |
26 | #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" |
27 | #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" |
28 | #include "llvm/CodeGen/MachineInstrBuilder.h" |
29 | #include "llvm/CodeGen/MachineModuleInfoImpls.h" |
30 | #include "llvm/CodeGen/MachineRegisterInfo.h" |
31 | #include "llvm/CodeGen/TargetOpcodes.h" |
32 | #include "llvm/IR/IntrinsicsSPIRV.h" |
33 | #include "llvm/Support/Debug.h" |
34 | |
35 | namespace llvm { |
36 | |
37 | class SPIRVMachineModuleInfo : public MachineModuleInfoImpl { |
38 | public: |
39 | SyncScope::ID Work_ItemSSID; |
40 | SyncScope::ID WorkGroupSSID; |
41 | SyncScope::ID DeviceSSID; |
42 | SyncScope::ID AllSVMDevicesSSID; |
43 | SyncScope::ID SubGroupSSID; |
44 | |
45 | SPIRVMachineModuleInfo(const MachineModuleInfo &MMI) { |
46 | LLVMContext &CTX = MMI.getModule()->getContext(); |
47 | Work_ItemSSID = CTX.getOrInsertSyncScopeID(SSN: "work_item" ); |
48 | WorkGroupSSID = CTX.getOrInsertSyncScopeID(SSN: "workgroup" ); |
49 | DeviceSSID = CTX.getOrInsertSyncScopeID(SSN: "device" ); |
50 | AllSVMDevicesSSID = CTX.getOrInsertSyncScopeID(SSN: "all_svm_devices" ); |
51 | SubGroupSSID = CTX.getOrInsertSyncScopeID(SSN: "sub_group" ); |
52 | } |
53 | }; |
54 | |
55 | } // end namespace llvm |
56 | |
57 | #define DEBUG_TYPE "spirv-isel" |
58 | |
59 | using namespace llvm; |
60 | namespace CL = SPIRV::OpenCLExtInst; |
61 | namespace GL = SPIRV::GLSLExtInst; |
62 | |
63 | using ExtInstList = |
64 | std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>; |
65 | |
66 | namespace { |
67 | |
68 | #define GET_GLOBALISEL_PREDICATE_BITSET |
69 | #include "SPIRVGenGlobalISel.inc" |
70 | #undef GET_GLOBALISEL_PREDICATE_BITSET |
71 | |
72 | class SPIRVInstructionSelector : public InstructionSelector { |
73 | const SPIRVSubtarget &STI; |
74 | const SPIRVInstrInfo &TII; |
75 | const SPIRVRegisterInfo &TRI; |
76 | const RegisterBankInfo &RBI; |
77 | SPIRVGlobalRegistry &GR; |
78 | MachineRegisterInfo *MRI; |
79 | SPIRVMachineModuleInfo *MMI = nullptr; |
80 | |
81 | /// We need to keep track of the number we give to anonymous global values to |
82 | /// generate the same name every time when this is needed. |
83 | mutable DenseMap<const GlobalValue *, unsigned> UnnamedGlobalIDs; |
84 | |
85 | public: |
86 | SPIRVInstructionSelector(const SPIRVTargetMachine &TM, |
87 | const SPIRVSubtarget &ST, |
88 | const RegisterBankInfo &RBI); |
89 | void setupMF(MachineFunction &MF, GISelKnownBits *KB, |
90 | CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI, |
91 | BlockFrequencyInfo *BFI) override; |
92 | // Common selection code. Instruction-specific selection occurs in spvSelect. |
93 | bool select(MachineInstr &I) override; |
94 | static const char *getName() { return DEBUG_TYPE; } |
95 | |
96 | #define GET_GLOBALISEL_PREDICATES_DECL |
97 | #include "SPIRVGenGlobalISel.inc" |
98 | #undef GET_GLOBALISEL_PREDICATES_DECL |
99 | |
100 | #define GET_GLOBALISEL_TEMPORARIES_DECL |
101 | #include "SPIRVGenGlobalISel.inc" |
102 | #undef GET_GLOBALISEL_TEMPORARIES_DECL |
103 | |
104 | private: |
105 | // tblgen-erated 'select' implementation, used as the initial selector for |
106 | // the patterns that don't require complex C++. |
107 | bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; |
108 | |
109 | // All instruction-specific selection that didn't happen in "select()". |
110 | // Is basically a large Switch/Case delegating to all other select method. |
111 | bool spvSelect(Register ResVReg, const SPIRVType *ResType, |
112 | MachineInstr &I) const; |
113 | |
114 | bool selectGlobalValue(Register ResVReg, MachineInstr &I, |
115 | const MachineInstr *Init = nullptr) const; |
116 | |
117 | bool selectUnOpWithSrc(Register ResVReg, const SPIRVType *ResType, |
118 | MachineInstr &I, Register SrcReg, |
119 | unsigned Opcode) const; |
120 | bool selectUnOp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, |
121 | unsigned Opcode) const; |
122 | |
123 | bool selectBitcast(Register ResVReg, const SPIRVType *ResType, |
124 | MachineInstr &I) const; |
125 | |
126 | bool selectLoad(Register ResVReg, const SPIRVType *ResType, |
127 | MachineInstr &I) const; |
128 | bool selectStore(MachineInstr &I) const; |
129 | |
130 | bool selectStackSave(Register ResVReg, const SPIRVType *ResType, |
131 | MachineInstr &I) const; |
132 | bool selectStackRestore(MachineInstr &I) const; |
133 | |
134 | bool selectMemOperation(Register ResVReg, MachineInstr &I) const; |
135 | |
136 | bool selectAtomicRMW(Register ResVReg, const SPIRVType *ResType, |
137 | MachineInstr &I, unsigned NewOpcode, |
138 | unsigned NegateOpcode = 0) const; |
139 | |
140 | bool selectAtomicCmpXchg(Register ResVReg, const SPIRVType *ResType, |
141 | MachineInstr &I) const; |
142 | |
143 | bool selectFence(MachineInstr &I) const; |
144 | |
145 | bool selectAddrSpaceCast(Register ResVReg, const SPIRVType *ResType, |
146 | MachineInstr &I) const; |
147 | |
148 | bool selectAnyOrAll(Register ResVReg, const SPIRVType *ResType, |
149 | MachineInstr &I, unsigned OpType) const; |
150 | |
151 | bool selectAll(Register ResVReg, const SPIRVType *ResType, |
152 | MachineInstr &I) const; |
153 | |
154 | bool selectAny(Register ResVReg, const SPIRVType *ResType, |
155 | MachineInstr &I) const; |
156 | |
157 | bool selectBitreverse(Register ResVReg, const SPIRVType *ResType, |
158 | MachineInstr &I) const; |
159 | |
160 | bool selectConstVector(Register ResVReg, const SPIRVType *ResType, |
161 | MachineInstr &I) const; |
162 | bool selectSplatVector(Register ResVReg, const SPIRVType *ResType, |
163 | MachineInstr &I) const; |
164 | |
165 | bool selectCmp(Register ResVReg, const SPIRVType *ResType, |
166 | unsigned comparisonOpcode, MachineInstr &I) const; |
167 | |
168 | bool selectICmp(Register ResVReg, const SPIRVType *ResType, |
169 | MachineInstr &I) const; |
170 | bool selectFCmp(Register ResVReg, const SPIRVType *ResType, |
171 | MachineInstr &I) const; |
172 | |
173 | bool selectFmix(Register ResVReg, const SPIRVType *ResType, |
174 | MachineInstr &I) const; |
175 | |
176 | void renderImm32(MachineInstrBuilder &MIB, const MachineInstr &I, |
177 | int OpIdx) const; |
178 | void renderFImm32(MachineInstrBuilder &MIB, const MachineInstr &I, |
179 | int OpIdx) const; |
180 | |
181 | bool selectConst(Register ResVReg, const SPIRVType *ResType, const APInt &Imm, |
182 | MachineInstr &I) const; |
183 | |
184 | bool selectSelect(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, |
185 | bool IsSigned) const; |
186 | bool selectIToF(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, |
187 | bool IsSigned, unsigned Opcode) const; |
188 | bool selectExt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, |
189 | bool IsSigned) const; |
190 | |
191 | bool selectTrunc(Register ResVReg, const SPIRVType *ResType, |
192 | MachineInstr &I) const; |
193 | |
194 | bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I, |
195 | const SPIRVType *intTy, const SPIRVType *boolTy) const; |
196 | |
197 | bool selectOpUndef(Register ResVReg, const SPIRVType *ResType, |
198 | MachineInstr &I) const; |
199 | bool selectFreeze(Register ResVReg, const SPIRVType *ResType, |
200 | MachineInstr &I) const; |
201 | bool selectIntrinsic(Register ResVReg, const SPIRVType *ResType, |
202 | MachineInstr &I) const; |
203 | bool selectExtractVal(Register ResVReg, const SPIRVType *ResType, |
204 | MachineInstr &I) const; |
205 | bool selectInsertVal(Register ResVReg, const SPIRVType *ResType, |
206 | MachineInstr &I) const; |
207 | bool selectExtractElt(Register ResVReg, const SPIRVType *ResType, |
208 | MachineInstr &I) const; |
209 | bool selectInsertElt(Register ResVReg, const SPIRVType *ResType, |
210 | MachineInstr &I) const; |
211 | bool selectGEP(Register ResVReg, const SPIRVType *ResType, |
212 | MachineInstr &I) const; |
213 | |
214 | bool selectFrameIndex(Register ResVReg, const SPIRVType *ResType, |
215 | MachineInstr &I) const; |
216 | bool selectAllocaArray(Register ResVReg, const SPIRVType *ResType, |
217 | MachineInstr &I) const; |
218 | |
219 | bool selectBranch(MachineInstr &I) const; |
220 | bool selectBranchCond(MachineInstr &I) const; |
221 | |
222 | bool selectPhi(Register ResVReg, const SPIRVType *ResType, |
223 | MachineInstr &I) const; |
224 | |
225 | bool selectExtInst(Register ResVReg, const SPIRVType *ResType, |
226 | MachineInstr &I, CL::OpenCLExtInst CLInst) const; |
227 | bool selectExtInst(Register ResVReg, const SPIRVType *ResType, |
228 | MachineInstr &I, CL::OpenCLExtInst CLInst, |
229 | GL::GLSLExtInst GLInst) const; |
230 | bool selectExtInst(Register ResVReg, const SPIRVType *ResType, |
231 | MachineInstr &I, const ExtInstList &ExtInsts) const; |
232 | |
233 | bool selectLog10(Register ResVReg, const SPIRVType *ResType, |
234 | MachineInstr &I) const; |
235 | |
236 | bool selectSpvThreadId(Register ResVReg, const SPIRVType *ResType, |
237 | MachineInstr &I) const; |
238 | |
239 | bool selectUnmergeValues(MachineInstr &I) const; |
240 | |
241 | Register buildI32Constant(uint32_t Val, MachineInstr &I, |
242 | const SPIRVType *ResType = nullptr) const; |
243 | |
244 | Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const; |
245 | Register buildZerosValF(const SPIRVType *ResType, MachineInstr &I) const; |
246 | Register buildOnesVal(bool AllOnes, const SPIRVType *ResType, |
247 | MachineInstr &I) const; |
248 | |
249 | bool wrapIntoSpecConstantOp(MachineInstr &I, |
250 | SmallVector<Register> &CompositeArgs) const; |
251 | }; |
252 | |
253 | } // end anonymous namespace |
254 | |
255 | #define GET_GLOBALISEL_IMPL |
256 | #include "SPIRVGenGlobalISel.inc" |
257 | #undef GET_GLOBALISEL_IMPL |
258 | |
259 | SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM, |
260 | const SPIRVSubtarget &ST, |
261 | const RegisterBankInfo &RBI) |
262 | : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()), |
263 | TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()), |
264 | #define GET_GLOBALISEL_PREDICATES_INIT |
265 | #include "SPIRVGenGlobalISel.inc" |
266 | #undef GET_GLOBALISEL_PREDICATES_INIT |
267 | #define GET_GLOBALISEL_TEMPORARIES_INIT |
268 | #include "SPIRVGenGlobalISel.inc" |
269 | #undef GET_GLOBALISEL_TEMPORARIES_INIT |
270 | { |
271 | } |
272 | |
273 | void SPIRVInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits *KB, |
274 | CodeGenCoverage *CoverageInfo, |
275 | ProfileSummaryInfo *PSI, |
276 | BlockFrequencyInfo *BFI) { |
277 | MMI = &MF.getMMI().getObjFileInfo<SPIRVMachineModuleInfo>(); |
278 | MRI = &MF.getRegInfo(); |
279 | GR.setCurrentFunc(MF); |
280 | InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI); |
281 | } |
282 | |
283 | static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI); |
284 | |
285 | // Defined in SPIRVLegalizerInfo.cpp. |
286 | extern bool isTypeFoldingSupported(unsigned Opcode); |
287 | |
288 | bool SPIRVInstructionSelector::select(MachineInstr &I) { |
289 | assert(I.getParent() && "Instruction should be in a basic block!" ); |
290 | assert(I.getParent()->getParent() && "Instruction should be in a function!" ); |
291 | |
292 | Register Opcode = I.getOpcode(); |
293 | // If it's not a GMIR instruction, we've selected it already. |
294 | if (!isPreISelGenericOpcode(Opcode)) { |
295 | if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more. |
296 | Register DstReg = I.getOperand(i: 0).getReg(); |
297 | Register SrcReg = I.getOperand(i: 1).getReg(); |
298 | auto *Def = MRI->getVRegDef(Reg: SrcReg); |
299 | if (isTypeFoldingSupported(Opcode: Def->getOpcode())) { |
300 | if (MRI->getType(Reg: DstReg).isPointer()) |
301 | MRI->setType(VReg: DstReg, Ty: LLT::scalar(SizeInBits: 32)); |
302 | bool Res = selectImpl(I, CoverageInfo&: *CoverageInfo); |
303 | assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT); |
304 | if (Res) |
305 | return Res; |
306 | } |
307 | MRI->replaceRegWith(FromReg: SrcReg, ToReg: DstReg); |
308 | I.removeFromParent(); |
309 | return true; |
310 | } else if (I.getNumDefs() == 1) { |
311 | // Make all vregs 32 bits (for SPIR-V IDs). |
312 | MRI->setType(VReg: I.getOperand(i: 0).getReg(), Ty: LLT::scalar(SizeInBits: 32)); |
313 | } |
314 | return constrainSelectedInstRegOperands(I, TII, TRI, RBI); |
315 | } |
316 | |
317 | if (I.getNumOperands() != I.getNumExplicitOperands()) { |
318 | LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n" ); |
319 | return false; |
320 | } |
321 | |
322 | // Common code for getting return reg+type, and removing selected instr |
323 | // from parent occurs here. Instr-specific selection happens in spvSelect(). |
324 | bool HasDefs = I.getNumDefs() > 0; |
325 | Register ResVReg = HasDefs ? I.getOperand(i: 0).getReg() : Register(0); |
326 | SPIRVType *ResType = HasDefs ? GR.getSPIRVTypeForVReg(VReg: ResVReg) : nullptr; |
327 | assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE); |
328 | if (spvSelect(ResVReg, ResType, I)) { |
329 | if (HasDefs) // Make all vregs 32 bits (for SPIR-V IDs). |
330 | for (unsigned i = 0; i < I.getNumDefs(); ++i) |
331 | MRI->setType(VReg: I.getOperand(i).getReg(), Ty: LLT::scalar(SizeInBits: 32)); |
332 | I.removeFromParent(); |
333 | return true; |
334 | } |
335 | return false; |
336 | } |
337 | |
338 | bool SPIRVInstructionSelector::spvSelect(Register ResVReg, |
339 | const SPIRVType *ResType, |
340 | MachineInstr &I) const { |
341 | const unsigned Opcode = I.getOpcode(); |
342 | if (isTypeFoldingSupported(Opcode) && Opcode != TargetOpcode::G_CONSTANT) |
343 | return selectImpl(I, CoverageInfo&: *CoverageInfo); |
344 | switch (Opcode) { |
345 | case TargetOpcode::G_CONSTANT: |
346 | return selectConst(ResVReg, ResType, Imm: I.getOperand(i: 1).getCImm()->getValue(), |
347 | I); |
348 | case TargetOpcode::G_GLOBAL_VALUE: |
349 | return selectGlobalValue(ResVReg, I); |
350 | case TargetOpcode::G_IMPLICIT_DEF: |
351 | return selectOpUndef(ResVReg, ResType, I); |
352 | case TargetOpcode::G_FREEZE: |
353 | return selectFreeze(ResVReg, ResType, I); |
354 | |
355 | case TargetOpcode::G_INTRINSIC: |
356 | case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: |
357 | case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: |
358 | return selectIntrinsic(ResVReg, ResType, I); |
359 | case TargetOpcode::G_BITREVERSE: |
360 | return selectBitreverse(ResVReg, ResType, I); |
361 | |
362 | case TargetOpcode::G_BUILD_VECTOR: |
363 | return selectConstVector(ResVReg, ResType, I); |
364 | case TargetOpcode::G_SPLAT_VECTOR: |
365 | return selectSplatVector(ResVReg, ResType, I); |
366 | |
367 | case TargetOpcode::G_SHUFFLE_VECTOR: { |
368 | MachineBasicBlock &BB = *I.getParent(); |
369 | auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle)) |
370 | .addDef(ResVReg) |
371 | .addUse(GR.getSPIRVTypeID(ResType)) |
372 | .addUse(I.getOperand(1).getReg()) |
373 | .addUse(I.getOperand(2).getReg()); |
374 | for (auto V : I.getOperand(i: 3).getShuffleMask()) |
375 | MIB.addImm(V); |
376 | return MIB.constrainAllUses(TII, TRI, RBI); |
377 | } |
378 | case TargetOpcode::G_MEMMOVE: |
379 | case TargetOpcode::G_MEMCPY: |
380 | case TargetOpcode::G_MEMSET: |
381 | return selectMemOperation(ResVReg, I); |
382 | |
383 | case TargetOpcode::G_ICMP: |
384 | return selectICmp(ResVReg, ResType, I); |
385 | case TargetOpcode::G_FCMP: |
386 | return selectFCmp(ResVReg, ResType, I); |
387 | |
388 | case TargetOpcode::G_FRAME_INDEX: |
389 | return selectFrameIndex(ResVReg, ResType, I); |
390 | |
391 | case TargetOpcode::G_LOAD: |
392 | return selectLoad(ResVReg, ResType, I); |
393 | case TargetOpcode::G_STORE: |
394 | return selectStore(I); |
395 | |
396 | case TargetOpcode::G_BR: |
397 | return selectBranch(I); |
398 | case TargetOpcode::G_BRCOND: |
399 | return selectBranchCond(I); |
400 | |
401 | case TargetOpcode::G_PHI: |
402 | return selectPhi(ResVReg, ResType, I); |
403 | |
404 | case TargetOpcode::G_FPTOSI: |
405 | return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS); |
406 | case TargetOpcode::G_FPTOUI: |
407 | return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU); |
408 | |
409 | case TargetOpcode::G_SITOFP: |
410 | return selectIToF(ResVReg, ResType, I, true, SPIRV::OpConvertSToF); |
411 | case TargetOpcode::G_UITOFP: |
412 | return selectIToF(ResVReg, ResType, I, false, SPIRV::OpConvertUToF); |
413 | |
414 | case TargetOpcode::G_CTPOP: |
415 | return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitCount); |
416 | case TargetOpcode::G_SMIN: |
417 | return selectExtInst(ResVReg, ResType, I, CL::s_min, GL::SMin); |
418 | case TargetOpcode::G_UMIN: |
419 | return selectExtInst(ResVReg, ResType, I, CL::u_min, GL::UMin); |
420 | |
421 | case TargetOpcode::G_SMAX: |
422 | return selectExtInst(ResVReg, ResType, I, CL::s_max, GL::SMax); |
423 | case TargetOpcode::G_UMAX: |
424 | return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax); |
425 | |
426 | case TargetOpcode::G_FMA: |
427 | return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma); |
428 | |
429 | case TargetOpcode::G_FPOW: |
430 | return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow); |
431 | case TargetOpcode::G_FPOWI: |
432 | return selectExtInst(ResVReg, ResType, I, CL::pown); |
433 | |
434 | case TargetOpcode::G_FEXP: |
435 | return selectExtInst(ResVReg, ResType, I, CL::exp, GL::Exp); |
436 | case TargetOpcode::G_FEXP2: |
437 | return selectExtInst(ResVReg, ResType, I, CL::exp2, GL::Exp2); |
438 | |
439 | case TargetOpcode::G_FLOG: |
440 | return selectExtInst(ResVReg, ResType, I, CL::log, GL::Log); |
441 | case TargetOpcode::G_FLOG2: |
442 | return selectExtInst(ResVReg, ResType, I, CL::log2, GL::Log2); |
443 | case TargetOpcode::G_FLOG10: |
444 | return selectLog10(ResVReg, ResType, I); |
445 | |
446 | case TargetOpcode::G_FABS: |
447 | return selectExtInst(ResVReg, ResType, I, CL::fabs, GL::FAbs); |
448 | case TargetOpcode::G_ABS: |
449 | return selectExtInst(ResVReg, ResType, I, CL::s_abs, GL::SAbs); |
450 | |
451 | case TargetOpcode::G_FMINNUM: |
452 | case TargetOpcode::G_FMINIMUM: |
453 | return selectExtInst(ResVReg, ResType, I, CL::fmin, GL::NMin); |
454 | case TargetOpcode::G_FMAXNUM: |
455 | case TargetOpcode::G_FMAXIMUM: |
456 | return selectExtInst(ResVReg, ResType, I, CL::fmax, GL::NMax); |
457 | |
458 | case TargetOpcode::G_FCOPYSIGN: |
459 | return selectExtInst(ResVReg, ResType, I, CL::copysign); |
460 | |
461 | case TargetOpcode::G_FCEIL: |
462 | return selectExtInst(ResVReg, ResType, I, CL::ceil, GL::Ceil); |
463 | case TargetOpcode::G_FFLOOR: |
464 | return selectExtInst(ResVReg, ResType, I, CL::floor, GL::Floor); |
465 | |
466 | case TargetOpcode::G_FCOS: |
467 | return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos); |
468 | case TargetOpcode::G_FSIN: |
469 | return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin); |
470 | |
471 | case TargetOpcode::G_FSQRT: |
472 | return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt); |
473 | |
474 | case TargetOpcode::G_CTTZ: |
475 | case TargetOpcode::G_CTTZ_ZERO_UNDEF: |
476 | return selectExtInst(ResVReg, ResType, I, CL::ctz); |
477 | case TargetOpcode::G_CTLZ: |
478 | case TargetOpcode::G_CTLZ_ZERO_UNDEF: |
479 | return selectExtInst(ResVReg, ResType, I, CL::clz); |
480 | |
481 | case TargetOpcode::G_INTRINSIC_ROUND: |
482 | return selectExtInst(ResVReg, ResType, I, CL::round, GL::Round); |
483 | case TargetOpcode::G_INTRINSIC_ROUNDEVEN: |
484 | return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); |
485 | case TargetOpcode::G_INTRINSIC_TRUNC: |
486 | return selectExtInst(ResVReg, ResType, I, CL::trunc, GL::Trunc); |
487 | case TargetOpcode::G_FRINT: |
488 | case TargetOpcode::G_FNEARBYINT: |
489 | return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); |
490 | |
491 | case TargetOpcode::G_SMULH: |
492 | return selectExtInst(ResVReg, ResType, I, CL::s_mul_hi); |
493 | case TargetOpcode::G_UMULH: |
494 | return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi); |
495 | |
496 | case TargetOpcode::G_SEXT: |
497 | return selectExt(ResVReg, ResType, I, IsSigned: true); |
498 | case TargetOpcode::G_ANYEXT: |
499 | case TargetOpcode::G_ZEXT: |
500 | return selectExt(ResVReg, ResType, I, IsSigned: false); |
501 | case TargetOpcode::G_TRUNC: |
502 | return selectTrunc(ResVReg, ResType, I); |
503 | case TargetOpcode::G_FPTRUNC: |
504 | case TargetOpcode::G_FPEXT: |
505 | return selectUnOp(ResVReg, ResType, I, SPIRV::OpFConvert); |
506 | |
507 | case TargetOpcode::G_PTRTOINT: |
508 | return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertPtrToU); |
509 | case TargetOpcode::G_INTTOPTR: |
510 | return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertUToPtr); |
511 | case TargetOpcode::G_BITCAST: |
512 | return selectBitcast(ResVReg, ResType, I); |
513 | case TargetOpcode::G_ADDRSPACE_CAST: |
514 | return selectAddrSpaceCast(ResVReg, ResType, I); |
515 | case TargetOpcode::G_PTR_ADD: { |
516 | // Currently, we get G_PTR_ADD only as a result of translating |
517 | // global variables, initialized with constant expressions like GV + Const |
518 | // (see test opencl/basic/progvar_prog_scope_init.ll). |
519 | // TODO: extend the handler once we have other cases. |
520 | assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); |
521 | Register GV = I.getOperand(i: 1).getReg(); |
522 | MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(RegNo: GV); |
523 | (void)II; |
524 | assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || |
525 | (*II).getOpcode() == TargetOpcode::COPY || |
526 | (*II).getOpcode() == SPIRV::OpVariable) && |
527 | isImm(I.getOperand(2), MRI)); |
528 | Register Idx = buildZerosVal(ResType: GR.getOrCreateSPIRVIntegerType(BitWidth: 32, I, TII), I); |
529 | MachineBasicBlock &BB = *I.getParent(); |
530 | auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) |
531 | .addDef(ResVReg) |
532 | .addUse(GR.getSPIRVTypeID(ResType)) |
533 | .addImm(static_cast<uint32_t>( |
534 | SPIRV::Opcode::InBoundsPtrAccessChain)) |
535 | .addUse(GV) |
536 | .addUse(Idx) |
537 | .addUse(I.getOperand(2).getReg()); |
538 | return MIB.constrainAllUses(TII, TRI, RBI); |
539 | } |
540 | |
541 | case TargetOpcode::G_ATOMICRMW_OR: |
542 | return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicOr); |
543 | case TargetOpcode::G_ATOMICRMW_ADD: |
544 | return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicIAdd); |
545 | case TargetOpcode::G_ATOMICRMW_AND: |
546 | return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicAnd); |
547 | case TargetOpcode::G_ATOMICRMW_MAX: |
548 | return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMax); |
549 | case TargetOpcode::G_ATOMICRMW_MIN: |
550 | return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMin); |
551 | case TargetOpcode::G_ATOMICRMW_SUB: |
552 | return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicISub); |
553 | case TargetOpcode::G_ATOMICRMW_XOR: |
554 | return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicXor); |
555 | case TargetOpcode::G_ATOMICRMW_UMAX: |
556 | return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMax); |
557 | case TargetOpcode::G_ATOMICRMW_UMIN: |
558 | return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMin); |
559 | case TargetOpcode::G_ATOMICRMW_XCHG: |
560 | return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicExchange); |
561 | case TargetOpcode::G_ATOMIC_CMPXCHG: |
562 | return selectAtomicCmpXchg(ResVReg, ResType, I); |
563 | |
564 | case TargetOpcode::G_ATOMICRMW_FADD: |
565 | return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT); |
566 | case TargetOpcode::G_ATOMICRMW_FSUB: |
567 | // Translate G_ATOMICRMW_FSUB to OpAtomicFAddEXT with negative value operand |
568 | return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT, |
569 | SPIRV::OpFNegate); |
570 | case TargetOpcode::G_ATOMICRMW_FMIN: |
571 | return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMinEXT); |
572 | case TargetOpcode::G_ATOMICRMW_FMAX: |
573 | return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMaxEXT); |
574 | |
575 | case TargetOpcode::G_FENCE: |
576 | return selectFence(I); |
577 | |
578 | case TargetOpcode::G_STACKSAVE: |
579 | return selectStackSave(ResVReg, ResType, I); |
580 | case TargetOpcode::G_STACKRESTORE: |
581 | return selectStackRestore(I); |
582 | |
583 | case TargetOpcode::G_UNMERGE_VALUES: |
584 | return selectUnmergeValues(I); |
585 | |
586 | default: |
587 | return false; |
588 | } |
589 | } |
590 | |
591 | bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, |
592 | const SPIRVType *ResType, |
593 | MachineInstr &I, |
594 | CL::OpenCLExtInst CLInst) const { |
595 | return selectExtInst(ResVReg, ResType, I, |
596 | {{SPIRV::InstructionSet::OpenCL_std, CLInst}}); |
597 | } |
598 | |
599 | bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, |
600 | const SPIRVType *ResType, |
601 | MachineInstr &I, |
602 | CL::OpenCLExtInst CLInst, |
603 | GL::GLSLExtInst GLInst) const { |
604 | ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst}, |
605 | {SPIRV::InstructionSet::GLSL_std_450, GLInst}}; |
606 | return selectExtInst(ResVReg, ResType, I, ExtInsts); |
607 | } |
608 | |
609 | bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, |
610 | const SPIRVType *ResType, |
611 | MachineInstr &I, |
612 | const ExtInstList &Insts) const { |
613 | |
614 | for (const auto &Ex : Insts) { |
615 | SPIRV::InstructionSet::InstructionSet Set = Ex.first; |
616 | uint32_t Opcode = Ex.second; |
617 | if (STI.canUseExtInstSet(Set)) { |
618 | MachineBasicBlock &BB = *I.getParent(); |
619 | auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) |
620 | .addDef(ResVReg) |
621 | .addUse(GR.getSPIRVTypeID(ResType)) |
622 | .addImm(static_cast<uint32_t>(Set)) |
623 | .addImm(Opcode); |
624 | const unsigned NumOps = I.getNumOperands(); |
625 | for (unsigned i = 1; i < NumOps; ++i) |
626 | MIB.add(I.getOperand(i)); |
627 | return MIB.constrainAllUses(TII, TRI, RBI); |
628 | } |
629 | } |
630 | return false; |
631 | } |
632 | |
633 | bool SPIRVInstructionSelector::selectUnOpWithSrc(Register ResVReg, |
634 | const SPIRVType *ResType, |
635 | MachineInstr &I, |
636 | Register SrcReg, |
637 | unsigned Opcode) const { |
638 | return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) |
639 | .addDef(ResVReg) |
640 | .addUse(GR.getSPIRVTypeID(SpirvType: ResType)) |
641 | .addUse(SrcReg) |
642 | .constrainAllUses(TII, TRI, RBI); |
643 | } |
644 | |
645 | bool SPIRVInstructionSelector::selectUnOp(Register ResVReg, |
646 | const SPIRVType *ResType, |
647 | MachineInstr &I, |
648 | unsigned Opcode) const { |
649 | if (STI.isOpenCLEnv() && I.getOperand(i: 1).isReg()) { |
650 | Register SrcReg = I.getOperand(i: 1).getReg(); |
651 | bool IsGV = false; |
652 | for (MachineRegisterInfo::def_instr_iterator DefIt = |
653 | MRI->def_instr_begin(RegNo: SrcReg); |
654 | DefIt != MRI->def_instr_end(); DefIt = std::next(x: DefIt)) { |
655 | if ((*DefIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE) { |
656 | IsGV = true; |
657 | break; |
658 | } |
659 | } |
660 | if (IsGV) { |
661 | uint32_t SpecOpcode = 0; |
662 | switch (Opcode) { |
663 | case SPIRV::OpConvertPtrToU: |
664 | SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertPtrToU); |
665 | break; |
666 | case SPIRV::OpConvertUToPtr: |
667 | SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertUToPtr); |
668 | break; |
669 | } |
670 | if (SpecOpcode) |
671 | return BuildMI(*I.getParent(), I, I.getDebugLoc(), |
672 | TII.get(SPIRV::OpSpecConstantOp)) |
673 | .addDef(ResVReg) |
674 | .addUse(GR.getSPIRVTypeID(ResType)) |
675 | .addImm(SpecOpcode) |
676 | .addUse(SrcReg) |
677 | .constrainAllUses(TII, TRI, RBI); |
678 | } |
679 | } |
680 | return selectUnOpWithSrc(ResVReg, ResType, I, SrcReg: I.getOperand(i: 1).getReg(), |
681 | Opcode); |
682 | } |
683 | |
684 | bool SPIRVInstructionSelector::selectBitcast(Register ResVReg, |
685 | const SPIRVType *ResType, |
686 | MachineInstr &I) const { |
687 | Register OpReg = I.getOperand(i: 1).getReg(); |
688 | SPIRVType *OpType = OpReg.isValid() ? GR.getSPIRVTypeForVReg(VReg: OpReg) : nullptr; |
689 | if (!GR.isBitcastCompatible(Type1: ResType, Type2: OpType)) |
690 | report_fatal_error(reason: "incompatible result and operand types in a bitcast" ); |
691 | return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast); |
692 | } |
693 | |
694 | static SPIRV::Scope::Scope getScope(SyncScope::ID Ord, |
695 | SPIRVMachineModuleInfo *MMI) { |
696 | if (Ord == SyncScope::SingleThread || Ord == MMI->Work_ItemSSID) |
697 | return SPIRV::Scope::Invocation; |
698 | else if (Ord == SyncScope::System || Ord == MMI->DeviceSSID) |
699 | return SPIRV::Scope::Device; |
700 | else if (Ord == MMI->WorkGroupSSID) |
701 | return SPIRV::Scope::Workgroup; |
702 | else if (Ord == MMI->AllSVMDevicesSSID) |
703 | return SPIRV::Scope::CrossDevice; |
704 | else if (Ord == MMI->SubGroupSSID) |
705 | return SPIRV::Scope::Subgroup; |
706 | else |
707 | // OpenCL approach is: "The functions that do not have memory_scope argument |
708 | // have the same semantics as the corresponding functions with the |
709 | // memory_scope argument set to memory_scope_device." See ref.: // |
710 | // https://registry.khronos.org/OpenCL/specs/3.0-unified/html/OpenCL_C.html#atomic-functions |
711 | // In our case if the scope is unknown, assuming that SPIR-V code is to be |
712 | // consumed in an OpenCL environment, we use the same approach and set the |
713 | // scope to memory_scope_device. |
714 | return SPIRV::Scope::Device; |
715 | } |
716 | |
717 | static void addMemoryOperands(MachineMemOperand *MemOp, |
718 | MachineInstrBuilder &MIB) { |
719 | uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); |
720 | if (MemOp->isVolatile()) |
721 | SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); |
722 | if (MemOp->isNonTemporal()) |
723 | SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); |
724 | if (MemOp->getAlign().value()) |
725 | SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned); |
726 | |
727 | if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) { |
728 | MIB.addImm(Val: SpvMemOp); |
729 | if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned)) |
730 | MIB.addImm(Val: MemOp->getAlign().value()); |
731 | } |
732 | } |
733 | |
734 | static void addMemoryOperands(uint64_t Flags, MachineInstrBuilder &MIB) { |
735 | uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); |
736 | if (Flags & MachineMemOperand::Flags::MOVolatile) |
737 | SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); |
738 | if (Flags & MachineMemOperand::Flags::MONonTemporal) |
739 | SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); |
740 | |
741 | if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) |
742 | MIB.addImm(Val: SpvMemOp); |
743 | } |
744 | |
745 | bool SPIRVInstructionSelector::selectLoad(Register ResVReg, |
746 | const SPIRVType *ResType, |
747 | MachineInstr &I) const { |
748 | unsigned OpOffset = isa<GIntrinsic>(Val: I) ? 1 : 0; |
749 | Register Ptr = I.getOperand(i: 1 + OpOffset).getReg(); |
750 | auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) |
751 | .addDef(ResVReg) |
752 | .addUse(GR.getSPIRVTypeID(ResType)) |
753 | .addUse(Ptr); |
754 | if (!I.getNumMemOperands()) { |
755 | assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || |
756 | I.getOpcode() == |
757 | TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); |
758 | addMemoryOperands(I.getOperand(i: 2 + OpOffset).getImm(), MIB); |
759 | } else { |
760 | addMemoryOperands(*I.memoperands_begin(), MIB); |
761 | } |
762 | return MIB.constrainAllUses(TII, TRI, RBI); |
763 | } |
764 | |
765 | bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const { |
766 | unsigned OpOffset = isa<GIntrinsic>(Val: I) ? 1 : 0; |
767 | Register StoreVal = I.getOperand(i: 0 + OpOffset).getReg(); |
768 | Register Ptr = I.getOperand(i: 1 + OpOffset).getReg(); |
769 | MachineBasicBlock &BB = *I.getParent(); |
770 | auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpStore)) |
771 | .addUse(Ptr) |
772 | .addUse(StoreVal); |
773 | if (!I.getNumMemOperands()) { |
774 | assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || |
775 | I.getOpcode() == |
776 | TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); |
777 | addMemoryOperands(I.getOperand(i: 2 + OpOffset).getImm(), MIB); |
778 | } else { |
779 | addMemoryOperands(*I.memoperands_begin(), MIB); |
780 | } |
781 | return MIB.constrainAllUses(TII, TRI, RBI); |
782 | } |
783 | |
784 | bool SPIRVInstructionSelector::selectStackSave(Register ResVReg, |
785 | const SPIRVType *ResType, |
786 | MachineInstr &I) const { |
787 | if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) |
788 | report_fatal_error( |
789 | reason: "llvm.stacksave intrinsic: this instruction requires the following " |
790 | "SPIR-V extension: SPV_INTEL_variable_length_array" , |
791 | gen_crash_diag: false); |
792 | MachineBasicBlock &BB = *I.getParent(); |
793 | return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSaveMemoryINTEL)) |
794 | .addDef(ResVReg) |
795 | .addUse(GR.getSPIRVTypeID(ResType)) |
796 | .constrainAllUses(TII, TRI, RBI); |
797 | } |
798 | |
799 | bool SPIRVInstructionSelector::selectStackRestore(MachineInstr &I) const { |
800 | if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) |
801 | report_fatal_error( |
802 | reason: "llvm.stackrestore intrinsic: this instruction requires the following " |
803 | "SPIR-V extension: SPV_INTEL_variable_length_array" , |
804 | gen_crash_diag: false); |
805 | if (!I.getOperand(i: 0).isReg()) |
806 | return false; |
807 | MachineBasicBlock &BB = *I.getParent(); |
808 | return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpRestoreMemoryINTEL)) |
809 | .addUse(I.getOperand(0).getReg()) |
810 | .constrainAllUses(TII, TRI, RBI); |
811 | } |
812 | |
813 | bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg, |
814 | MachineInstr &I) const { |
815 | MachineBasicBlock &BB = *I.getParent(); |
816 | Register SrcReg = I.getOperand(i: 1).getReg(); |
817 | if (I.getOpcode() == TargetOpcode::G_MEMSET) { |
818 | assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); |
819 | unsigned Val = getIConstVal(ConstReg: I.getOperand(i: 1).getReg(), MRI); |
820 | unsigned Num = getIConstVal(ConstReg: I.getOperand(i: 2).getReg(), MRI); |
821 | SPIRVType *ValTy = GR.getOrCreateSPIRVIntegerType(BitWidth: 8, I, TII); |
822 | SPIRVType *ArrTy = GR.getOrCreateSPIRVArrayType(BaseType: ValTy, NumElements: Num, I, TII); |
823 | Register Const = GR.getOrCreateConsIntArray(Val, I, SpvType: ArrTy, TII); |
824 | SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType( |
825 | ArrTy, I, TII, SPIRV::StorageClass::UniformConstant); |
826 | // TODO: check if we have such GV, add init, use buildGlobalVariable. |
827 | Function &CurFunction = GR.CurMF->getFunction(); |
828 | Type *LLVMArrTy = |
829 | ArrayType::get(ElementType: IntegerType::get(C&: CurFunction.getContext(), NumBits: 8), NumElements: Num); |
830 | // Module takes ownership of the global var. |
831 | GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy, |
832 | true, GlobalValue::InternalLinkage, |
833 | Constant::getNullValue(Ty: LLVMArrTy)); |
834 | Register VarReg = MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 32)); |
835 | GR.add(GV, MF: GR.CurMF, R: VarReg); |
836 | |
837 | buildOpDecorate(VarReg, I, TII, SPIRV::Decoration::Constant, {}); |
838 | BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable)) |
839 | .addDef(VarReg) |
840 | .addUse(GR.getSPIRVTypeID(VarTy)) |
841 | .addImm(SPIRV::StorageClass::UniformConstant) |
842 | .addUse(Const) |
843 | .constrainAllUses(TII, TRI, RBI); |
844 | SPIRVType *SourceTy = GR.getOrCreateSPIRVPointerType( |
845 | ValTy, I, TII, SPIRV::StorageClass::UniformConstant); |
846 | SrcReg = MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 32)); |
847 | selectUnOpWithSrc(SrcReg, SourceTy, I, VarReg, SPIRV::OpBitcast); |
848 | } |
849 | auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemorySized)) |
850 | .addUse(I.getOperand(0).getReg()) |
851 | .addUse(SrcReg) |
852 | .addUse(I.getOperand(2).getReg()); |
853 | if (I.getNumMemOperands()) |
854 | addMemoryOperands(*I.memoperands_begin(), MIB); |
855 | bool Result = MIB.constrainAllUses(TII, TRI, RBI); |
856 | if (ResVReg.isValid() && ResVReg != MIB->getOperand(0).getReg()) |
857 | BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), ResVReg) |
858 | .addUse(MIB->getOperand(0).getReg()); |
859 | return Result; |
860 | } |
861 | |
862 | bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg, |
863 | const SPIRVType *ResType, |
864 | MachineInstr &I, |
865 | unsigned NewOpcode, |
866 | unsigned NegateOpcode) const { |
867 | assert(I.hasOneMemOperand()); |
868 | const MachineMemOperand *MemOp = *I.memoperands_begin(); |
869 | uint32_t Scope = |
870 | static_cast<uint32_t>(getScope(MemOp->getSyncScopeID(), MMI)); |
871 | Register ScopeReg = buildI32Constant(Val: Scope, I); |
872 | |
873 | Register Ptr = I.getOperand(i: 1).getReg(); |
874 | // TODO: Changed as it's implemented in the translator. See test/atomicrmw.ll |
875 | // auto ScSem = |
876 | // getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr)); |
877 | AtomicOrdering AO = MemOp->getSuccessOrdering(); |
878 | uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); |
879 | Register MemSemReg = buildI32Constant(Val: MemSem /*| ScSem*/, I); |
880 | |
881 | bool Result = false; |
882 | Register ValueReg = I.getOperand(i: 2).getReg(); |
883 | if (NegateOpcode != 0) { |
884 | // Translation with negative value operand is requested |
885 | Register TmpReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); |
886 | Result |= selectUnOpWithSrc(ResVReg: TmpReg, ResType, I, SrcReg: ValueReg, Opcode: NegateOpcode); |
887 | ValueReg = TmpReg; |
888 | } |
889 | |
890 | Result |= BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(NewOpcode)) |
891 | .addDef(ResVReg) |
892 | .addUse(GR.getSPIRVTypeID(SpirvType: ResType)) |
893 | .addUse(Ptr) |
894 | .addUse(ScopeReg) |
895 | .addUse(MemSemReg) |
896 | .addUse(ValueReg) |
897 | .constrainAllUses(TII, TRI, RBI); |
898 | return Result; |
899 | } |
900 | |
901 | bool SPIRVInstructionSelector::selectUnmergeValues(MachineInstr &I) const { |
902 | unsigned ArgI = I.getNumOperands() - 1; |
903 | Register SrcReg = |
904 | I.getOperand(i: ArgI).isReg() ? I.getOperand(i: ArgI).getReg() : Register(0); |
905 | SPIRVType *DefType = |
906 | SrcReg.isValid() ? GR.getSPIRVTypeForVReg(VReg: SrcReg) : nullptr; |
907 | if (!DefType || DefType->getOpcode() != SPIRV::OpTypeVector) |
908 | report_fatal_error( |
909 | reason: "cannot select G_UNMERGE_VALUES with a non-vector argument" ); |
910 | |
911 | SPIRVType *ScalarType = |
912 | GR.getSPIRVTypeForVReg(VReg: DefType->getOperand(i: 1).getReg()); |
913 | MachineBasicBlock &BB = *I.getParent(); |
914 | bool Res = false; |
915 | for (unsigned i = 0; i < I.getNumDefs(); ++i) { |
916 | Register ResVReg = I.getOperand(i).getReg(); |
917 | SPIRVType *ResType = GR.getSPIRVTypeForVReg(VReg: ResVReg); |
918 | if (!ResType) { |
919 | // There was no "assign type" actions, let's fix this now |
920 | ResType = ScalarType; |
921 | MRI->setRegClass(ResVReg, &SPIRV::IDRegClass); |
922 | MRI->setType(VReg: ResVReg, Ty: LLT::scalar(SizeInBits: GR.getScalarOrVectorBitWidth(Type: ResType))); |
923 | GR.assignSPIRVTypeToVReg(Type: ResType, VReg: ResVReg, MF&: *GR.CurMF); |
924 | } |
925 | auto MIB = |
926 | BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) |
927 | .addDef(ResVReg) |
928 | .addUse(GR.getSPIRVTypeID(ResType)) |
929 | .addUse(SrcReg) |
930 | .addImm(static_cast<int64_t>(i)); |
931 | Res |= MIB.constrainAllUses(TII, TRI, RBI); |
932 | } |
933 | return Res; |
934 | } |
935 | |
936 | bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const { |
937 | AtomicOrdering AO = AtomicOrdering(I.getOperand(i: 0).getImm()); |
938 | uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); |
939 | Register MemSemReg = buildI32Constant(Val: MemSem, I); |
940 | SyncScope::ID Ord = SyncScope::ID(I.getOperand(i: 1).getImm()); |
941 | uint32_t Scope = static_cast<uint32_t>(getScope(Ord, MMI)); |
942 | Register ScopeReg = buildI32Constant(Val: Scope, I); |
943 | MachineBasicBlock &BB = *I.getParent(); |
944 | return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemoryBarrier)) |
945 | .addUse(ScopeReg) |
946 | .addUse(MemSemReg) |
947 | .constrainAllUses(TII, TRI, RBI); |
948 | } |
949 | |
950 | bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg, |
951 | const SPIRVType *ResType, |
952 | MachineInstr &I) const { |
953 | Register ScopeReg; |
954 | Register MemSemEqReg; |
955 | Register MemSemNeqReg; |
956 | Register Ptr = I.getOperand(i: 2).getReg(); |
957 | if (!isa<GIntrinsic>(Val: I)) { |
958 | assert(I.hasOneMemOperand()); |
959 | const MachineMemOperand *MemOp = *I.memoperands_begin(); |
960 | unsigned Scope = |
961 | static_cast<uint32_t>(getScope(MemOp->getSyncScopeID(), MMI)); |
962 | ScopeReg = buildI32Constant(Val: Scope, I); |
963 | |
964 | unsigned ScSem = static_cast<uint32_t>( |
965 | getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr))); |
966 | AtomicOrdering AO = MemOp->getSuccessOrdering(); |
967 | unsigned MemSemEq = static_cast<uint32_t>(getMemSemantics(AO)) | ScSem; |
968 | MemSemEqReg = buildI32Constant(Val: MemSemEq, I); |
969 | AtomicOrdering FO = MemOp->getFailureOrdering(); |
970 | unsigned MemSemNeq = static_cast<uint32_t>(getMemSemantics(FO)) | ScSem; |
971 | MemSemNeqReg = |
972 | MemSemEq == MemSemNeq ? MemSemEqReg : buildI32Constant(Val: MemSemNeq, I); |
973 | } else { |
974 | ScopeReg = I.getOperand(i: 5).getReg(); |
975 | MemSemEqReg = I.getOperand(i: 6).getReg(); |
976 | MemSemNeqReg = I.getOperand(i: 7).getReg(); |
977 | } |
978 | |
979 | Register Cmp = I.getOperand(i: 3).getReg(); |
980 | Register Val = I.getOperand(i: 4).getReg(); |
981 | SPIRVType *SpvValTy = GR.getSPIRVTypeForVReg(VReg: Val); |
982 | Register ACmpRes = MRI->createVirtualRegister(&SPIRV::IDRegClass); |
983 | const DebugLoc &DL = I.getDebugLoc(); |
984 | bool Result = |
985 | BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpAtomicCompareExchange)) |
986 | .addDef(ACmpRes) |
987 | .addUse(GR.getSPIRVTypeID(SpvValTy)) |
988 | .addUse(Ptr) |
989 | .addUse(ScopeReg) |
990 | .addUse(MemSemEqReg) |
991 | .addUse(MemSemNeqReg) |
992 | .addUse(Val) |
993 | .addUse(Cmp) |
994 | .constrainAllUses(TII, TRI, RBI); |
995 | Register CmpSuccReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); |
996 | SPIRVType *BoolTy = GR.getOrCreateSPIRVBoolType(I, TII); |
997 | Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpIEqual)) |
998 | .addDef(CmpSuccReg) |
999 | .addUse(GR.getSPIRVTypeID(BoolTy)) |
1000 | .addUse(ACmpRes) |
1001 | .addUse(Cmp) |
1002 | .constrainAllUses(TII, TRI, RBI); |
1003 | Register TmpReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); |
1004 | Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) |
1005 | .addDef(TmpReg) |
1006 | .addUse(GR.getSPIRVTypeID(ResType)) |
1007 | .addUse(ACmpRes) |
1008 | .addUse(GR.getOrCreateUndef(I, ResType, TII)) |
1009 | .addImm(0) |
1010 | .constrainAllUses(TII, TRI, RBI); |
1011 | Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) |
1012 | .addDef(ResVReg) |
1013 | .addUse(GR.getSPIRVTypeID(ResType)) |
1014 | .addUse(CmpSuccReg) |
1015 | .addUse(TmpReg) |
1016 | .addImm(1) |
1017 | .constrainAllUses(TII, TRI, RBI); |
1018 | return Result; |
1019 | } |
1020 | |
1021 | static bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC) { |
1022 | switch (SC) { |
1023 | case SPIRV::StorageClass::Workgroup: |
1024 | case SPIRV::StorageClass::CrossWorkgroup: |
1025 | case SPIRV::StorageClass::Function: |
1026 | return true; |
1027 | default: |
1028 | return false; |
1029 | } |
1030 | } |
1031 | |
1032 | static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC) { |
1033 | switch (SC) { |
1034 | case SPIRV::StorageClass::DeviceOnlyINTEL: |
1035 | case SPIRV::StorageClass::HostOnlyINTEL: |
1036 | return true; |
1037 | default: |
1038 | return false; |
1039 | } |
1040 | } |
1041 | |
1042 | // In SPIR-V address space casting can only happen to and from the Generic |
1043 | // storage class. We can also only cast Workgroup, CrossWorkgroup, or Function |
1044 | // pointers to and from Generic pointers. As such, we can convert e.g. from |
1045 | // Workgroup to Function by going via a Generic pointer as an intermediary. All |
1046 | // other combinations can only be done by a bitcast, and are probably not safe. |
1047 | bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg, |
1048 | const SPIRVType *ResType, |
1049 | MachineInstr &I) const { |
1050 | // If the AddrSpaceCast user is single and in OpConstantComposite or |
1051 | // OpVariable, we should select OpSpecConstantOp. |
1052 | auto UIs = MRI->use_instructions(Reg: ResVReg); |
1053 | if (!UIs.empty() && ++UIs.begin() == UIs.end() && |
1054 | (UIs.begin()->getOpcode() == SPIRV::OpConstantComposite || |
1055 | UIs.begin()->getOpcode() == SPIRV::OpVariable || |
1056 | isSpvIntrinsic(*UIs.begin(), Intrinsic::spv_init_global))) { |
1057 | Register NewReg = I.getOperand(i: 1).getReg(); |
1058 | MachineBasicBlock &BB = *I.getParent(); |
1059 | SPIRVType *SpvBaseTy = GR.getOrCreateSPIRVIntegerType(BitWidth: 8, I, TII); |
1060 | ResType = GR.getOrCreateSPIRVPointerType(SpvBaseTy, I, TII, |
1061 | SPIRV::StorageClass::Generic); |
1062 | bool Result = |
1063 | BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) |
1064 | .addDef(ResVReg) |
1065 | .addUse(GR.getSPIRVTypeID(ResType)) |
1066 | .addImm(static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric)) |
1067 | .addUse(NewReg) |
1068 | .constrainAllUses(TII, TRI, RBI); |
1069 | return Result; |
1070 | } |
1071 | Register SrcPtr = I.getOperand(i: 1).getReg(); |
1072 | SPIRVType *SrcPtrTy = GR.getSPIRVTypeForVReg(VReg: SrcPtr); |
1073 | SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(SrcPtr); |
1074 | SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(ResVReg); |
1075 | |
1076 | // don't generate a cast between identical storage classes |
1077 | if (SrcSC == DstSC) |
1078 | return true; |
1079 | |
1080 | // Casting from an eligible pointer to Generic. |
1081 | if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC)) |
1082 | return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); |
1083 | // Casting from Generic to an eligible pointer. |
1084 | if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(DstSC)) |
1085 | return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); |
1086 | // Casting between 2 eligible pointers using Generic as an intermediary. |
1087 | if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { |
1088 | Register Tmp = MRI->createVirtualRegister(&SPIRV::IDRegClass); |
1089 | SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( |
1090 | SrcPtrTy, I, TII, SPIRV::StorageClass::Generic); |
1091 | MachineBasicBlock &BB = *I.getParent(); |
1092 | const DebugLoc &DL = I.getDebugLoc(); |
1093 | bool Success = BuildMI(BB, I, DL, TII.get(SPIRV::OpPtrCastToGeneric)) |
1094 | .addDef(Tmp) |
1095 | .addUse(GR.getSPIRVTypeID(GenericPtrTy)) |
1096 | .addUse(SrcPtr) |
1097 | .constrainAllUses(TII, TRI, RBI); |
1098 | return Success && BuildMI(BB, I, DL, TII.get(SPIRV::OpGenericCastToPtr)) |
1099 | .addDef(ResVReg) |
1100 | .addUse(GR.getSPIRVTypeID(ResType)) |
1101 | .addUse(Tmp) |
1102 | .constrainAllUses(TII, TRI, RBI); |
1103 | } |
1104 | |
1105 | // Check if instructions from the SPV_INTEL_usm_storage_classes extension may |
1106 | // be applied |
1107 | if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::CrossWorkgroup) |
1108 | return selectUnOp(ResVReg, ResType, I, |
1109 | SPIRV::OpPtrCastToCrossWorkgroupINTEL); |
1110 | if (SrcSC == SPIRV::StorageClass::CrossWorkgroup && isUSMStorageClass(DstSC)) |
1111 | return selectUnOp(ResVReg, ResType, I, |
1112 | SPIRV::OpCrossWorkgroupCastToPtrINTEL); |
1113 | |
1114 | // TODO Should this case just be disallowed completely? |
1115 | // We're casting 2 other arbitrary address spaces, so have to bitcast. |
1116 | return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast); |
1117 | } |
1118 | |
1119 | static unsigned getFCmpOpcode(unsigned PredNum) { |
1120 | auto Pred = static_cast<CmpInst::Predicate>(PredNum); |
1121 | switch (Pred) { |
1122 | case CmpInst::FCMP_OEQ: |
1123 | return SPIRV::OpFOrdEqual; |
1124 | case CmpInst::FCMP_OGE: |
1125 | return SPIRV::OpFOrdGreaterThanEqual; |
1126 | case CmpInst::FCMP_OGT: |
1127 | return SPIRV::OpFOrdGreaterThan; |
1128 | case CmpInst::FCMP_OLE: |
1129 | return SPIRV::OpFOrdLessThanEqual; |
1130 | case CmpInst::FCMP_OLT: |
1131 | return SPIRV::OpFOrdLessThan; |
1132 | case CmpInst::FCMP_ONE: |
1133 | return SPIRV::OpFOrdNotEqual; |
1134 | case CmpInst::FCMP_ORD: |
1135 | return SPIRV::OpOrdered; |
1136 | case CmpInst::FCMP_UEQ: |
1137 | return SPIRV::OpFUnordEqual; |
1138 | case CmpInst::FCMP_UGE: |
1139 | return SPIRV::OpFUnordGreaterThanEqual; |
1140 | case CmpInst::FCMP_UGT: |
1141 | return SPIRV::OpFUnordGreaterThan; |
1142 | case CmpInst::FCMP_ULE: |
1143 | return SPIRV::OpFUnordLessThanEqual; |
1144 | case CmpInst::FCMP_ULT: |
1145 | return SPIRV::OpFUnordLessThan; |
1146 | case CmpInst::FCMP_UNE: |
1147 | return SPIRV::OpFUnordNotEqual; |
1148 | case CmpInst::FCMP_UNO: |
1149 | return SPIRV::OpUnordered; |
1150 | default: |
1151 | llvm_unreachable("Unknown predicate type for FCmp" ); |
1152 | } |
1153 | } |
1154 | |
1155 | static unsigned getICmpOpcode(unsigned PredNum) { |
1156 | auto Pred = static_cast<CmpInst::Predicate>(PredNum); |
1157 | switch (Pred) { |
1158 | case CmpInst::ICMP_EQ: |
1159 | return SPIRV::OpIEqual; |
1160 | case CmpInst::ICMP_NE: |
1161 | return SPIRV::OpINotEqual; |
1162 | case CmpInst::ICMP_SGE: |
1163 | return SPIRV::OpSGreaterThanEqual; |
1164 | case CmpInst::ICMP_SGT: |
1165 | return SPIRV::OpSGreaterThan; |
1166 | case CmpInst::ICMP_SLE: |
1167 | return SPIRV::OpSLessThanEqual; |
1168 | case CmpInst::ICMP_SLT: |
1169 | return SPIRV::OpSLessThan; |
1170 | case CmpInst::ICMP_UGE: |
1171 | return SPIRV::OpUGreaterThanEqual; |
1172 | case CmpInst::ICMP_UGT: |
1173 | return SPIRV::OpUGreaterThan; |
1174 | case CmpInst::ICMP_ULE: |
1175 | return SPIRV::OpULessThanEqual; |
1176 | case CmpInst::ICMP_ULT: |
1177 | return SPIRV::OpULessThan; |
1178 | default: |
1179 | llvm_unreachable("Unknown predicate type for ICmp" ); |
1180 | } |
1181 | } |
1182 | |
1183 | static unsigned getPtrCmpOpcode(unsigned Pred) { |
1184 | switch (static_cast<CmpInst::Predicate>(Pred)) { |
1185 | case CmpInst::ICMP_EQ: |
1186 | return SPIRV::OpPtrEqual; |
1187 | case CmpInst::ICMP_NE: |
1188 | return SPIRV::OpPtrNotEqual; |
1189 | default: |
1190 | llvm_unreachable("Unknown predicate type for pointer comparison" ); |
1191 | } |
1192 | } |
1193 | |
1194 | // Return the logical operation, or abort if none exists. |
1195 | static unsigned getBoolCmpOpcode(unsigned PredNum) { |
1196 | auto Pred = static_cast<CmpInst::Predicate>(PredNum); |
1197 | switch (Pred) { |
1198 | case CmpInst::ICMP_EQ: |
1199 | return SPIRV::OpLogicalEqual; |
1200 | case CmpInst::ICMP_NE: |
1201 | return SPIRV::OpLogicalNotEqual; |
1202 | default: |
1203 | llvm_unreachable("Unknown predicate type for Bool comparison" ); |
1204 | } |
1205 | } |
1206 | |
1207 | bool SPIRVInstructionSelector::selectAnyOrAll(Register ResVReg, |
1208 | const SPIRVType *ResType, |
1209 | MachineInstr &I, |
1210 | unsigned OpAnyOrAll) const { |
1211 | assert(I.getNumOperands() == 3); |
1212 | assert(I.getOperand(2).isReg()); |
1213 | MachineBasicBlock &BB = *I.getParent(); |
1214 | Register InputRegister = I.getOperand(i: 2).getReg(); |
1215 | SPIRVType *InputType = GR.getSPIRVTypeForVReg(VReg: InputRegister); |
1216 | |
1217 | if (!InputType) |
1218 | report_fatal_error(reason: "Input Type could not be determined." ); |
1219 | |
1220 | bool IsBoolTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeBool); |
1221 | bool IsVectorTy = InputType->getOpcode() == SPIRV::OpTypeVector; |
1222 | if (IsBoolTy && !IsVectorTy) { |
1223 | assert(ResVReg == I.getOperand(0).getReg()); |
1224 | return BuildMI(*I.getParent(), I, I.getDebugLoc(), |
1225 | TII.get(TargetOpcode::COPY)) |
1226 | .addDef(ResVReg) |
1227 | .addUse(InputRegister) |
1228 | .constrainAllUses(TII, TRI, RBI); |
1229 | } |
1230 | |
1231 | bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); |
1232 | unsigned SpirvNotEqualId = |
1233 | IsFloatTy ? SPIRV::OpFOrdNotEqual : SPIRV::OpINotEqual; |
1234 | SPIRVType *SpvBoolScalarTy = GR.getOrCreateSPIRVBoolType(I, TII); |
1235 | SPIRVType *SpvBoolTy = SpvBoolScalarTy; |
1236 | Register NotEqualReg = ResVReg; |
1237 | |
1238 | if (IsVectorTy) { |
1239 | NotEqualReg = IsBoolTy ? InputRegister |
1240 | : MRI->createVirtualRegister(&SPIRV::IDRegClass); |
1241 | const unsigned NumElts = InputType->getOperand(i: 2).getImm(); |
1242 | SpvBoolTy = GR.getOrCreateSPIRVVectorType(BaseType: SpvBoolTy, NumElements: NumElts, I, TII); |
1243 | } |
1244 | |
1245 | if (!IsBoolTy) { |
1246 | Register ConstZeroReg = |
1247 | IsFloatTy ? buildZerosValF(ResType: InputType, I) : buildZerosVal(ResType: InputType, I); |
1248 | |
1249 | BuildMI(BB, I, I.getDebugLoc(), TII.get(SpirvNotEqualId)) |
1250 | .addDef(NotEqualReg) |
1251 | .addUse(GR.getSPIRVTypeID(SpirvType: SpvBoolTy)) |
1252 | .addUse(InputRegister) |
1253 | .addUse(ConstZeroReg) |
1254 | .constrainAllUses(TII, TRI, RBI); |
1255 | } |
1256 | |
1257 | if (!IsVectorTy) |
1258 | return true; |
1259 | |
1260 | return BuildMI(BB, I, I.getDebugLoc(), TII.get(OpAnyOrAll)) |
1261 | .addDef(ResVReg) |
1262 | .addUse(GR.getSPIRVTypeID(SpirvType: SpvBoolScalarTy)) |
1263 | .addUse(NotEqualReg) |
1264 | .constrainAllUses(TII, TRI, RBI); |
1265 | } |
1266 | |
1267 | bool SPIRVInstructionSelector::selectAll(Register ResVReg, |
1268 | const SPIRVType *ResType, |
1269 | MachineInstr &I) const { |
1270 | return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAll); |
1271 | } |
1272 | |
1273 | bool SPIRVInstructionSelector::selectAny(Register ResVReg, |
1274 | const SPIRVType *ResType, |
1275 | MachineInstr &I) const { |
1276 | return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAny); |
1277 | } |
1278 | |
1279 | bool SPIRVInstructionSelector::selectFmix(Register ResVReg, |
1280 | const SPIRVType *ResType, |
1281 | MachineInstr &I) const { |
1282 | |
1283 | assert(I.getNumOperands() == 5); |
1284 | assert(I.getOperand(2).isReg()); |
1285 | assert(I.getOperand(3).isReg()); |
1286 | assert(I.getOperand(4).isReg()); |
1287 | MachineBasicBlock &BB = *I.getParent(); |
1288 | |
1289 | return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) |
1290 | .addDef(ResVReg) |
1291 | .addUse(GR.getSPIRVTypeID(ResType)) |
1292 | .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) |
1293 | .addImm(GL::FMix) |
1294 | .addUse(I.getOperand(2).getReg()) |
1295 | .addUse(I.getOperand(3).getReg()) |
1296 | .addUse(I.getOperand(4).getReg()) |
1297 | .constrainAllUses(TII, TRI, RBI); |
1298 | } |
1299 | |
1300 | bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg, |
1301 | const SPIRVType *ResType, |
1302 | MachineInstr &I) const { |
1303 | MachineBasicBlock &BB = *I.getParent(); |
1304 | return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitReverse)) |
1305 | .addDef(ResVReg) |
1306 | .addUse(GR.getSPIRVTypeID(ResType)) |
1307 | .addUse(I.getOperand(1).getReg()) |
1308 | .constrainAllUses(TII, TRI, RBI); |
1309 | } |
1310 | |
1311 | bool SPIRVInstructionSelector::selectFreeze(Register ResVReg, |
1312 | const SPIRVType *ResType, |
1313 | MachineInstr &I) const { |
1314 | // There is no way to implement `freeze` correctly without support on SPIR-V |
1315 | // standard side, but we may at least address a simple (static) case when |
1316 | // undef/poison value presence is obvious. The main benefit of even |
1317 | // incomplete `freeze` support is preventing of translation from crashing due |
1318 | // to lack of support on legalization and instruction selection steps. |
1319 | if (!I.getOperand(i: 0).isReg() || !I.getOperand(i: 1).isReg()) |
1320 | return false; |
1321 | Register OpReg = I.getOperand(i: 1).getReg(); |
1322 | if (MachineInstr *Def = MRI->getVRegDef(Reg: OpReg)) { |
1323 | Register Reg; |
1324 | switch (Def->getOpcode()) { |
1325 | case SPIRV::ASSIGN_TYPE: |
1326 | if (MachineInstr *AssignToDef = |
1327 | MRI->getVRegDef(Reg: Def->getOperand(i: 1).getReg())) { |
1328 | if (AssignToDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF) |
1329 | Reg = Def->getOperand(i: 2).getReg(); |
1330 | } |
1331 | break; |
1332 | case SPIRV::OpUndef: |
1333 | Reg = Def->getOperand(i: 1).getReg(); |
1334 | break; |
1335 | } |
1336 | unsigned DestOpCode; |
1337 | if (Reg.isValid()) { |
1338 | DestOpCode = SPIRV::OpConstantNull; |
1339 | } else { |
1340 | DestOpCode = TargetOpcode::COPY; |
1341 | Reg = OpReg; |
1342 | } |
1343 | return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DestOpCode)) |
1344 | .addDef(I.getOperand(i: 0).getReg()) |
1345 | .addUse(Reg) |
1346 | .constrainAllUses(TII, TRI, RBI); |
1347 | } |
1348 | return false; |
1349 | } |
1350 | |
1351 | bool SPIRVInstructionSelector::selectConstVector(Register ResVReg, |
1352 | const SPIRVType *ResType, |
1353 | MachineInstr &I) const { |
1354 | // TODO: only const case is supported for now. |
1355 | assert(std::all_of( |
1356 | I.operands_begin(), I.operands_end(), [this](const MachineOperand &MO) { |
1357 | if (MO.isDef()) |
1358 | return true; |
1359 | if (!MO.isReg()) |
1360 | return false; |
1361 | SPIRVType *ConstTy = this->MRI->getVRegDef(MO.getReg()); |
1362 | assert(ConstTy && ConstTy->getOpcode() == SPIRV::ASSIGN_TYPE && |
1363 | ConstTy->getOperand(1).isReg()); |
1364 | Register ConstReg = ConstTy->getOperand(1).getReg(); |
1365 | const MachineInstr *Const = this->MRI->getVRegDef(ConstReg); |
1366 | assert(Const); |
1367 | return (Const->getOpcode() == TargetOpcode::G_CONSTANT || |
1368 | Const->getOpcode() == TargetOpcode::G_FCONSTANT); |
1369 | })); |
1370 | |
1371 | auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), |
1372 | TII.get(SPIRV::OpConstantComposite)) |
1373 | .addDef(ResVReg) |
1374 | .addUse(GR.getSPIRVTypeID(ResType)); |
1375 | for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i) |
1376 | MIB.addUse(I.getOperand(i).getReg()); |
1377 | return MIB.constrainAllUses(TII, TRI, RBI); |
1378 | } |
1379 | |
1380 | static unsigned getArrayComponentCount(MachineRegisterInfo *MRI, |
1381 | const SPIRVType *ResType) { |
1382 | Register OpReg = ResType->getOperand(i: 2).getReg(); |
1383 | SPIRVType *OpDef = MRI->getVRegDef(Reg: OpReg); |
1384 | if (!OpDef) |
1385 | return 0; |
1386 | if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && |
1387 | OpDef->getOperand(1).isReg()) { |
1388 | if (SPIRVType *RefDef = MRI->getVRegDef(Reg: OpDef->getOperand(i: 1).getReg())) |
1389 | OpDef = RefDef; |
1390 | } |
1391 | unsigned N = OpDef->getOpcode() == TargetOpcode::G_CONSTANT |
1392 | ? OpDef->getOperand(i: 1).getCImm()->getValue().getZExtValue() |
1393 | : 0; |
1394 | return N; |
1395 | } |
1396 | |
1397 | // Return true if the type represents a constant register |
1398 | static bool isConstReg(MachineRegisterInfo *MRI, SPIRVType *OpDef) { |
1399 | if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && |
1400 | OpDef->getOperand(1).isReg()) { |
1401 | if (SPIRVType *RefDef = MRI->getVRegDef(Reg: OpDef->getOperand(i: 1).getReg())) |
1402 | OpDef = RefDef; |
1403 | } |
1404 | return OpDef->getOpcode() == TargetOpcode::G_CONSTANT || |
1405 | OpDef->getOpcode() == TargetOpcode::G_FCONSTANT; |
1406 | } |
1407 | |
1408 | // Return true if the virtual register represents a constant |
1409 | static bool isConstReg(MachineRegisterInfo *MRI, Register OpReg) { |
1410 | if (SPIRVType *OpDef = MRI->getVRegDef(Reg: OpReg)) |
1411 | return isConstReg(MRI, OpDef); |
1412 | return false; |
1413 | } |
1414 | |
1415 | bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg, |
1416 | const SPIRVType *ResType, |
1417 | MachineInstr &I) const { |
1418 | unsigned N = 0; |
1419 | if (ResType->getOpcode() == SPIRV::OpTypeVector) |
1420 | N = GR.getScalarOrVectorComponentCount(Type: ResType); |
1421 | else if (ResType->getOpcode() == SPIRV::OpTypeArray) |
1422 | N = getArrayComponentCount(MRI, ResType); |
1423 | else |
1424 | report_fatal_error(reason: "Cannot select G_SPLAT_VECTOR with a non-vector result" ); |
1425 | |
1426 | unsigned OpIdx = I.getNumExplicitDefs(); |
1427 | if (!I.getOperand(i: OpIdx).isReg()) |
1428 | report_fatal_error(reason: "Unexpected argument in G_SPLAT_VECTOR" ); |
1429 | |
1430 | // check if we may construct a constant vector |
1431 | Register OpReg = I.getOperand(i: OpIdx).getReg(); |
1432 | bool IsConst = isConstReg(MRI, OpReg); |
1433 | |
1434 | if (!IsConst && N < 2) |
1435 | report_fatal_error( |
1436 | reason: "There must be at least two constituent operands in a vector" ); |
1437 | |
1438 | auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), |
1439 | TII.get(IsConst ? SPIRV::OpConstantComposite |
1440 | : SPIRV::OpCompositeConstruct)) |
1441 | .addDef(ResVReg) |
1442 | .addUse(GR.getSPIRVTypeID(ResType)); |
1443 | for (unsigned i = 0; i < N; ++i) |
1444 | MIB.addUse(OpReg); |
1445 | return MIB.constrainAllUses(TII, TRI, RBI); |
1446 | } |
1447 | |
1448 | bool SPIRVInstructionSelector::selectCmp(Register ResVReg, |
1449 | const SPIRVType *ResType, |
1450 | unsigned CmpOpc, |
1451 | MachineInstr &I) const { |
1452 | Register Cmp0 = I.getOperand(i: 2).getReg(); |
1453 | Register Cmp1 = I.getOperand(i: 3).getReg(); |
1454 | assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() == |
1455 | GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() && |
1456 | "CMP operands should have the same type" ); |
1457 | return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CmpOpc)) |
1458 | .addDef(ResVReg) |
1459 | .addUse(GR.getSPIRVTypeID(SpirvType: ResType)) |
1460 | .addUse(Cmp0) |
1461 | .addUse(Cmp1) |
1462 | .constrainAllUses(TII, TRI, RBI); |
1463 | } |
1464 | |
1465 | bool SPIRVInstructionSelector::selectICmp(Register ResVReg, |
1466 | const SPIRVType *ResType, |
1467 | MachineInstr &I) const { |
1468 | auto Pred = I.getOperand(i: 1).getPredicate(); |
1469 | unsigned CmpOpc; |
1470 | |
1471 | Register CmpOperand = I.getOperand(i: 2).getReg(); |
1472 | if (GR.isScalarOfType(CmpOperand, SPIRV::OpTypePointer)) |
1473 | CmpOpc = getPtrCmpOpcode(Pred); |
1474 | else if (GR.isScalarOrVectorOfType(CmpOperand, SPIRV::OpTypeBool)) |
1475 | CmpOpc = getBoolCmpOpcode(PredNum: Pred); |
1476 | else |
1477 | CmpOpc = getICmpOpcode(PredNum: Pred); |
1478 | return selectCmp(ResVReg, ResType, CmpOpc, I); |
1479 | } |
1480 | |
1481 | void SPIRVInstructionSelector::renderFImm32(MachineInstrBuilder &MIB, |
1482 | const MachineInstr &I, |
1483 | int OpIdx) const { |
1484 | assert(I.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && |
1485 | "Expected G_FCONSTANT" ); |
1486 | const ConstantFP *FPImm = I.getOperand(i: 1).getFPImm(); |
1487 | addNumImm(Imm: FPImm->getValueAPF().bitcastToAPInt(), MIB); |
1488 | } |
1489 | |
1490 | void SPIRVInstructionSelector::renderImm32(MachineInstrBuilder &MIB, |
1491 | const MachineInstr &I, |
1492 | int OpIdx) const { |
1493 | assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && |
1494 | "Expected G_CONSTANT" ); |
1495 | addNumImm(Imm: I.getOperand(i: 1).getCImm()->getValue(), MIB); |
1496 | } |
1497 | |
1498 | Register |
1499 | SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I, |
1500 | const SPIRVType *ResType) const { |
1501 | Type *LLVMTy = IntegerType::get(C&: GR.CurMF->getFunction().getContext(), NumBits: 32); |
1502 | const SPIRVType *SpvI32Ty = |
1503 | ResType ? ResType : GR.getOrCreateSPIRVIntegerType(BitWidth: 32, I, TII); |
1504 | // Find a constant in DT or build a new one. |
1505 | auto ConstInt = ConstantInt::get(Ty: LLVMTy, V: Val); |
1506 | Register NewReg = GR.find(C: ConstInt, MF: GR.CurMF); |
1507 | if (!NewReg.isValid()) { |
1508 | NewReg = MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 32)); |
1509 | GR.add(C: ConstInt, MF: GR.CurMF, R: NewReg); |
1510 | MachineInstr *MI; |
1511 | MachineBasicBlock &BB = *I.getParent(); |
1512 | if (Val == 0) { |
1513 | MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) |
1514 | .addDef(NewReg) |
1515 | .addUse(GR.getSPIRVTypeID(SpvI32Ty)); |
1516 | } else { |
1517 | MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) |
1518 | .addDef(NewReg) |
1519 | .addUse(GR.getSPIRVTypeID(SpvI32Ty)) |
1520 | .addImm(APInt(32, Val).getZExtValue()); |
1521 | } |
1522 | constrainSelectedInstRegOperands(*MI, TII, TRI, RBI); |
1523 | } |
1524 | return NewReg; |
1525 | } |
1526 | |
1527 | bool SPIRVInstructionSelector::selectFCmp(Register ResVReg, |
1528 | const SPIRVType *ResType, |
1529 | MachineInstr &I) const { |
1530 | unsigned CmpOp = getFCmpOpcode(PredNum: I.getOperand(i: 1).getPredicate()); |
1531 | return selectCmp(ResVReg, ResType, CmpOpc: CmpOp, I); |
1532 | } |
1533 | |
1534 | Register SPIRVInstructionSelector::buildZerosVal(const SPIRVType *ResType, |
1535 | MachineInstr &I) const { |
1536 | // OpenCL uses nulls for Zero. In HLSL we don't use null constants. |
1537 | bool ZeroAsNull = STI.isOpenCLEnv(); |
1538 | if (ResType->getOpcode() == SPIRV::OpTypeVector) |
1539 | return GR.getOrCreateConstVector(Val: 0UL, I, SpvType: ResType, TII, ZeroAsNull); |
1540 | return GR.getOrCreateConstInt(Val: 0, I, SpvType: ResType, TII, ZeroAsNull); |
1541 | } |
1542 | |
1543 | static APFloat getZeroFP(const Type *LLVMFloatTy) { |
1544 | if (!LLVMFloatTy) |
1545 | return APFloat::getZero(Sem: APFloat::IEEEsingle()); |
1546 | switch (LLVMFloatTy->getScalarType()->getTypeID()) { |
1547 | case Type::HalfTyID: |
1548 | return APFloat::getZero(Sem: APFloat::IEEEhalf()); |
1549 | default: |
1550 | case Type::FloatTyID: |
1551 | return APFloat::getZero(Sem: APFloat::IEEEsingle()); |
1552 | case Type::DoubleTyID: |
1553 | return APFloat::getZero(Sem: APFloat::IEEEdouble()); |
1554 | } |
1555 | } |
1556 | |
1557 | Register SPIRVInstructionSelector::buildZerosValF(const SPIRVType *ResType, |
1558 | MachineInstr &I) const { |
1559 | // OpenCL uses nulls for Zero. In HLSL we don't use null constants. |
1560 | bool ZeroAsNull = STI.isOpenCLEnv(); |
1561 | APFloat VZero = getZeroFP(LLVMFloatTy: GR.getTypeForSPIRVType(Ty: ResType)); |
1562 | if (ResType->getOpcode() == SPIRV::OpTypeVector) |
1563 | return GR.getOrCreateConstVector(Val: VZero, I, SpvType: ResType, TII, ZeroAsNull); |
1564 | return GR.getOrCreateConstFP(Val: VZero, I, SpvType: ResType, TII, ZeroAsNull); |
1565 | } |
1566 | |
1567 | Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes, |
1568 | const SPIRVType *ResType, |
1569 | MachineInstr &I) const { |
1570 | unsigned BitWidth = GR.getScalarOrVectorBitWidth(Type: ResType); |
1571 | APInt One = |
1572 | AllOnes ? APInt::getAllOnes(numBits: BitWidth) : APInt::getOneBitSet(numBits: BitWidth, BitNo: 0); |
1573 | if (ResType->getOpcode() == SPIRV::OpTypeVector) |
1574 | return GR.getOrCreateConstVector(Val: One.getZExtValue(), I, SpvType: ResType, TII); |
1575 | return GR.getOrCreateConstInt(Val: One.getZExtValue(), I, SpvType: ResType, TII); |
1576 | } |
1577 | |
1578 | bool SPIRVInstructionSelector::selectSelect(Register ResVReg, |
1579 | const SPIRVType *ResType, |
1580 | MachineInstr &I, |
1581 | bool IsSigned) const { |
1582 | // To extend a bool, we need to use OpSelect between constants. |
1583 | Register ZeroReg = buildZerosVal(ResType, I); |
1584 | Register OneReg = buildOnesVal(AllOnes: IsSigned, ResType, I); |
1585 | bool IsScalarBool = |
1586 | GR.isScalarOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool); |
1587 | unsigned Opcode = |
1588 | IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectSIVCond; |
1589 | return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) |
1590 | .addDef(ResVReg) |
1591 | .addUse(GR.getSPIRVTypeID(SpirvType: ResType)) |
1592 | .addUse(I.getOperand(i: 1).getReg()) |
1593 | .addUse(OneReg) |
1594 | .addUse(ZeroReg) |
1595 | .constrainAllUses(TII, TRI, RBI); |
1596 | } |
1597 | |
1598 | bool SPIRVInstructionSelector::selectIToF(Register ResVReg, |
1599 | const SPIRVType *ResType, |
1600 | MachineInstr &I, bool IsSigned, |
1601 | unsigned Opcode) const { |
1602 | Register SrcReg = I.getOperand(i: 1).getReg(); |
1603 | // We can convert bool value directly to float type without OpConvert*ToF, |
1604 | // however the translator generates OpSelect+OpConvert*ToF, so we do the same. |
1605 | if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) { |
1606 | unsigned BitWidth = GR.getScalarOrVectorBitWidth(Type: ResType); |
1607 | SPIRVType *TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII); |
1608 | if (ResType->getOpcode() == SPIRV::OpTypeVector) { |
1609 | const unsigned NumElts = ResType->getOperand(i: 2).getImm(); |
1610 | TmpType = GR.getOrCreateSPIRVVectorType(BaseType: TmpType, NumElements: NumElts, I, TII); |
1611 | } |
1612 | SrcReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); |
1613 | selectSelect(ResVReg: SrcReg, ResType: TmpType, I, IsSigned: false); |
1614 | } |
1615 | return selectUnOpWithSrc(ResVReg, ResType, I, SrcReg, Opcode); |
1616 | } |
1617 | |
1618 | bool SPIRVInstructionSelector::selectExt(Register ResVReg, |
1619 | const SPIRVType *ResType, |
1620 | MachineInstr &I, bool IsSigned) const { |
1621 | Register SrcReg = I.getOperand(i: 1).getReg(); |
1622 | if (GR.isScalarOrVectorOfType(SrcReg, SPIRV::OpTypeBool)) |
1623 | return selectSelect(ResVReg, ResType, I, IsSigned); |
1624 | |
1625 | SPIRVType *SrcType = GR.getSPIRVTypeForVReg(VReg: SrcReg); |
1626 | if (SrcType == ResType) |
1627 | return BuildMI(*I.getParent(), I, I.getDebugLoc(), |
1628 | TII.get(TargetOpcode::COPY)) |
1629 | .addDef(ResVReg) |
1630 | .addUse(SrcReg) |
1631 | .constrainAllUses(TII, TRI, RBI); |
1632 | |
1633 | unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; |
1634 | return selectUnOp(ResVReg, ResType, I, Opcode); |
1635 | } |
1636 | |
1637 | bool SPIRVInstructionSelector::selectIntToBool(Register IntReg, |
1638 | Register ResVReg, |
1639 | MachineInstr &I, |
1640 | const SPIRVType *IntTy, |
1641 | const SPIRVType *BoolTy) const { |
1642 | // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero. |
1643 | Register BitIntReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); |
1644 | bool IsVectorTy = IntTy->getOpcode() == SPIRV::OpTypeVector; |
1645 | unsigned Opcode = IsVectorTy ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS; |
1646 | Register Zero = buildZerosVal(ResType: IntTy, I); |
1647 | Register One = buildOnesVal(AllOnes: false, ResType: IntTy, I); |
1648 | MachineBasicBlock &BB = *I.getParent(); |
1649 | BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) |
1650 | .addDef(BitIntReg) |
1651 | .addUse(GR.getSPIRVTypeID(SpirvType: IntTy)) |
1652 | .addUse(IntReg) |
1653 | .addUse(One) |
1654 | .constrainAllUses(TII, TRI, RBI); |
1655 | return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) |
1656 | .addDef(ResVReg) |
1657 | .addUse(GR.getSPIRVTypeID(BoolTy)) |
1658 | .addUse(BitIntReg) |
1659 | .addUse(Zero) |
1660 | .constrainAllUses(TII, TRI, RBI); |
1661 | } |
1662 | |
1663 | bool SPIRVInstructionSelector::selectTrunc(Register ResVReg, |
1664 | const SPIRVType *ResType, |
1665 | MachineInstr &I) const { |
1666 | Register IntReg = I.getOperand(i: 1).getReg(); |
1667 | const SPIRVType *ArgType = GR.getSPIRVTypeForVReg(VReg: IntReg); |
1668 | if (GR.isScalarOrVectorOfType(ResVReg, SPIRV::OpTypeBool)) |
1669 | return selectIntToBool(IntReg, ResVReg, I, IntTy: ArgType, BoolTy: ResType); |
1670 | if (ArgType == ResType) |
1671 | return BuildMI(*I.getParent(), I, I.getDebugLoc(), |
1672 | TII.get(TargetOpcode::COPY)) |
1673 | .addDef(ResVReg) |
1674 | .addUse(IntReg) |
1675 | .constrainAllUses(TII, TRI, RBI); |
1676 | bool IsSigned = GR.isScalarOrVectorSigned(Type: ResType); |
1677 | unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; |
1678 | return selectUnOp(ResVReg, ResType, I, Opcode); |
1679 | } |
1680 | |
1681 | bool SPIRVInstructionSelector::selectConst(Register ResVReg, |
1682 | const SPIRVType *ResType, |
1683 | const APInt &Imm, |
1684 | MachineInstr &I) const { |
1685 | unsigned TyOpcode = ResType->getOpcode(); |
1686 | assert(TyOpcode != SPIRV::OpTypePointer || Imm.isZero()); |
1687 | MachineBasicBlock &BB = *I.getParent(); |
1688 | if ((TyOpcode == SPIRV::OpTypePointer || TyOpcode == SPIRV::OpTypeEvent) && |
1689 | Imm.isZero()) |
1690 | return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) |
1691 | .addDef(ResVReg) |
1692 | .addUse(GR.getSPIRVTypeID(ResType)) |
1693 | .constrainAllUses(TII, TRI, RBI); |
1694 | if (TyOpcode == SPIRV::OpTypeInt) { |
1695 | assert(Imm.getBitWidth() <= 64 && "Unsupported integer width!" ); |
1696 | Register Reg = GR.getOrCreateConstInt(Val: Imm.getZExtValue(), I, SpvType: ResType, TII); |
1697 | if (Reg == ResVReg) |
1698 | return true; |
1699 | return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) |
1700 | .addDef(ResVReg) |
1701 | .addUse(Reg) |
1702 | .constrainAllUses(TII, TRI, RBI); |
1703 | } |
1704 | auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) |
1705 | .addDef(ResVReg) |
1706 | .addUse(GR.getSPIRVTypeID(ResType)); |
1707 | // <=32-bit integers should be caught by the sdag pattern. |
1708 | assert(Imm.getBitWidth() > 32); |
1709 | addNumImm(Imm, MIB); |
1710 | return MIB.constrainAllUses(TII, TRI, RBI); |
1711 | } |
1712 | |
1713 | bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg, |
1714 | const SPIRVType *ResType, |
1715 | MachineInstr &I) const { |
1716 | return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) |
1717 | .addDef(ResVReg) |
1718 | .addUse(GR.getSPIRVTypeID(ResType)) |
1719 | .constrainAllUses(TII, TRI, RBI); |
1720 | } |
1721 | |
1722 | static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { |
1723 | assert(MO.isReg()); |
1724 | const SPIRVType *TypeInst = MRI->getVRegDef(Reg: MO.getReg()); |
1725 | if (TypeInst->getOpcode() != SPIRV::ASSIGN_TYPE) |
1726 | return false; |
1727 | assert(TypeInst->getOperand(1).isReg()); |
1728 | MachineInstr *ImmInst = MRI->getVRegDef(Reg: TypeInst->getOperand(i: 1).getReg()); |
1729 | return ImmInst->getOpcode() == TargetOpcode::G_CONSTANT; |
1730 | } |
1731 | |
1732 | static int64_t foldImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { |
1733 | const SPIRVType *TypeInst = MRI->getVRegDef(Reg: MO.getReg()); |
1734 | MachineInstr *ImmInst = MRI->getVRegDef(Reg: TypeInst->getOperand(i: 1).getReg()); |
1735 | assert(ImmInst->getOpcode() == TargetOpcode::G_CONSTANT); |
1736 | return ImmInst->getOperand(i: 1).getCImm()->getZExtValue(); |
1737 | } |
1738 | |
1739 | bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg, |
1740 | const SPIRVType *ResType, |
1741 | MachineInstr &I) const { |
1742 | MachineBasicBlock &BB = *I.getParent(); |
1743 | auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeInsert)) |
1744 | .addDef(ResVReg) |
1745 | .addUse(GR.getSPIRVTypeID(ResType)) |
1746 | // object to insert |
1747 | .addUse(I.getOperand(3).getReg()) |
1748 | // composite to insert into |
1749 | .addUse(I.getOperand(2).getReg()); |
1750 | for (unsigned i = 4; i < I.getNumOperands(); i++) |
1751 | MIB.addImm(foldImm(MO: I.getOperand(i), MRI)); |
1752 | return MIB.constrainAllUses(TII, TRI, RBI); |
1753 | } |
1754 | |
1755 | bool SPIRVInstructionSelector::(Register ResVReg, |
1756 | const SPIRVType *ResType, |
1757 | MachineInstr &I) const { |
1758 | MachineBasicBlock &BB = *I.getParent(); |
1759 | auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) |
1760 | .addDef(ResVReg) |
1761 | .addUse(GR.getSPIRVTypeID(ResType)) |
1762 | .addUse(I.getOperand(2).getReg()); |
1763 | for (unsigned i = 3; i < I.getNumOperands(); i++) |
1764 | MIB.addImm(foldImm(MO: I.getOperand(i), MRI)); |
1765 | return MIB.constrainAllUses(TII, TRI, RBI); |
1766 | } |
1767 | |
1768 | bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg, |
1769 | const SPIRVType *ResType, |
1770 | MachineInstr &I) const { |
1771 | if (isImm(MO: I.getOperand(i: 4), MRI)) |
1772 | return selectInsertVal(ResVReg, ResType, I); |
1773 | MachineBasicBlock &BB = *I.getParent(); |
1774 | return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorInsertDynamic)) |
1775 | .addDef(ResVReg) |
1776 | .addUse(GR.getSPIRVTypeID(ResType)) |
1777 | .addUse(I.getOperand(2).getReg()) |
1778 | .addUse(I.getOperand(3).getReg()) |
1779 | .addUse(I.getOperand(4).getReg()) |
1780 | .constrainAllUses(TII, TRI, RBI); |
1781 | } |
1782 | |
1783 | bool SPIRVInstructionSelector::(Register ResVReg, |
1784 | const SPIRVType *ResType, |
1785 | MachineInstr &I) const { |
1786 | if (isImm(MO: I.getOperand(i: 3), MRI)) |
1787 | return selectExtractVal(ResVReg, ResType, I); |
1788 | MachineBasicBlock &BB = *I.getParent(); |
1789 | return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorExtractDynamic)) |
1790 | .addDef(ResVReg) |
1791 | .addUse(GR.getSPIRVTypeID(ResType)) |
1792 | .addUse(I.getOperand(2).getReg()) |
1793 | .addUse(I.getOperand(3).getReg()) |
1794 | .constrainAllUses(TII, TRI, RBI); |
1795 | } |
1796 | |
1797 | bool SPIRVInstructionSelector::selectGEP(Register ResVReg, |
1798 | const SPIRVType *ResType, |
1799 | MachineInstr &I) const { |
1800 | const bool IsGEPInBounds = I.getOperand(i: 2).getImm(); |
1801 | |
1802 | // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator only |
1803 | // relies on PtrAccessChain, so we'll try not to deviate. For Vulkan however, |
1804 | // we have to use Op[InBounds]AccessChain. |
1805 | const unsigned Opcode = STI.isVulkanEnv() |
1806 | ? (IsGEPInBounds ? SPIRV::OpInBoundsAccessChain |
1807 | : SPIRV::OpAccessChain) |
1808 | : (IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain |
1809 | : SPIRV::OpPtrAccessChain); |
1810 | |
1811 | auto Res = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) |
1812 | .addDef(ResVReg) |
1813 | .addUse(GR.getSPIRVTypeID(SpirvType: ResType)) |
1814 | // Object to get a pointer to. |
1815 | .addUse(I.getOperand(i: 3).getReg()); |
1816 | // Adding indices. |
1817 | const unsigned StartingIndex = |
1818 | (Opcode == SPIRV::OpAccessChain || Opcode == SPIRV::OpInBoundsAccessChain) |
1819 | ? 5 |
1820 | : 4; |
1821 | for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i) |
1822 | Res.addUse(I.getOperand(i).getReg()); |
1823 | return Res.constrainAllUses(TII, TRI, RBI); |
1824 | } |
1825 | |
1826 | // Maybe wrap a value into OpSpecConstantOp |
1827 | bool SPIRVInstructionSelector::wrapIntoSpecConstantOp( |
1828 | MachineInstr &I, SmallVector<Register> &CompositeArgs) const { |
1829 | bool Result = true; |
1830 | unsigned Lim = I.getNumExplicitOperands(); |
1831 | for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) { |
1832 | Register OpReg = I.getOperand(i).getReg(); |
1833 | SPIRVType *OpDefine = MRI->getVRegDef(Reg: OpReg); |
1834 | SPIRVType *OpType = GR.getSPIRVTypeForVReg(VReg: OpReg); |
1835 | if (!OpDefine || !OpType || isConstReg(MRI, OpDef: OpDefine) || |
1836 | OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST) { |
1837 | // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed |
1838 | // by selectAddrSpaceCast() |
1839 | CompositeArgs.push_back(Elt: OpReg); |
1840 | continue; |
1841 | } |
1842 | MachineFunction *MF = I.getMF(); |
1843 | Register WrapReg = GR.find(MI: OpDefine, MF); |
1844 | if (WrapReg.isValid()) { |
1845 | CompositeArgs.push_back(Elt: WrapReg); |
1846 | continue; |
1847 | } |
1848 | // Create a new register for the wrapper |
1849 | WrapReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); |
1850 | GR.add(MI: OpDefine, MF, R: WrapReg); |
1851 | CompositeArgs.push_back(Elt: WrapReg); |
1852 | // Decorate the wrapper register and generate a new instruction |
1853 | MRI->setType(VReg: WrapReg, Ty: LLT::pointer(AddressSpace: 0, SizeInBits: 32)); |
1854 | GR.assignSPIRVTypeToVReg(Type: OpType, VReg: WrapReg, MF&: *MF); |
1855 | MachineBasicBlock &BB = *I.getParent(); |
1856 | Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) |
1857 | .addDef(WrapReg) |
1858 | .addUse(GR.getSPIRVTypeID(OpType)) |
1859 | .addImm(static_cast<uint32_t>(SPIRV::Opcode::Bitcast)) |
1860 | .addUse(OpReg) |
1861 | .constrainAllUses(TII, TRI, RBI); |
1862 | if (!Result) |
1863 | break; |
1864 | } |
1865 | return Result; |
1866 | } |
1867 | |
1868 | bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, |
1869 | const SPIRVType *ResType, |
1870 | MachineInstr &I) const { |
1871 | MachineBasicBlock &BB = *I.getParent(); |
1872 | Intrinsic::ID IID = cast<GIntrinsic>(Val&: I).getIntrinsicID(); |
1873 | switch (IID) { |
1874 | case Intrinsic::spv_load: |
1875 | return selectLoad(ResVReg, ResType, I); |
1876 | case Intrinsic::spv_store: |
1877 | return selectStore(I); |
1878 | case Intrinsic::spv_extractv: |
1879 | return selectExtractVal(ResVReg, ResType, I); |
1880 | case Intrinsic::spv_insertv: |
1881 | return selectInsertVal(ResVReg, ResType, I); |
1882 | case Intrinsic::spv_extractelt: |
1883 | return selectExtractElt(ResVReg, ResType, I); |
1884 | case Intrinsic::spv_insertelt: |
1885 | return selectInsertElt(ResVReg, ResType, I); |
1886 | case Intrinsic::spv_gep: |
1887 | return selectGEP(ResVReg, ResType, I); |
1888 | case Intrinsic::spv_unref_global: |
1889 | case Intrinsic::spv_init_global: { |
1890 | MachineInstr *MI = MRI->getVRegDef(Reg: I.getOperand(i: 1).getReg()); |
1891 | MachineInstr *Init = I.getNumExplicitOperands() > 2 |
1892 | ? MRI->getVRegDef(Reg: I.getOperand(i: 2).getReg()) |
1893 | : nullptr; |
1894 | assert(MI); |
1895 | return selectGlobalValue(ResVReg: MI->getOperand(i: 0).getReg(), I&: *MI, Init); |
1896 | } |
1897 | case Intrinsic::spv_undef: { |
1898 | auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) |
1899 | .addDef(ResVReg) |
1900 | .addUse(GR.getSPIRVTypeID(ResType)); |
1901 | return MIB.constrainAllUses(TII, TRI, RBI); |
1902 | } |
1903 | case Intrinsic::spv_const_composite: { |
1904 | // If no values are attached, the composite is null constant. |
1905 | bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands(); |
1906 | // Select a proper instruction. |
1907 | unsigned Opcode = SPIRV::OpConstantNull; |
1908 | SmallVector<Register> CompositeArgs; |
1909 | if (!IsNull) { |
1910 | Opcode = SPIRV::OpConstantComposite; |
1911 | if (!wrapIntoSpecConstantOp(I, CompositeArgs)) |
1912 | return false; |
1913 | } |
1914 | auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) |
1915 | .addDef(ResVReg) |
1916 | .addUse(GR.getSPIRVTypeID(SpirvType: ResType)); |
1917 | // skip type MD node we already used when generated assign.type for this |
1918 | if (!IsNull) { |
1919 | for (Register OpReg : CompositeArgs) |
1920 | MIB.addUse(OpReg); |
1921 | } |
1922 | return MIB.constrainAllUses(TII, TRI, RBI); |
1923 | } |
1924 | case Intrinsic::spv_assign_name: { |
1925 | auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpName)); |
1926 | MIB.addUse(I.getOperand(i: I.getNumExplicitDefs() + 1).getReg()); |
1927 | for (unsigned i = I.getNumExplicitDefs() + 2; |
1928 | i < I.getNumExplicitOperands(); ++i) { |
1929 | MIB.addImm(I.getOperand(i).getImm()); |
1930 | } |
1931 | return MIB.constrainAllUses(TII, TRI, RBI); |
1932 | } |
1933 | case Intrinsic::spv_switch: { |
1934 | auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSwitch)); |
1935 | for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { |
1936 | if (I.getOperand(i).isReg()) |
1937 | MIB.addReg(I.getOperand(i).getReg()); |
1938 | else if (I.getOperand(i).isCImm()) |
1939 | addNumImm(I.getOperand(i).getCImm()->getValue(), MIB); |
1940 | else if (I.getOperand(i).isMBB()) |
1941 | MIB.addMBB(I.getOperand(i).getMBB()); |
1942 | else |
1943 | llvm_unreachable("Unexpected OpSwitch operand" ); |
1944 | } |
1945 | return MIB.constrainAllUses(TII, TRI, RBI); |
1946 | } |
1947 | case Intrinsic::spv_cmpxchg: |
1948 | return selectAtomicCmpXchg(ResVReg, ResType, I); |
1949 | case Intrinsic::spv_unreachable: |
1950 | BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUnreachable)); |
1951 | break; |
1952 | case Intrinsic::spv_alloca: |
1953 | return selectFrameIndex(ResVReg, ResType, I); |
1954 | case Intrinsic::spv_alloca_array: |
1955 | return selectAllocaArray(ResVReg, ResType, I); |
1956 | case Intrinsic::spv_assume: |
1957 | if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) |
1958 | BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAssumeTrueKHR)) |
1959 | .addUse(I.getOperand(1).getReg()); |
1960 | break; |
1961 | case Intrinsic::spv_expect: |
1962 | if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) |
1963 | BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExpectKHR)) |
1964 | .addDef(ResVReg) |
1965 | .addUse(GR.getSPIRVTypeID(ResType)) |
1966 | .addUse(I.getOperand(2).getReg()) |
1967 | .addUse(I.getOperand(3).getReg()); |
1968 | break; |
1969 | case Intrinsic::spv_thread_id: |
1970 | return selectSpvThreadId(ResVReg, ResType, I); |
1971 | case Intrinsic::spv_all: |
1972 | return selectAll(ResVReg, ResType, I); |
1973 | case Intrinsic::spv_any: |
1974 | return selectAny(ResVReg, ResType, I); |
1975 | case Intrinsic::spv_lerp: |
1976 | return selectFmix(ResVReg, ResType, I); |
1977 | case Intrinsic::spv_lifetime_start: |
1978 | case Intrinsic::spv_lifetime_end: { |
1979 | unsigned Op = IID == Intrinsic::spv_lifetime_start ? SPIRV::OpLifetimeStart |
1980 | : SPIRV::OpLifetimeStop; |
1981 | int64_t Size = I.getOperand(i: I.getNumExplicitDefs() + 1).getImm(); |
1982 | Register PtrReg = I.getOperand(i: I.getNumExplicitDefs() + 2).getReg(); |
1983 | unsigned PonteeOpType = GR.getPointeeTypeOp(PtrReg); |
1984 | bool IsNonvoidPtr = PonteeOpType != 0 && PonteeOpType != SPIRV::OpTypeVoid; |
1985 | if (Size == -1 || IsNonvoidPtr) |
1986 | Size = 0; |
1987 | BuildMI(BB, I, I.getDebugLoc(), TII.get(Op)).addUse(PtrReg).addImm(Size); |
1988 | } break; |
1989 | default: { |
1990 | std::string DiagMsg; |
1991 | raw_string_ostream OS(DiagMsg); |
1992 | I.print(OS); |
1993 | DiagMsg = "Intrinsic selection not implemented: " + DiagMsg; |
1994 | report_fatal_error(reason: DiagMsg.c_str(), gen_crash_diag: false); |
1995 | } |
1996 | } |
1997 | return true; |
1998 | } |
1999 | |
2000 | bool SPIRVInstructionSelector::selectAllocaArray(Register ResVReg, |
2001 | const SPIRVType *ResType, |
2002 | MachineInstr &I) const { |
2003 | // there was an allocation size parameter to the allocation instruction |
2004 | // that is not 1 |
2005 | MachineBasicBlock &BB = *I.getParent(); |
2006 | return BuildMI(BB, I, I.getDebugLoc(), |
2007 | TII.get(SPIRV::OpVariableLengthArrayINTEL)) |
2008 | .addDef(ResVReg) |
2009 | .addUse(GR.getSPIRVTypeID(ResType)) |
2010 | .addUse(I.getOperand(2).getReg()) |
2011 | .constrainAllUses(TII, TRI, RBI); |
2012 | } |
2013 | |
2014 | bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg, |
2015 | const SPIRVType *ResType, |
2016 | MachineInstr &I) const { |
2017 | // Change order of instructions if needed: all OpVariable instructions in a |
2018 | // function must be the first instructions in the first block |
2019 | MachineFunction *MF = I.getParent()->getParent(); |
2020 | MachineBasicBlock *MBB = &MF->front(); |
2021 | auto It = MBB->SkipPHIsAndLabels(I: MBB->begin()), E = MBB->end(); |
2022 | bool = false; |
2023 | unsigned Opcode; |
2024 | for (; It != E && It != I; ++It) { |
2025 | Opcode = It->getOpcode(); |
2026 | if (Opcode == SPIRV::OpFunction || Opcode == SPIRV::OpFunctionParameter) { |
2027 | IsHeader = true; |
2028 | } else if (IsHeader && |
2029 | !(Opcode == SPIRV::ASSIGN_TYPE || Opcode == SPIRV::OpLabel)) { |
2030 | ++It; |
2031 | break; |
2032 | } |
2033 | } |
2034 | return BuildMI(*MBB, It, It->getDebugLoc(), TII.get(SPIRV::OpVariable)) |
2035 | .addDef(ResVReg) |
2036 | .addUse(GR.getSPIRVTypeID(ResType)) |
2037 | .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function)) |
2038 | .constrainAllUses(TII, TRI, RBI); |
2039 | } |
2040 | |
2041 | bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const { |
2042 | // InstructionSelector walks backwards through the instructions. We can use |
2043 | // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR |
2044 | // first, so can generate an OpBranchConditional here. If there is no |
2045 | // G_BRCOND, we just use OpBranch for a regular unconditional branch. |
2046 | const MachineInstr *PrevI = I.getPrevNode(); |
2047 | MachineBasicBlock &MBB = *I.getParent(); |
2048 | if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) { |
2049 | return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) |
2050 | .addUse(PrevI->getOperand(0).getReg()) |
2051 | .addMBB(PrevI->getOperand(1).getMBB()) |
2052 | .addMBB(I.getOperand(0).getMBB()) |
2053 | .constrainAllUses(TII, TRI, RBI); |
2054 | } |
2055 | return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranch)) |
2056 | .addMBB(I.getOperand(0).getMBB()) |
2057 | .constrainAllUses(TII, TRI, RBI); |
2058 | } |
2059 | |
2060 | bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const { |
2061 | // InstructionSelector walks backwards through the instructions. For an |
2062 | // explicit conditional branch with no fallthrough, we use both a G_BR and a |
2063 | // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and |
2064 | // generate the OpBranchConditional in selectBranch above. |
2065 | // |
2066 | // If an OpBranchConditional has been generated, we simply return, as the work |
2067 | // is alread done. If there is no OpBranchConditional, LLVM must be relying on |
2068 | // implicit fallthrough to the next basic block, so we need to create an |
2069 | // OpBranchConditional with an explicit "false" argument pointing to the next |
2070 | // basic block that LLVM would fall through to. |
2071 | const MachineInstr *NextI = I.getNextNode(); |
2072 | // Check if this has already been successfully selected. |
2073 | if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional) |
2074 | return true; |
2075 | // Must be relying on implicit block fallthrough, so generate an |
2076 | // OpBranchConditional with the "next" basic block as the "false" target. |
2077 | MachineBasicBlock &MBB = *I.getParent(); |
2078 | unsigned NextMBBNum = MBB.getNextNode()->getNumber(); |
2079 | MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(N: NextMBBNum); |
2080 | return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) |
2081 | .addUse(I.getOperand(0).getReg()) |
2082 | .addMBB(I.getOperand(1).getMBB()) |
2083 | .addMBB(NextMBB) |
2084 | .constrainAllUses(TII, TRI, RBI); |
2085 | } |
2086 | |
2087 | bool SPIRVInstructionSelector::selectPhi(Register ResVReg, |
2088 | const SPIRVType *ResType, |
2089 | MachineInstr &I) const { |
2090 | auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpPhi)) |
2091 | .addDef(ResVReg) |
2092 | .addUse(GR.getSPIRVTypeID(ResType)); |
2093 | const unsigned NumOps = I.getNumOperands(); |
2094 | for (unsigned i = 1; i < NumOps; i += 2) { |
2095 | MIB.addUse(I.getOperand(i: i + 0).getReg()); |
2096 | MIB.addMBB(I.getOperand(i: i + 1).getMBB()); |
2097 | } |
2098 | return MIB.constrainAllUses(TII, TRI, RBI); |
2099 | } |
2100 | |
2101 | bool SPIRVInstructionSelector::selectGlobalValue( |
2102 | Register ResVReg, MachineInstr &I, const MachineInstr *Init) const { |
2103 | // FIXME: don't use MachineIRBuilder here, replace it with BuildMI. |
2104 | MachineIRBuilder MIRBuilder(I); |
2105 | const GlobalValue *GV = I.getOperand(i: 1).getGlobal(); |
2106 | Type *GVType = GR.getDeducedGlobalValueType(Global: GV); |
2107 | SPIRVType *PointerBaseType; |
2108 | if (GVType->isArrayTy()) { |
2109 | SPIRVType *ArrayElementType = |
2110 | GR.getOrCreateSPIRVType(GVType->getArrayElementType(), MIRBuilder, |
2111 | SPIRV::AccessQualifier::ReadWrite, false); |
2112 | PointerBaseType = GR.getOrCreateSPIRVArrayType( |
2113 | BaseType: ArrayElementType, NumElements: GVType->getArrayNumElements(), I, TII); |
2114 | } else { |
2115 | PointerBaseType = GR.getOrCreateSPIRVType( |
2116 | GVType, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); |
2117 | } |
2118 | SPIRVType *ResType = GR.getOrCreateSPIRVPointerType( |
2119 | BaseType: PointerBaseType, I, TII, |
2120 | SClass: addressSpaceToStorageClass(AddrSpace: GV->getAddressSpace(), STI)); |
2121 | |
2122 | std::string GlobalIdent; |
2123 | if (!GV->hasName()) { |
2124 | unsigned &ID = UnnamedGlobalIDs[GV]; |
2125 | if (ID == 0) |
2126 | ID = UnnamedGlobalIDs.size(); |
2127 | GlobalIdent = "__unnamed_" + Twine(ID).str(); |
2128 | } else { |
2129 | GlobalIdent = GV->getGlobalIdentifier(); |
2130 | } |
2131 | |
2132 | // Behaviour of functions as operands depends on availability of the |
2133 | // corresponding extension (SPV_INTEL_function_pointers): |
2134 | // - If there is an extension to operate with functions as operands: |
2135 | // We create a proper constant operand and evaluate a correct type for a |
2136 | // function pointer. |
2137 | // - Without the required extension: |
2138 | // We have functions as operands in tests with blocks of instruction e.g. in |
2139 | // transcoding/global_block.ll. These operands are not used and should be |
2140 | // substituted by zero constants. Their type is expected to be always |
2141 | // OpTypePointer Function %uchar. |
2142 | if (isa<Function>(Val: GV)) { |
2143 | const Constant *ConstVal = GV; |
2144 | MachineBasicBlock &BB = *I.getParent(); |
2145 | Register NewReg = GR.find(C: ConstVal, MF: GR.CurMF); |
2146 | if (!NewReg.isValid()) { |
2147 | Register NewReg = ResVReg; |
2148 | GR.add(C: ConstVal, MF: GR.CurMF, R: NewReg); |
2149 | const Function *GVFun = |
2150 | STI.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers) |
2151 | ? dyn_cast<Function>(GV) |
2152 | : nullptr; |
2153 | if (GVFun) { |
2154 | // References to a function via function pointers generate virtual |
2155 | // registers without a definition. We will resolve it later, during |
2156 | // module analysis stage. |
2157 | MachineRegisterInfo *MRI = MIRBuilder.getMRI(); |
2158 | Register FuncVReg = MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 32)); |
2159 | MRI->setRegClass(FuncVReg, &SPIRV::IDRegClass); |
2160 | MachineInstrBuilder MB = |
2161 | BuildMI(BB, I, I.getDebugLoc(), |
2162 | TII.get(SPIRV::OpConstantFunctionPointerINTEL)) |
2163 | .addDef(NewReg) |
2164 | .addUse(GR.getSPIRVTypeID(ResType)) |
2165 | .addUse(FuncVReg); |
2166 | // mapping the function pointer to the used Function |
2167 | GR.recordFunctionPointer(MO: &MB.getInstr()->getOperand(i: 2), F: GVFun); |
2168 | return MB.constrainAllUses(TII, TRI, RBI); |
2169 | } |
2170 | return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) |
2171 | .addDef(NewReg) |
2172 | .addUse(GR.getSPIRVTypeID(ResType)) |
2173 | .constrainAllUses(TII, TRI, RBI); |
2174 | } |
2175 | assert(NewReg != ResVReg); |
2176 | return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) |
2177 | .addDef(ResVReg) |
2178 | .addUse(NewReg) |
2179 | .constrainAllUses(TII, TRI, RBI); |
2180 | } |
2181 | auto GlobalVar = cast<GlobalVariable>(Val: GV); |
2182 | assert(GlobalVar->getName() != "llvm.global.annotations" ); |
2183 | |
2184 | bool HasInit = GlobalVar->hasInitializer() && |
2185 | !isa<UndefValue>(Val: GlobalVar->getInitializer()); |
2186 | // Skip empty declaration for GVs with initilaizers till we get the decl with |
2187 | // passed initializer. |
2188 | if (HasInit && !Init) |
2189 | return true; |
2190 | |
2191 | unsigned AddrSpace = GV->getAddressSpace(); |
2192 | SPIRV::StorageClass::StorageClass Storage = |
2193 | addressSpaceToStorageClass(AddrSpace, STI); |
2194 | bool HasLnkTy = GV->getLinkage() != GlobalValue::InternalLinkage && |
2195 | Storage != SPIRV::StorageClass::Function; |
2196 | SPIRV::LinkageType::LinkageType LnkType = |
2197 | (GV->isDeclaration() || GV->hasAvailableExternallyLinkage()) |
2198 | ? SPIRV::LinkageType::Import |
2199 | : (GV->getLinkage() == GlobalValue::LinkOnceODRLinkage && |
2200 | STI.canUseExtension(SPIRV::Extension::SPV_KHR_linkonce_odr) |
2201 | ? SPIRV::LinkageType::LinkOnceODR |
2202 | : SPIRV::LinkageType::Export); |
2203 | |
2204 | Register Reg = GR.buildGlobalVariable(ResVReg, ResType, GlobalIdent, GV, |
2205 | Storage, Init, GlobalVar->isConstant(), |
2206 | HasLnkTy, LnkType, MIRBuilder, true); |
2207 | return Reg.isValid(); |
2208 | } |
2209 | |
2210 | bool SPIRVInstructionSelector::selectLog10(Register ResVReg, |
2211 | const SPIRVType *ResType, |
2212 | MachineInstr &I) const { |
2213 | if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) { |
2214 | return selectExtInst(ResVReg, ResType, I, CL::log10); |
2215 | } |
2216 | |
2217 | // There is no log10 instruction in the GLSL Extended Instruction set, so it |
2218 | // is implemented as: |
2219 | // log10(x) = log2(x) * (1 / log2(10)) |
2220 | // = log2(x) * 0.30103 |
2221 | |
2222 | MachineIRBuilder MIRBuilder(I); |
2223 | MachineBasicBlock &BB = *I.getParent(); |
2224 | |
2225 | // Build log2(x). |
2226 | Register VarReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); |
2227 | bool Result = |
2228 | BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) |
2229 | .addDef(VarReg) |
2230 | .addUse(GR.getSPIRVTypeID(ResType)) |
2231 | .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) |
2232 | .addImm(GL::Log2) |
2233 | .add(I.getOperand(1)) |
2234 | .constrainAllUses(TII, TRI, RBI); |
2235 | |
2236 | // Build 0.30103. |
2237 | assert(ResType->getOpcode() == SPIRV::OpTypeVector || |
2238 | ResType->getOpcode() == SPIRV::OpTypeFloat); |
2239 | // TODO: Add matrix implementation once supported by the HLSL frontend. |
2240 | const SPIRVType *SpirvScalarType = |
2241 | ResType->getOpcode() == SPIRV::OpTypeVector |
2242 | ? GR.getSPIRVTypeForVReg(ResType->getOperand(1).getReg()) |
2243 | : ResType; |
2244 | Register ScaleReg = |
2245 | GR.buildConstantFP(Val: APFloat(0.30103f), MIRBuilder, SpvType: SpirvScalarType); |
2246 | |
2247 | // Multiply log2(x) by 0.30103 to get log10(x) result. |
2248 | auto Opcode = ResType->getOpcode() == SPIRV::OpTypeVector |
2249 | ? SPIRV::OpVectorTimesScalar |
2250 | : SPIRV::OpFMulS; |
2251 | Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) |
2252 | .addDef(ResVReg) |
2253 | .addUse(GR.getSPIRVTypeID(SpirvType: ResType)) |
2254 | .addUse(VarReg) |
2255 | .addUse(ScaleReg) |
2256 | .constrainAllUses(TII, TRI, RBI); |
2257 | |
2258 | return Result; |
2259 | } |
2260 | |
2261 | bool SPIRVInstructionSelector::selectSpvThreadId(Register ResVReg, |
2262 | const SPIRVType *ResType, |
2263 | MachineInstr &I) const { |
2264 | // DX intrinsic: @llvm.dx.thread.id(i32) |
2265 | // ID Name Description |
2266 | // 93 ThreadId reads the thread ID |
2267 | |
2268 | MachineIRBuilder MIRBuilder(I); |
2269 | const SPIRVType *U32Type = GR.getOrCreateSPIRVIntegerType(BitWidth: 32, MIRBuilder); |
2270 | const SPIRVType *Vec3Ty = |
2271 | GR.getOrCreateSPIRVVectorType(BaseType: U32Type, NumElements: 3, MIRBuilder); |
2272 | const SPIRVType *PtrType = GR.getOrCreateSPIRVPointerType( |
2273 | Vec3Ty, MIRBuilder, SPIRV::StorageClass::Input); |
2274 | |
2275 | // Create new register for GlobalInvocationID builtin variable. |
2276 | Register NewRegister = |
2277 | MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::IDRegClass); |
2278 | MIRBuilder.getMRI()->setType(VReg: NewRegister, Ty: LLT::pointer(AddressSpace: 0, SizeInBits: 32)); |
2279 | GR.assignSPIRVTypeToVReg(Type: PtrType, VReg: NewRegister, MF&: MIRBuilder.getMF()); |
2280 | |
2281 | // Build GlobalInvocationID global variable with the necessary decorations. |
2282 | Register Variable = GR.buildGlobalVariable( |
2283 | NewRegister, PtrType, |
2284 | getLinkStringForBuiltIn(SPIRV::BuiltIn::GlobalInvocationId), nullptr, |
2285 | SPIRV::StorageClass::Input, nullptr, true, true, |
2286 | SPIRV::LinkageType::Import, MIRBuilder, false); |
2287 | |
2288 | // Create new register for loading value. |
2289 | MachineRegisterInfo *MRI = MIRBuilder.getMRI(); |
2290 | Register LoadedRegister = MRI->createVirtualRegister(&SPIRV::IDRegClass); |
2291 | MIRBuilder.getMRI()->setType(VReg: LoadedRegister, Ty: LLT::pointer(AddressSpace: 0, SizeInBits: 32)); |
2292 | GR.assignSPIRVTypeToVReg(Type: Vec3Ty, VReg: LoadedRegister, MF&: MIRBuilder.getMF()); |
2293 | |
2294 | // Load v3uint value from the global variable. |
2295 | BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) |
2296 | .addDef(LoadedRegister) |
2297 | .addUse(GR.getSPIRVTypeID(Vec3Ty)) |
2298 | .addUse(Variable); |
2299 | |
2300 | // Get Thread ID index. Expecting operand is a constant immediate value, |
2301 | // wrapped in a type assignment. |
2302 | assert(I.getOperand(2).isReg()); |
2303 | Register ThreadIdReg = I.getOperand(i: 2).getReg(); |
2304 | SPIRVType *ConstTy = this->MRI->getVRegDef(Reg: ThreadIdReg); |
2305 | assert(ConstTy && ConstTy->getOpcode() == SPIRV::ASSIGN_TYPE && |
2306 | ConstTy->getOperand(1).isReg()); |
2307 | Register ConstReg = ConstTy->getOperand(i: 1).getReg(); |
2308 | const MachineInstr *Const = this->MRI->getVRegDef(Reg: ConstReg); |
2309 | assert(Const && Const->getOpcode() == TargetOpcode::G_CONSTANT); |
2310 | const llvm::APInt &Val = Const->getOperand(i: 1).getCImm()->getValue(); |
2311 | const uint32_t ThreadId = Val.getZExtValue(); |
2312 | |
2313 | // Extract the thread ID from the loaded vector value. |
2314 | MachineBasicBlock &BB = *I.getParent(); |
2315 | auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) |
2316 | .addDef(ResVReg) |
2317 | .addUse(GR.getSPIRVTypeID(ResType)) |
2318 | .addUse(LoadedRegister) |
2319 | .addImm(ThreadId); |
2320 | return MIB.constrainAllUses(TII, TRI, RBI); |
2321 | } |
2322 | |
2323 | namespace llvm { |
2324 | InstructionSelector * |
2325 | createSPIRVInstructionSelector(const SPIRVTargetMachine &TM, |
2326 | const SPIRVSubtarget &Subtarget, |
2327 | const RegisterBankInfo &RBI) { |
2328 | return new SPIRVInstructionSelector(TM, Subtarget, RBI); |
2329 | } |
2330 | } // namespace llvm |
2331 | |