1 | //===--- SPIRVUtils.cpp ---- SPIR-V Utility Functions -----------*- C++ -*-===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file contains miscellaneous utility functions. |
10 | // |
11 | //===----------------------------------------------------------------------===// |
12 | |
13 | #include "SPIRVUtils.h" |
14 | #include "MCTargetDesc/SPIRVBaseInfo.h" |
15 | #include "SPIRV.h" |
16 | #include "SPIRVInstrInfo.h" |
17 | #include "SPIRVSubtarget.h" |
18 | #include "llvm/ADT/StringRef.h" |
19 | #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" |
20 | #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" |
21 | #include "llvm/CodeGen/MachineInstr.h" |
22 | #include "llvm/CodeGen/MachineInstrBuilder.h" |
23 | #include "llvm/Demangle/Demangle.h" |
24 | #include "llvm/IR/IntrinsicsSPIRV.h" |
25 | |
26 | namespace llvm { |
27 | |
28 | // The following functions are used to add these string literals as a series of |
29 | // 32-bit integer operands with the correct format, and unpack them if necessary |
30 | // when making string comparisons in compiler passes. |
31 | // SPIR-V requires null-terminated UTF-8 strings padded to 32-bit alignment. |
32 | static uint32_t convertCharsToWord(const StringRef &Str, unsigned i) { |
33 | uint32_t Word = 0u; // Build up this 32-bit word from 4 8-bit chars. |
34 | for (unsigned WordIndex = 0; WordIndex < 4; ++WordIndex) { |
35 | unsigned StrIndex = i + WordIndex; |
36 | uint8_t CharToAdd = 0; // Initilize char as padding/null. |
37 | if (StrIndex < Str.size()) { // If it's within the string, get a real char. |
38 | CharToAdd = Str[StrIndex]; |
39 | } |
40 | Word |= (CharToAdd << (WordIndex * 8)); |
41 | } |
42 | return Word; |
43 | } |
44 | |
45 | // Get length including padding and null terminator. |
46 | static size_t getPaddedLen(const StringRef &Str) { |
47 | const size_t Len = Str.size() + 1; |
48 | return (Len % 4 == 0) ? Len : Len + (4 - (Len % 4)); |
49 | } |
50 | |
51 | void addStringImm(const StringRef &Str, MCInst &Inst) { |
52 | const size_t PaddedLen = getPaddedLen(Str); |
53 | for (unsigned i = 0; i < PaddedLen; i += 4) { |
54 | // Add an operand for the 32-bits of chars or padding. |
55 | Inst.addOperand(Op: MCOperand::createImm(Val: convertCharsToWord(Str, i))); |
56 | } |
57 | } |
58 | |
59 | void addStringImm(const StringRef &Str, MachineInstrBuilder &MIB) { |
60 | const size_t PaddedLen = getPaddedLen(Str); |
61 | for (unsigned i = 0; i < PaddedLen; i += 4) { |
62 | // Add an operand for the 32-bits of chars or padding. |
63 | MIB.addImm(Val: convertCharsToWord(Str, i)); |
64 | } |
65 | } |
66 | |
67 | void addStringImm(const StringRef &Str, IRBuilder<> &B, |
68 | std::vector<Value *> &Args) { |
69 | const size_t PaddedLen = getPaddedLen(Str); |
70 | for (unsigned i = 0; i < PaddedLen; i += 4) { |
71 | // Add a vector element for the 32-bits of chars or padding. |
72 | Args.push_back(B.getInt32(convertCharsToWord(Str, i))); |
73 | } |
74 | } |
75 | |
76 | std::string getStringImm(const MachineInstr &MI, unsigned StartIndex) { |
77 | return getSPIRVStringOperand(MI, StartIndex); |
78 | } |
79 | |
80 | void addNumImm(const APInt &Imm, MachineInstrBuilder &MIB) { |
81 | const auto Bitwidth = Imm.getBitWidth(); |
82 | if (Bitwidth == 1) |
83 | return; // Already handled |
84 | else if (Bitwidth <= 32) { |
85 | MIB.addImm(Val: Imm.getZExtValue()); |
86 | // Asm Printer needs this info to print floating-type correctly |
87 | if (Bitwidth == 16) |
88 | MIB.getInstr()->setAsmPrinterFlag(SPIRV::ASM_PRINTER_WIDTH16); |
89 | return; |
90 | } else if (Bitwidth <= 64) { |
91 | uint64_t FullImm = Imm.getZExtValue(); |
92 | uint32_t LowBits = FullImm & 0xffffffff; |
93 | uint32_t HighBits = (FullImm >> 32) & 0xffffffff; |
94 | MIB.addImm(Val: LowBits).addImm(Val: HighBits); |
95 | return; |
96 | } |
97 | report_fatal_error(reason: "Unsupported constant bitwidth" ); |
98 | } |
99 | |
100 | void buildOpName(Register Target, const StringRef &Name, |
101 | MachineIRBuilder &MIRBuilder) { |
102 | if (!Name.empty()) { |
103 | auto MIB = MIRBuilder.buildInstr(SPIRV::OpName).addUse(Target); |
104 | addStringImm(Name, MIB); |
105 | } |
106 | } |
107 | |
108 | static void finishBuildOpDecorate(MachineInstrBuilder &MIB, |
109 | const std::vector<uint32_t> &DecArgs, |
110 | StringRef StrImm) { |
111 | if (!StrImm.empty()) |
112 | addStringImm(Str: StrImm, MIB); |
113 | for (const auto &DecArg : DecArgs) |
114 | MIB.addImm(DecArg); |
115 | } |
116 | |
117 | void buildOpDecorate(Register Reg, MachineIRBuilder &MIRBuilder, |
118 | SPIRV::Decoration::Decoration Dec, |
119 | const std::vector<uint32_t> &DecArgs, StringRef StrImm) { |
120 | auto MIB = MIRBuilder.buildInstr(SPIRV::OpDecorate) |
121 | .addUse(Reg) |
122 | .addImm(static_cast<uint32_t>(Dec)); |
123 | finishBuildOpDecorate(MIB, DecArgs, StrImm); |
124 | } |
125 | |
126 | void buildOpDecorate(Register Reg, MachineInstr &I, const SPIRVInstrInfo &TII, |
127 | SPIRV::Decoration::Decoration Dec, |
128 | const std::vector<uint32_t> &DecArgs, StringRef StrImm) { |
129 | MachineBasicBlock &MBB = *I.getParent(); |
130 | auto MIB = BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpDecorate)) |
131 | .addUse(Reg) |
132 | .addImm(static_cast<uint32_t>(Dec)); |
133 | finishBuildOpDecorate(MIB, DecArgs, StrImm); |
134 | } |
135 | |
136 | // TODO: maybe the following two functions should be handled in the subtarget |
137 | // to allow for different OpenCL vs Vulkan handling. |
138 | unsigned storageClassToAddressSpace(SPIRV::StorageClass::StorageClass SC) { |
139 | switch (SC) { |
140 | case SPIRV::StorageClass::Function: |
141 | return 0; |
142 | case SPIRV::StorageClass::CrossWorkgroup: |
143 | return 1; |
144 | case SPIRV::StorageClass::UniformConstant: |
145 | return 2; |
146 | case SPIRV::StorageClass::Workgroup: |
147 | return 3; |
148 | case SPIRV::StorageClass::Generic: |
149 | return 4; |
150 | case SPIRV::StorageClass::DeviceOnlyINTEL: |
151 | return 5; |
152 | case SPIRV::StorageClass::HostOnlyINTEL: |
153 | return 6; |
154 | case SPIRV::StorageClass::Input: |
155 | return 7; |
156 | default: |
157 | report_fatal_error(reason: "Unable to get address space id" ); |
158 | } |
159 | } |
160 | |
161 | SPIRV::StorageClass::StorageClass |
162 | addressSpaceToStorageClass(unsigned AddrSpace, const SPIRVSubtarget &STI) { |
163 | switch (AddrSpace) { |
164 | case 0: |
165 | return SPIRV::StorageClass::Function; |
166 | case 1: |
167 | return SPIRV::StorageClass::CrossWorkgroup; |
168 | case 2: |
169 | return SPIRV::StorageClass::UniformConstant; |
170 | case 3: |
171 | return SPIRV::StorageClass::Workgroup; |
172 | case 4: |
173 | return SPIRV::StorageClass::Generic; |
174 | case 5: |
175 | return STI.canUseExtension(SPIRV::Extension::SPV_INTEL_usm_storage_classes) |
176 | ? SPIRV::StorageClass::DeviceOnlyINTEL |
177 | : SPIRV::StorageClass::CrossWorkgroup; |
178 | case 6: |
179 | return STI.canUseExtension(SPIRV::Extension::SPV_INTEL_usm_storage_classes) |
180 | ? SPIRV::StorageClass::HostOnlyINTEL |
181 | : SPIRV::StorageClass::CrossWorkgroup; |
182 | case 7: |
183 | return SPIRV::StorageClass::Input; |
184 | default: |
185 | report_fatal_error(reason: "Unknown address space" ); |
186 | } |
187 | } |
188 | |
189 | SPIRV::MemorySemantics::MemorySemantics |
190 | getMemSemanticsForStorageClass(SPIRV::StorageClass::StorageClass SC) { |
191 | switch (SC) { |
192 | case SPIRV::StorageClass::StorageBuffer: |
193 | case SPIRV::StorageClass::Uniform: |
194 | return SPIRV::MemorySemantics::UniformMemory; |
195 | case SPIRV::StorageClass::Workgroup: |
196 | return SPIRV::MemorySemantics::WorkgroupMemory; |
197 | case SPIRV::StorageClass::CrossWorkgroup: |
198 | return SPIRV::MemorySemantics::CrossWorkgroupMemory; |
199 | case SPIRV::StorageClass::AtomicCounter: |
200 | return SPIRV::MemorySemantics::AtomicCounterMemory; |
201 | case SPIRV::StorageClass::Image: |
202 | return SPIRV::MemorySemantics::ImageMemory; |
203 | default: |
204 | return SPIRV::MemorySemantics::None; |
205 | } |
206 | } |
207 | |
208 | SPIRV::MemorySemantics::MemorySemantics getMemSemantics(AtomicOrdering Ord) { |
209 | switch (Ord) { |
210 | case AtomicOrdering::Acquire: |
211 | return SPIRV::MemorySemantics::Acquire; |
212 | case AtomicOrdering::Release: |
213 | return SPIRV::MemorySemantics::Release; |
214 | case AtomicOrdering::AcquireRelease: |
215 | return SPIRV::MemorySemantics::AcquireRelease; |
216 | case AtomicOrdering::SequentiallyConsistent: |
217 | return SPIRV::MemorySemantics::SequentiallyConsistent; |
218 | case AtomicOrdering::Unordered: |
219 | case AtomicOrdering::Monotonic: |
220 | case AtomicOrdering::NotAtomic: |
221 | return SPIRV::MemorySemantics::None; |
222 | } |
223 | llvm_unreachable(nullptr); |
224 | } |
225 | |
226 | MachineInstr *getDefInstrMaybeConstant(Register &ConstReg, |
227 | const MachineRegisterInfo *MRI) { |
228 | MachineInstr *ConstInstr = MRI->getVRegDef(Reg: ConstReg); |
229 | if (auto *GI = dyn_cast<GIntrinsic>(ConstInstr)) { |
230 | if (GI->is(Intrinsic::spv_track_constant)) { |
231 | ConstReg = ConstInstr->getOperand(i: 2).getReg(); |
232 | return MRI->getVRegDef(Reg: ConstReg); |
233 | } |
234 | } else if (ConstInstr->getOpcode() == SPIRV::ASSIGN_TYPE) { |
235 | ConstReg = ConstInstr->getOperand(i: 1).getReg(); |
236 | return MRI->getVRegDef(Reg: ConstReg); |
237 | } |
238 | return MRI->getVRegDef(Reg: ConstReg); |
239 | } |
240 | |
241 | uint64_t getIConstVal(Register ConstReg, const MachineRegisterInfo *MRI) { |
242 | const MachineInstr *MI = getDefInstrMaybeConstant(ConstReg, MRI); |
243 | assert(MI && MI->getOpcode() == TargetOpcode::G_CONSTANT); |
244 | return MI->getOperand(i: 1).getCImm()->getValue().getZExtValue(); |
245 | } |
246 | |
247 | bool isSpvIntrinsic(const MachineInstr &MI, Intrinsic::ID IntrinsicID) { |
248 | if (const auto *GI = dyn_cast<GIntrinsic>(&MI)) |
249 | return GI->is(IntrinsicID); |
250 | return false; |
251 | } |
252 | |
253 | Type *getMDOperandAsType(const MDNode *N, unsigned I) { |
254 | Type *ElementTy = cast<ValueAsMetadata>(N->getOperand(I))->getType(); |
255 | return toTypedPointer(Ty: ElementTy, Ctx&: N->getContext()); |
256 | } |
257 | |
258 | // The set of names is borrowed from the SPIR-V translator. |
259 | // TODO: may be implemented in SPIRVBuiltins.td. |
260 | static bool isPipeOrAddressSpaceCastBI(const StringRef MangledName) { |
261 | return MangledName == "write_pipe_2" || MangledName == "read_pipe_2" || |
262 | MangledName == "write_pipe_2_bl" || MangledName == "read_pipe_2_bl" || |
263 | MangledName == "write_pipe_4" || MangledName == "read_pipe_4" || |
264 | MangledName == "reserve_write_pipe" || |
265 | MangledName == "reserve_read_pipe" || |
266 | MangledName == "commit_write_pipe" || |
267 | MangledName == "commit_read_pipe" || |
268 | MangledName == "work_group_reserve_write_pipe" || |
269 | MangledName == "work_group_reserve_read_pipe" || |
270 | MangledName == "work_group_commit_write_pipe" || |
271 | MangledName == "work_group_commit_read_pipe" || |
272 | MangledName == "get_pipe_num_packets_ro" || |
273 | MangledName == "get_pipe_max_packets_ro" || |
274 | MangledName == "get_pipe_num_packets_wo" || |
275 | MangledName == "get_pipe_max_packets_wo" || |
276 | MangledName == "sub_group_reserve_write_pipe" || |
277 | MangledName == "sub_group_reserve_read_pipe" || |
278 | MangledName == "sub_group_commit_write_pipe" || |
279 | MangledName == "sub_group_commit_read_pipe" || |
280 | MangledName == "to_global" || MangledName == "to_local" || |
281 | MangledName == "to_private" ; |
282 | } |
283 | |
284 | static bool isEnqueueKernelBI(const StringRef MangledName) { |
285 | return MangledName == "__enqueue_kernel_basic" || |
286 | MangledName == "__enqueue_kernel_basic_events" || |
287 | MangledName == "__enqueue_kernel_varargs" || |
288 | MangledName == "__enqueue_kernel_events_varargs" ; |
289 | } |
290 | |
291 | static bool isKernelQueryBI(const StringRef MangledName) { |
292 | return MangledName == "__get_kernel_work_group_size_impl" || |
293 | MangledName == "__get_kernel_sub_group_count_for_ndrange_impl" || |
294 | MangledName == "__get_kernel_max_sub_group_size_for_ndrange_impl" || |
295 | MangledName == "__get_kernel_preferred_work_group_size_multiple_impl" ; |
296 | } |
297 | |
298 | static bool isNonMangledOCLBuiltin(StringRef Name) { |
299 | if (!Name.starts_with(Prefix: "__" )) |
300 | return false; |
301 | |
302 | return isEnqueueKernelBI(MangledName: Name) || isKernelQueryBI(MangledName: Name) || |
303 | isPipeOrAddressSpaceCastBI(MangledName: Name.drop_front(N: 2)) || |
304 | Name == "__translate_sampler_initializer" ; |
305 | } |
306 | |
307 | std::string getOclOrSpirvBuiltinDemangledName(StringRef Name) { |
308 | bool IsNonMangledOCL = isNonMangledOCLBuiltin(Name); |
309 | bool IsNonMangledSPIRV = Name.starts_with(Prefix: "__spirv_" ); |
310 | bool IsNonMangledHLSL = Name.starts_with(Prefix: "__hlsl_" ); |
311 | bool IsMangled = Name.starts_with(Prefix: "_Z" ); |
312 | |
313 | // Otherwise use simple demangling to return the function name. |
314 | if (IsNonMangledOCL || IsNonMangledSPIRV || IsNonMangledHLSL || !IsMangled) |
315 | return Name.str(); |
316 | |
317 | // Try to use the itanium demangler. |
318 | if (char *DemangledName = itaniumDemangle(mangled_name: Name.data())) { |
319 | std::string Result = DemangledName; |
320 | free(ptr: DemangledName); |
321 | return Result; |
322 | } |
323 | |
324 | // Autocheck C++, maybe need to do explicit check of the source language. |
325 | // OpenCL C++ built-ins are declared in cl namespace. |
326 | // TODO: consider using 'St' abbriviation for cl namespace mangling. |
327 | // Similar to ::std:: in C++. |
328 | size_t Start, Len = 0; |
329 | size_t DemangledNameLenStart = 2; |
330 | if (Name.starts_with(Prefix: "_ZN" )) { |
331 | // Skip CV and ref qualifiers. |
332 | size_t NameSpaceStart = Name.find_first_not_of(Chars: "rVKRO" , From: 3); |
333 | // All built-ins are in the ::cl:: namespace. |
334 | if (Name.substr(Start: NameSpaceStart, N: 11) != "2cl7__spirv" ) |
335 | return std::string(); |
336 | DemangledNameLenStart = NameSpaceStart + 11; |
337 | } |
338 | Start = Name.find_first_not_of(Chars: "0123456789" , From: DemangledNameLenStart); |
339 | Name.substr(Start: DemangledNameLenStart, N: Start - DemangledNameLenStart) |
340 | .getAsInteger(10, Len); |
341 | return Name.substr(Start, N: Len).str(); |
342 | } |
343 | |
344 | bool hasBuiltinTypePrefix(StringRef Name) { |
345 | if (Name.starts_with(Prefix: "opencl." ) || Name.starts_with(Prefix: "ocl_" ) || |
346 | Name.starts_with(Prefix: "spirv." )) |
347 | return true; |
348 | return false; |
349 | } |
350 | |
351 | bool isSpecialOpaqueType(const Type *Ty) { |
352 | if (const TargetExtType *EType = dyn_cast<TargetExtType>(Ty)) |
353 | return hasBuiltinTypePrefix(Name: EType->getName()); |
354 | |
355 | return false; |
356 | } |
357 | |
358 | bool isEntryPoint(const Function &F) { |
359 | // OpenCL handling: any function with the SPIR_KERNEL |
360 | // calling convention will be a potential entry point. |
361 | if (F.getCallingConv() == CallingConv::SPIR_KERNEL) |
362 | return true; |
363 | |
364 | // HLSL handling: special attribute are emitted from the |
365 | // front-end. |
366 | if (F.getFnAttribute(Kind: "hlsl.shader" ).isValid()) |
367 | return true; |
368 | |
369 | return false; |
370 | } |
371 | |
372 | Type *parseBasicTypeName(StringRef &TypeName, LLVMContext &Ctx) { |
373 | TypeName.consume_front(Prefix: "atomic_" ); |
374 | if (TypeName.consume_front(Prefix: "void" )) |
375 | return Type::getVoidTy(C&: Ctx); |
376 | else if (TypeName.consume_front(Prefix: "bool" )) |
377 | return Type::getIntNTy(C&: Ctx, N: 1); |
378 | else if (TypeName.consume_front(Prefix: "char" ) || |
379 | TypeName.consume_front(Prefix: "unsigned char" ) || |
380 | TypeName.consume_front(Prefix: "uchar" )) |
381 | return Type::getInt8Ty(C&: Ctx); |
382 | else if (TypeName.consume_front(Prefix: "short" ) || |
383 | TypeName.consume_front(Prefix: "unsigned short" ) || |
384 | TypeName.consume_front(Prefix: "ushort" )) |
385 | return Type::getInt16Ty(C&: Ctx); |
386 | else if (TypeName.consume_front(Prefix: "int" ) || |
387 | TypeName.consume_front(Prefix: "unsigned int" ) || |
388 | TypeName.consume_front(Prefix: "uint" )) |
389 | return Type::getInt32Ty(C&: Ctx); |
390 | else if (TypeName.consume_front(Prefix: "long" ) || |
391 | TypeName.consume_front(Prefix: "unsigned long" ) || |
392 | TypeName.consume_front(Prefix: "ulong" )) |
393 | return Type::getInt64Ty(C&: Ctx); |
394 | else if (TypeName.consume_front(Prefix: "half" )) |
395 | return Type::getHalfTy(C&: Ctx); |
396 | else if (TypeName.consume_front(Prefix: "float" )) |
397 | return Type::getFloatTy(C&: Ctx); |
398 | else if (TypeName.consume_front(Prefix: "double" )) |
399 | return Type::getDoubleTy(C&: Ctx); |
400 | |
401 | // Unable to recognize SPIRV type name |
402 | return nullptr; |
403 | } |
404 | |
405 | } // namespace llvm |
406 | |