1 | //===- NVPTXUtilities.cpp - Utility Functions -----------------------------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file contains miscellaneous utility functions |
10 | // |
11 | //===----------------------------------------------------------------------===// |
12 | |
13 | #include "NVPTXUtilities.h" |
14 | #include "NVPTX.h" |
15 | #include "NVPTXTargetMachine.h" |
16 | #include "llvm/IR/Constants.h" |
17 | #include "llvm/IR/Function.h" |
18 | #include "llvm/IR/GlobalVariable.h" |
19 | #include "llvm/IR/InstIterator.h" |
20 | #include "llvm/IR/Module.h" |
21 | #include "llvm/IR/Operator.h" |
22 | #include "llvm/Support/Mutex.h" |
23 | #include <algorithm> |
24 | #include <cstring> |
25 | #include <map> |
26 | #include <mutex> |
27 | #include <string> |
28 | #include <vector> |
29 | |
30 | namespace llvm { |
31 | |
32 | namespace { |
33 | typedef std::map<std::string, std::vector<unsigned> > key_val_pair_t; |
34 | typedef std::map<const GlobalValue *, key_val_pair_t> global_val_annot_t; |
35 | |
36 | struct AnnotationCache { |
37 | sys::Mutex Lock; |
38 | std::map<const Module *, global_val_annot_t> Cache; |
39 | }; |
40 | |
41 | AnnotationCache &getAnnotationCache() { |
42 | static AnnotationCache AC; |
43 | return AC; |
44 | } |
45 | } // anonymous namespace |
46 | |
47 | void clearAnnotationCache(const Module *Mod) { |
48 | auto &AC = getAnnotationCache(); |
49 | std::lock_guard<sys::Mutex> Guard(AC.Lock); |
50 | AC.Cache.erase(x: Mod); |
51 | } |
52 | |
53 | static void cacheAnnotationFromMD(const MDNode *md, key_val_pair_t &retval) { |
54 | auto &AC = getAnnotationCache(); |
55 | std::lock_guard<sys::Mutex> Guard(AC.Lock); |
56 | assert(md && "Invalid mdnode for annotation" ); |
57 | assert((md->getNumOperands() % 2) == 1 && "Invalid number of operands" ); |
58 | // start index = 1, to skip the global variable key |
59 | // increment = 2, to skip the value for each property-value pairs |
60 | for (unsigned i = 1, e = md->getNumOperands(); i != e; i += 2) { |
61 | // property |
62 | const MDString *prop = dyn_cast<MDString>(Val: md->getOperand(I: i)); |
63 | assert(prop && "Annotation property not a string" ); |
64 | |
65 | // value |
66 | ConstantInt *Val = mdconst::dyn_extract<ConstantInt>(MD: md->getOperand(I: i + 1)); |
67 | assert(Val && "Value operand not a constant int" ); |
68 | |
69 | std::string keyname = prop->getString().str(); |
70 | if (retval.find(x: keyname) != retval.end()) |
71 | retval[keyname].push_back(x: Val->getZExtValue()); |
72 | else { |
73 | std::vector<unsigned> tmp; |
74 | tmp.push_back(x: Val->getZExtValue()); |
75 | retval[keyname] = tmp; |
76 | } |
77 | } |
78 | } |
79 | |
80 | static void cacheAnnotationFromMD(const Module *m, const GlobalValue *gv) { |
81 | auto &AC = getAnnotationCache(); |
82 | std::lock_guard<sys::Mutex> Guard(AC.Lock); |
83 | NamedMDNode *NMD = m->getNamedMetadata(Name: "nvvm.annotations" ); |
84 | if (!NMD) |
85 | return; |
86 | key_val_pair_t tmp; |
87 | for (unsigned i = 0, e = NMD->getNumOperands(); i != e; ++i) { |
88 | const MDNode *elem = NMD->getOperand(i); |
89 | |
90 | GlobalValue *entity = |
91 | mdconst::dyn_extract_or_null<GlobalValue>(MD: elem->getOperand(I: 0)); |
92 | // entity may be null due to DCE |
93 | if (!entity) |
94 | continue; |
95 | if (entity != gv) |
96 | continue; |
97 | |
98 | // accumulate annotations for entity in tmp |
99 | cacheAnnotationFromMD(md: elem, retval&: tmp); |
100 | } |
101 | |
102 | if (tmp.empty()) // no annotations for this gv |
103 | return; |
104 | |
105 | if (AC.Cache.find(x: m) != AC.Cache.end()) |
106 | AC.Cache[m][gv] = std::move(tmp); |
107 | else { |
108 | global_val_annot_t tmp1; |
109 | tmp1[gv] = std::move(tmp); |
110 | AC.Cache[m] = std::move(tmp1); |
111 | } |
112 | } |
113 | |
114 | bool findOneNVVMAnnotation(const GlobalValue *gv, const std::string &prop, |
115 | unsigned &retval) { |
116 | auto &AC = getAnnotationCache(); |
117 | std::lock_guard<sys::Mutex> Guard(AC.Lock); |
118 | const Module *m = gv->getParent(); |
119 | if (AC.Cache.find(x: m) == AC.Cache.end()) |
120 | cacheAnnotationFromMD(m, gv); |
121 | else if (AC.Cache[m].find(x: gv) == AC.Cache[m].end()) |
122 | cacheAnnotationFromMD(m, gv); |
123 | if (AC.Cache[m][gv].find(x: prop) == AC.Cache[m][gv].end()) |
124 | return false; |
125 | retval = AC.Cache[m][gv][prop][0]; |
126 | return true; |
127 | } |
128 | |
129 | bool findAllNVVMAnnotation(const GlobalValue *gv, const std::string &prop, |
130 | std::vector<unsigned> &retval) { |
131 | auto &AC = getAnnotationCache(); |
132 | std::lock_guard<sys::Mutex> Guard(AC.Lock); |
133 | const Module *m = gv->getParent(); |
134 | if (AC.Cache.find(x: m) == AC.Cache.end()) |
135 | cacheAnnotationFromMD(m, gv); |
136 | else if (AC.Cache[m].find(x: gv) == AC.Cache[m].end()) |
137 | cacheAnnotationFromMD(m, gv); |
138 | if (AC.Cache[m][gv].find(x: prop) == AC.Cache[m][gv].end()) |
139 | return false; |
140 | retval = AC.Cache[m][gv][prop]; |
141 | return true; |
142 | } |
143 | |
144 | bool isTexture(const Value &val) { |
145 | if (const GlobalValue *gv = dyn_cast<GlobalValue>(Val: &val)) { |
146 | unsigned annot; |
147 | if (findOneNVVMAnnotation(gv, prop: "texture" , retval&: annot)) { |
148 | assert((annot == 1) && "Unexpected annotation on a texture symbol" ); |
149 | return true; |
150 | } |
151 | } |
152 | return false; |
153 | } |
154 | |
155 | bool isSurface(const Value &val) { |
156 | if (const GlobalValue *gv = dyn_cast<GlobalValue>(Val: &val)) { |
157 | unsigned annot; |
158 | if (findOneNVVMAnnotation(gv, prop: "surface" , retval&: annot)) { |
159 | assert((annot == 1) && "Unexpected annotation on a surface symbol" ); |
160 | return true; |
161 | } |
162 | } |
163 | return false; |
164 | } |
165 | |
166 | bool isSampler(const Value &val) { |
167 | const char *AnnotationName = "sampler" ; |
168 | |
169 | if (const GlobalValue *gv = dyn_cast<GlobalValue>(Val: &val)) { |
170 | unsigned annot; |
171 | if (findOneNVVMAnnotation(gv, prop: AnnotationName, retval&: annot)) { |
172 | assert((annot == 1) && "Unexpected annotation on a sampler symbol" ); |
173 | return true; |
174 | } |
175 | } |
176 | if (const Argument *arg = dyn_cast<Argument>(Val: &val)) { |
177 | const Function *func = arg->getParent(); |
178 | std::vector<unsigned> annot; |
179 | if (findAllNVVMAnnotation(gv: func, prop: AnnotationName, retval&: annot)) { |
180 | if (is_contained(Range&: annot, Element: arg->getArgNo())) |
181 | return true; |
182 | } |
183 | } |
184 | return false; |
185 | } |
186 | |
187 | bool isImageReadOnly(const Value &val) { |
188 | if (const Argument *arg = dyn_cast<Argument>(Val: &val)) { |
189 | const Function *func = arg->getParent(); |
190 | std::vector<unsigned> annot; |
191 | if (findAllNVVMAnnotation(gv: func, prop: "rdoimage" , retval&: annot)) { |
192 | if (is_contained(Range&: annot, Element: arg->getArgNo())) |
193 | return true; |
194 | } |
195 | } |
196 | return false; |
197 | } |
198 | |
199 | bool isImageWriteOnly(const Value &val) { |
200 | if (const Argument *arg = dyn_cast<Argument>(Val: &val)) { |
201 | const Function *func = arg->getParent(); |
202 | std::vector<unsigned> annot; |
203 | if (findAllNVVMAnnotation(gv: func, prop: "wroimage" , retval&: annot)) { |
204 | if (is_contained(Range&: annot, Element: arg->getArgNo())) |
205 | return true; |
206 | } |
207 | } |
208 | return false; |
209 | } |
210 | |
211 | bool isImageReadWrite(const Value &val) { |
212 | if (const Argument *arg = dyn_cast<Argument>(Val: &val)) { |
213 | const Function *func = arg->getParent(); |
214 | std::vector<unsigned> annot; |
215 | if (findAllNVVMAnnotation(gv: func, prop: "rdwrimage" , retval&: annot)) { |
216 | if (is_contained(Range&: annot, Element: arg->getArgNo())) |
217 | return true; |
218 | } |
219 | } |
220 | return false; |
221 | } |
222 | |
223 | bool isImage(const Value &val) { |
224 | return isImageReadOnly(val) || isImageWriteOnly(val) || isImageReadWrite(val); |
225 | } |
226 | |
227 | bool isManaged(const Value &val) { |
228 | if(const GlobalValue *gv = dyn_cast<GlobalValue>(Val: &val)) { |
229 | unsigned annot; |
230 | if (findOneNVVMAnnotation(gv, prop: "managed" , retval&: annot)) { |
231 | assert((annot == 1) && "Unexpected annotation on a managed symbol" ); |
232 | return true; |
233 | } |
234 | } |
235 | return false; |
236 | } |
237 | |
238 | std::string getTextureName(const Value &val) { |
239 | assert(val.hasName() && "Found texture variable with no name" ); |
240 | return std::string(val.getName()); |
241 | } |
242 | |
243 | std::string getSurfaceName(const Value &val) { |
244 | assert(val.hasName() && "Found surface variable with no name" ); |
245 | return std::string(val.getName()); |
246 | } |
247 | |
248 | std::string getSamplerName(const Value &val) { |
249 | assert(val.hasName() && "Found sampler variable with no name" ); |
250 | return std::string(val.getName()); |
251 | } |
252 | |
253 | bool getMaxNTIDx(const Function &F, unsigned &x) { |
254 | return findOneNVVMAnnotation(gv: &F, prop: "maxntidx" , retval&: x); |
255 | } |
256 | |
257 | bool getMaxNTIDy(const Function &F, unsigned &y) { |
258 | return findOneNVVMAnnotation(gv: &F, prop: "maxntidy" , retval&: y); |
259 | } |
260 | |
261 | bool getMaxNTIDz(const Function &F, unsigned &z) { |
262 | return findOneNVVMAnnotation(gv: &F, prop: "maxntidz" , retval&: z); |
263 | } |
264 | |
265 | bool getMaxClusterRank(const Function &F, unsigned &x) { |
266 | return findOneNVVMAnnotation(gv: &F, prop: "maxclusterrank" , retval&: x); |
267 | } |
268 | |
269 | bool getReqNTIDx(const Function &F, unsigned &x) { |
270 | return findOneNVVMAnnotation(gv: &F, prop: "reqntidx" , retval&: x); |
271 | } |
272 | |
273 | bool getReqNTIDy(const Function &F, unsigned &y) { |
274 | return findOneNVVMAnnotation(gv: &F, prop: "reqntidy" , retval&: y); |
275 | } |
276 | |
277 | bool getReqNTIDz(const Function &F, unsigned &z) { |
278 | return findOneNVVMAnnotation(gv: &F, prop: "reqntidz" , retval&: z); |
279 | } |
280 | |
281 | bool getMinCTASm(const Function &F, unsigned &x) { |
282 | return findOneNVVMAnnotation(gv: &F, prop: "minctasm" , retval&: x); |
283 | } |
284 | |
285 | bool getMaxNReg(const Function &F, unsigned &x) { |
286 | return findOneNVVMAnnotation(gv: &F, prop: "maxnreg" , retval&: x); |
287 | } |
288 | |
289 | bool isKernelFunction(const Function &F) { |
290 | unsigned x = 0; |
291 | bool retval = findOneNVVMAnnotation(gv: &F, prop: "kernel" , retval&: x); |
292 | if (!retval) { |
293 | // There is no NVVM metadata, check the calling convention |
294 | return F.getCallingConv() == CallingConv::PTX_Kernel; |
295 | } |
296 | return (x == 1); |
297 | } |
298 | |
299 | bool getAlign(const Function &F, unsigned index, unsigned &align) { |
300 | std::vector<unsigned> Vs; |
301 | bool retval = findAllNVVMAnnotation(gv: &F, prop: "align" , retval&: Vs); |
302 | if (!retval) |
303 | return false; |
304 | for (unsigned v : Vs) { |
305 | if ((v >> 16) == index) { |
306 | align = v & 0xFFFF; |
307 | return true; |
308 | } |
309 | } |
310 | return false; |
311 | } |
312 | |
313 | bool getAlign(const CallInst &I, unsigned index, unsigned &align) { |
314 | if (MDNode *alignNode = I.getMetadata(Kind: "callalign" )) { |
315 | for (int i = 0, n = alignNode->getNumOperands(); i < n; i++) { |
316 | if (const ConstantInt *CI = |
317 | mdconst::dyn_extract<ConstantInt>(MD: alignNode->getOperand(I: i))) { |
318 | unsigned v = CI->getZExtValue(); |
319 | if ((v >> 16) == index) { |
320 | align = v & 0xFFFF; |
321 | return true; |
322 | } |
323 | if ((v >> 16) > index) { |
324 | return false; |
325 | } |
326 | } |
327 | } |
328 | } |
329 | return false; |
330 | } |
331 | |
332 | Function *getMaybeBitcastedCallee(const CallBase *CB) { |
333 | return dyn_cast<Function>(Val: CB->getCalledOperand()->stripPointerCasts()); |
334 | } |
335 | |
336 | bool shouldEmitPTXNoReturn(const Value *V, const TargetMachine &TM) { |
337 | const auto &ST = |
338 | *static_cast<const NVPTXTargetMachine &>(TM).getSubtargetImpl(); |
339 | if (!ST.hasNoReturn()) |
340 | return false; |
341 | |
342 | assert((isa<Function>(V) || isa<CallInst>(V)) && |
343 | "Expect either a call instruction or a function" ); |
344 | |
345 | if (const CallInst *CallI = dyn_cast<CallInst>(Val: V)) |
346 | return CallI->doesNotReturn() && |
347 | CallI->getFunctionType()->getReturnType()->isVoidTy(); |
348 | |
349 | const Function *F = cast<Function>(Val: V); |
350 | return F->doesNotReturn() && |
351 | F->getFunctionType()->getReturnType()->isVoidTy() && |
352 | !isKernelFunction(F: *F); |
353 | } |
354 | |
355 | bool Isv2x16VT(EVT VT) { |
356 | return (VT == MVT::v2f16 || VT == MVT::v2bf16 || VT == MVT::v2i16); |
357 | } |
358 | |
359 | } // namespace llvm |
360 | |