1 | //===-- NVPTXCtorDtorLowering.cpp - Handle global ctors and dtors --------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | /// |
9 | /// \file |
10 | /// This pass creates a unified init and fini kernel with the required metadata |
11 | //===----------------------------------------------------------------------===// |
12 | |
13 | #include "NVPTXCtorDtorLowering.h" |
14 | #include "MCTargetDesc/NVPTXBaseInfo.h" |
15 | #include "NVPTX.h" |
16 | #include "llvm/ADT/StringExtras.h" |
17 | #include "llvm/IR/Constants.h" |
18 | #include "llvm/IR/Function.h" |
19 | #include "llvm/IR/GlobalVariable.h" |
20 | #include "llvm/IR/IRBuilder.h" |
21 | #include "llvm/IR/Module.h" |
22 | #include "llvm/IR/Value.h" |
23 | #include "llvm/Pass.h" |
24 | #include "llvm/Support/CommandLine.h" |
25 | #include "llvm/Transforms/Utils/ModuleUtils.h" |
26 | |
27 | using namespace llvm; |
28 | |
29 | #define DEBUG_TYPE "nvptx-lower-ctor-dtor" |
30 | |
31 | static cl::opt<std::string> |
32 | GlobalStr("nvptx-lower-global-ctor-dtor-id" , |
33 | cl::desc("Override unique ID of ctor/dtor globals." ), |
34 | cl::init(Val: "" ), cl::Hidden); |
35 | |
36 | static cl::opt<bool> |
37 | CreateKernels("nvptx-emit-init-fini-kernel" , |
38 | cl::desc("Emit kernels to call ctor/dtor globals." ), |
39 | cl::init(Val: true), cl::Hidden); |
40 | |
41 | namespace { |
42 | |
43 | static std::string getHash(StringRef Str) { |
44 | llvm::MD5 Hasher; |
45 | llvm::MD5::MD5Result Hash; |
46 | Hasher.update(Str); |
47 | Hasher.final(Result&: Hash); |
48 | return llvm::utohexstr(X: Hash.low(), /*LowerCase=*/true); |
49 | } |
50 | |
51 | static void addKernelMetadata(Module &M, GlobalValue *GV) { |
52 | llvm::LLVMContext &Ctx = M.getContext(); |
53 | |
54 | // Get "nvvm.annotations" metadata node. |
55 | llvm::NamedMDNode *MD = M.getOrInsertNamedMetadata(Name: "nvvm.annotations" ); |
56 | |
57 | llvm::Metadata *KernelMDVals[] = { |
58 | llvm::ConstantAsMetadata::get(C: GV), llvm::MDString::get(Context&: Ctx, Str: "kernel" ), |
59 | llvm::ConstantAsMetadata::get( |
60 | C: llvm::ConstantInt::get(Ty: llvm::Type::getInt32Ty(C&: Ctx), V: 1))}; |
61 | |
62 | // This kernel is only to be called single-threaded. |
63 | llvm::Metadata *ThreadXMDVals[] = { |
64 | llvm::ConstantAsMetadata::get(C: GV), llvm::MDString::get(Context&: Ctx, Str: "maxntidx" ), |
65 | llvm::ConstantAsMetadata::get( |
66 | C: llvm::ConstantInt::get(Ty: llvm::Type::getInt32Ty(C&: Ctx), V: 1))}; |
67 | llvm::Metadata *ThreadYMDVals[] = { |
68 | llvm::ConstantAsMetadata::get(C: GV), llvm::MDString::get(Context&: Ctx, Str: "maxntidy" ), |
69 | llvm::ConstantAsMetadata::get( |
70 | C: llvm::ConstantInt::get(Ty: llvm::Type::getInt32Ty(C&: Ctx), V: 1))}; |
71 | llvm::Metadata *ThreadZMDVals[] = { |
72 | llvm::ConstantAsMetadata::get(C: GV), llvm::MDString::get(Context&: Ctx, Str: "maxntidz" ), |
73 | llvm::ConstantAsMetadata::get( |
74 | C: llvm::ConstantInt::get(Ty: llvm::Type::getInt32Ty(C&: Ctx), V: 1))}; |
75 | |
76 | llvm::Metadata *BlockMDVals[] = { |
77 | llvm::ConstantAsMetadata::get(C: GV), |
78 | llvm::MDString::get(Context&: Ctx, Str: "maxclusterrank" ), |
79 | llvm::ConstantAsMetadata::get( |
80 | C: llvm::ConstantInt::get(Ty: llvm::Type::getInt32Ty(C&: Ctx), V: 1))}; |
81 | |
82 | // Append metadata to nvvm.annotations. |
83 | MD->addOperand(M: llvm::MDNode::get(Context&: Ctx, MDs: KernelMDVals)); |
84 | MD->addOperand(M: llvm::MDNode::get(Context&: Ctx, MDs: ThreadXMDVals)); |
85 | MD->addOperand(M: llvm::MDNode::get(Context&: Ctx, MDs: ThreadYMDVals)); |
86 | MD->addOperand(M: llvm::MDNode::get(Context&: Ctx, MDs: ThreadZMDVals)); |
87 | MD->addOperand(M: llvm::MDNode::get(Context&: Ctx, MDs: BlockMDVals)); |
88 | } |
89 | |
90 | static Function *createInitOrFiniKernelFunction(Module &M, bool IsCtor) { |
91 | StringRef InitOrFiniKernelName = |
92 | IsCtor ? "nvptx$device$init" : "nvptx$device$fini" ; |
93 | if (M.getFunction(Name: InitOrFiniKernelName)) |
94 | return nullptr; |
95 | |
96 | Function *InitOrFiniKernel = Function::createWithDefaultAttr( |
97 | Ty: FunctionType::get(Result: Type::getVoidTy(C&: M.getContext()), isVarArg: false), |
98 | Linkage: GlobalValue::WeakODRLinkage, AddrSpace: 0, N: InitOrFiniKernelName, M: &M); |
99 | addKernelMetadata(M, GV: InitOrFiniKernel); |
100 | |
101 | return InitOrFiniKernel; |
102 | } |
103 | |
104 | // We create the IR required to call each callback in this section. This is |
105 | // equivalent to the following code. Normally, the linker would provide us with |
106 | // the definitions of the init and fini array sections. The 'nvlink' linker does |
107 | // not do this so initializing these values is done by the runtime. |
108 | // |
109 | // extern "C" void **__init_array_start = nullptr; |
110 | // extern "C" void **__init_array_end = nullptr; |
111 | // extern "C" void **__fini_array_start = nullptr; |
112 | // extern "C" void **__fini_array_end = nullptr; |
113 | // |
114 | // using InitCallback = void(); |
115 | // using FiniCallback = void(); |
116 | // |
117 | // void call_init_array_callbacks() { |
118 | // for (auto start = __init_array_start; start != __init_array_end; ++start) |
119 | // reinterpret_cast<InitCallback *>(*start)(); |
120 | // } |
121 | // |
122 | // void call_init_array_callbacks() { |
123 | // size_t fini_array_size = __fini_array_end - __fini_array_start; |
124 | // for (size_t i = fini_array_size; i > 0; --i) |
125 | // reinterpret_cast<FiniCallback *>(__fini_array_start[i - 1])(); |
126 | // } |
127 | static void createInitOrFiniCalls(Function &F, bool IsCtor) { |
128 | Module &M = *F.getParent(); |
129 | LLVMContext &C = M.getContext(); |
130 | |
131 | IRBuilder<> IRB(BasicBlock::Create(Context&: C, Name: "entry" , Parent: &F)); |
132 | auto *LoopBB = BasicBlock::Create(Context&: C, Name: "while.entry" , Parent: &F); |
133 | auto *ExitBB = BasicBlock::Create(Context&: C, Name: "while.end" , Parent: &F); |
134 | Type *PtrTy = IRB.getPtrTy(AddrSpace: llvm::ADDRESS_SPACE_GLOBAL); |
135 | |
136 | auto *Begin = M.getOrInsertGlobal( |
137 | Name: IsCtor ? "__init_array_start" : "__fini_array_start" , |
138 | Ty: PointerType::get(C, AddressSpace: 0), CreateGlobalCallback: [&]() { |
139 | auto *GV = new GlobalVariable( |
140 | M, PointerType::get(C, AddressSpace: 0), |
141 | /*isConstant=*/false, GlobalValue::WeakAnyLinkage, |
142 | Constant::getNullValue(Ty: PointerType::get(C, AddressSpace: 0)), |
143 | IsCtor ? "__init_array_start" : "__fini_array_start" , |
144 | /*InsertBefore=*/nullptr, GlobalVariable::NotThreadLocal, |
145 | /*AddressSpace=*/llvm::ADDRESS_SPACE_GLOBAL); |
146 | GV->setVisibility(GlobalVariable::ProtectedVisibility); |
147 | return GV; |
148 | }); |
149 | auto *End = M.getOrInsertGlobal( |
150 | Name: IsCtor ? "__init_array_end" : "__fini_array_end" , Ty: PointerType::get(C, AddressSpace: 0), |
151 | CreateGlobalCallback: [&]() { |
152 | auto *GV = new GlobalVariable( |
153 | M, PointerType::get(C, AddressSpace: 0), |
154 | /*isConstant=*/false, GlobalValue::WeakAnyLinkage, |
155 | Constant::getNullValue(Ty: PointerType::get(C, AddressSpace: 0)), |
156 | IsCtor ? "__init_array_end" : "__fini_array_end" , |
157 | /*InsertBefore=*/nullptr, GlobalVariable::NotThreadLocal, |
158 | /*AddressSpace=*/llvm::ADDRESS_SPACE_GLOBAL); |
159 | GV->setVisibility(GlobalVariable::ProtectedVisibility); |
160 | return GV; |
161 | }); |
162 | |
163 | // The constructor type is suppoed to allow using the argument vectors, but |
164 | // for now we just call them with no arguments. |
165 | auto *CallBackTy = FunctionType::get(Result: IRB.getVoidTy(), isVarArg: {}); |
166 | |
167 | // The destructor array must be called in reverse order. Get an expression to |
168 | // the end of the array and iterate backwards in that case. |
169 | Value *BeginVal = IRB.CreateLoad(Ty: Begin->getType(), Ptr: Begin, Name: "begin" ); |
170 | Value *EndVal = IRB.CreateLoad(Ty: Begin->getType(), Ptr: End, Name: "stop" ); |
171 | if (!IsCtor) { |
172 | auto *BeginInt = IRB.CreatePtrToInt(V: BeginVal, DestTy: IntegerType::getInt64Ty(C)); |
173 | auto *EndInt = IRB.CreatePtrToInt(V: EndVal, DestTy: IntegerType::getInt64Ty(C)); |
174 | auto *SubInst = IRB.CreateSub(LHS: EndInt, RHS: BeginInt); |
175 | auto *Offset = IRB.CreateAShr( |
176 | LHS: SubInst, RHS: ConstantInt::get(Ty: IntegerType::getInt64Ty(C), V: 3), Name: "offset" , |
177 | /*IsExact=*/isExact: true); |
178 | auto *ValuePtr = IRB.CreateGEP(Ty: PointerType::get(C, AddressSpace: 0), Ptr: BeginVal, |
179 | IdxList: ArrayRef<Value *>({Offset})); |
180 | EndVal = BeginVal; |
181 | BeginVal = IRB.CreateInBoundsGEP( |
182 | Ty: PointerType::get(C, AddressSpace: 0), Ptr: ValuePtr, |
183 | IdxList: ArrayRef<Value *>(ConstantInt::get(Ty: IntegerType::getInt64Ty(C), V: -1)), |
184 | Name: "start" ); |
185 | } |
186 | IRB.CreateCondBr( |
187 | Cond: IRB.CreateCmp(Pred: IsCtor ? ICmpInst::ICMP_NE : ICmpInst::ICMP_UGT, LHS: BeginVal, |
188 | RHS: EndVal), |
189 | True: LoopBB, False: ExitBB); |
190 | IRB.SetInsertPoint(LoopBB); |
191 | auto *CallBackPHI = IRB.CreatePHI(Ty: PtrTy, NumReservedValues: 2, Name: "ptr" ); |
192 | auto *CallBack = IRB.CreateLoad(Ty: IRB.getPtrTy(AddrSpace: F.getAddressSpace()), |
193 | Ptr: CallBackPHI, Name: "callback" ); |
194 | IRB.CreateCall(FTy: CallBackTy, Callee: CallBack); |
195 | auto *NewCallBack = |
196 | IRB.CreateConstGEP1_64(Ty: PtrTy, Ptr: CallBackPHI, Idx0: IsCtor ? 1 : -1, Name: "next" ); |
197 | auto *EndCmp = IRB.CreateCmp(Pred: IsCtor ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_ULT, |
198 | LHS: NewCallBack, RHS: EndVal, Name: "end" ); |
199 | CallBackPHI->addIncoming(V: BeginVal, BB: &F.getEntryBlock()); |
200 | CallBackPHI->addIncoming(V: NewCallBack, BB: LoopBB); |
201 | IRB.CreateCondBr(Cond: EndCmp, True: ExitBB, False: LoopBB); |
202 | IRB.SetInsertPoint(ExitBB); |
203 | IRB.CreateRetVoid(); |
204 | } |
205 | |
206 | static bool createInitOrFiniGlobals(Module &M, GlobalVariable *GV, |
207 | bool IsCtor) { |
208 | ConstantArray *GA = dyn_cast<ConstantArray>(Val: GV->getInitializer()); |
209 | if (!GA || GA->getNumOperands() == 0) |
210 | return false; |
211 | |
212 | // NVPTX has no way to emit variables at specific sections or support for |
213 | // the traditional constructor sections. Instead, we emit mangled global |
214 | // names so the runtime can build the list manually. |
215 | for (Value *V : GA->operands()) { |
216 | auto *CS = cast<ConstantStruct>(Val: V); |
217 | auto *F = cast<Constant>(Val: CS->getOperand(i_nocapture: 1)); |
218 | uint64_t Priority = cast<ConstantInt>(Val: CS->getOperand(i_nocapture: 0))->getSExtValue(); |
219 | std::string PriorityStr = "." + std::to_string(val: Priority); |
220 | // We append a semi-unique hash and the priority to the global name. |
221 | std::string GlobalID = |
222 | !GlobalStr.empty() ? GlobalStr : getHash(Str: M.getSourceFileName()); |
223 | std::string NameStr = |
224 | ((IsCtor ? "__init_array_object_" : "__fini_array_object_" ) + |
225 | F->getName() + "_" + GlobalID + "_" + std::to_string(val: Priority)) |
226 | .str(); |
227 | // PTX does not support exported names with '.' in them. |
228 | llvm::transform(Range&: NameStr, d_first: NameStr.begin(), |
229 | F: [](char c) { return c == '.' ? '_' : c; }); |
230 | |
231 | auto *GV = new GlobalVariable(M, F->getType(), /*IsConstant=*/true, |
232 | GlobalValue::ExternalLinkage, F, NameStr, |
233 | nullptr, GlobalValue::NotThreadLocal, |
234 | /*AddressSpace=*/4); |
235 | // This isn't respected by Nvidia, simply put here for clarity. |
236 | GV->setSection(IsCtor ? ".init_array" + PriorityStr |
237 | : ".fini_array" + PriorityStr); |
238 | GV->setVisibility(GlobalVariable::ProtectedVisibility); |
239 | appendToUsed(M, Values: {GV}); |
240 | } |
241 | |
242 | return true; |
243 | } |
244 | |
245 | static bool createInitOrFiniKernel(Module &M, StringRef GlobalName, |
246 | bool IsCtor) { |
247 | GlobalVariable *GV = M.getGlobalVariable(Name: GlobalName); |
248 | if (!GV || !GV->hasInitializer()) |
249 | return false; |
250 | |
251 | if (!createInitOrFiniGlobals(M, GV, IsCtor)) |
252 | return false; |
253 | |
254 | if (!CreateKernels) |
255 | return true; |
256 | |
257 | Function *InitOrFiniKernel = createInitOrFiniKernelFunction(M, IsCtor); |
258 | if (!InitOrFiniKernel) |
259 | return false; |
260 | |
261 | createInitOrFiniCalls(F&: *InitOrFiniKernel, IsCtor); |
262 | |
263 | GV->eraseFromParent(); |
264 | return true; |
265 | } |
266 | |
267 | static bool lowerCtorsAndDtors(Module &M) { |
268 | bool Modified = false; |
269 | Modified |= createInitOrFiniKernel(M, GlobalName: "llvm.global_ctors" , /*IsCtor =*/true); |
270 | Modified |= createInitOrFiniKernel(M, GlobalName: "llvm.global_dtors" , /*IsCtor =*/false); |
271 | return Modified; |
272 | } |
273 | |
274 | class NVPTXCtorDtorLoweringLegacy final : public ModulePass { |
275 | public: |
276 | static char ID; |
277 | NVPTXCtorDtorLoweringLegacy() : ModulePass(ID) {} |
278 | bool runOnModule(Module &M) override { return lowerCtorsAndDtors(M); } |
279 | }; |
280 | |
281 | } // End anonymous namespace |
282 | |
283 | PreservedAnalyses NVPTXCtorDtorLoweringPass::run(Module &M, |
284 | ModuleAnalysisManager &AM) { |
285 | return lowerCtorsAndDtors(M) ? PreservedAnalyses::none() |
286 | : PreservedAnalyses::all(); |
287 | } |
288 | |
289 | char NVPTXCtorDtorLoweringLegacy::ID = 0; |
290 | char &llvm::NVPTXCtorDtorLoweringLegacyPassID = NVPTXCtorDtorLoweringLegacy::ID; |
291 | INITIALIZE_PASS(NVPTXCtorDtorLoweringLegacy, DEBUG_TYPE, |
292 | "Lower ctors and dtors for NVPTX" , false, false) |
293 | |
294 | ModulePass *llvm::createNVPTXCtorDtorLoweringLegacyPass() { |
295 | return new NVPTXCtorDtorLoweringLegacy(); |
296 | } |
297 | |