1//===-- NVPTXCtorDtorLowering.cpp - Handle global ctors and dtors --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This pass creates a unified init and fini kernel with the required metadata
11//===----------------------------------------------------------------------===//
12
13#include "NVPTXCtorDtorLowering.h"
14#include "MCTargetDesc/NVPTXBaseInfo.h"
15#include "NVPTX.h"
16#include "llvm/ADT/StringExtras.h"
17#include "llvm/IR/Constants.h"
18#include "llvm/IR/Function.h"
19#include "llvm/IR/GlobalVariable.h"
20#include "llvm/IR/IRBuilder.h"
21#include "llvm/IR/Module.h"
22#include "llvm/IR/Value.h"
23#include "llvm/Pass.h"
24#include "llvm/Support/CommandLine.h"
25#include "llvm/Transforms/Utils/ModuleUtils.h"
26
27using namespace llvm;
28
29#define DEBUG_TYPE "nvptx-lower-ctor-dtor"
30
31static cl::opt<std::string>
32 GlobalStr("nvptx-lower-global-ctor-dtor-id",
33 cl::desc("Override unique ID of ctor/dtor globals."),
34 cl::init(Val: ""), cl::Hidden);
35
36static cl::opt<bool>
37 CreateKernels("nvptx-emit-init-fini-kernel",
38 cl::desc("Emit kernels to call ctor/dtor globals."),
39 cl::init(Val: true), cl::Hidden);
40
41namespace {
42
43static std::string getHash(StringRef Str) {
44 llvm::MD5 Hasher;
45 llvm::MD5::MD5Result Hash;
46 Hasher.update(Str);
47 Hasher.final(Result&: Hash);
48 return llvm::utohexstr(X: Hash.low(), /*LowerCase=*/true);
49}
50
51static void addKernelMetadata(Module &M, GlobalValue *GV) {
52 llvm::LLVMContext &Ctx = M.getContext();
53
54 // Get "nvvm.annotations" metadata node.
55 llvm::NamedMDNode *MD = M.getOrInsertNamedMetadata(Name: "nvvm.annotations");
56
57 llvm::Metadata *KernelMDVals[] = {
58 llvm::ConstantAsMetadata::get(C: GV), llvm::MDString::get(Context&: Ctx, Str: "kernel"),
59 llvm::ConstantAsMetadata::get(
60 C: llvm::ConstantInt::get(Ty: llvm::Type::getInt32Ty(C&: Ctx), V: 1))};
61
62 // This kernel is only to be called single-threaded.
63 llvm::Metadata *ThreadXMDVals[] = {
64 llvm::ConstantAsMetadata::get(C: GV), llvm::MDString::get(Context&: Ctx, Str: "maxntidx"),
65 llvm::ConstantAsMetadata::get(
66 C: llvm::ConstantInt::get(Ty: llvm::Type::getInt32Ty(C&: Ctx), V: 1))};
67 llvm::Metadata *ThreadYMDVals[] = {
68 llvm::ConstantAsMetadata::get(C: GV), llvm::MDString::get(Context&: Ctx, Str: "maxntidy"),
69 llvm::ConstantAsMetadata::get(
70 C: llvm::ConstantInt::get(Ty: llvm::Type::getInt32Ty(C&: Ctx), V: 1))};
71 llvm::Metadata *ThreadZMDVals[] = {
72 llvm::ConstantAsMetadata::get(C: GV), llvm::MDString::get(Context&: Ctx, Str: "maxntidz"),
73 llvm::ConstantAsMetadata::get(
74 C: llvm::ConstantInt::get(Ty: llvm::Type::getInt32Ty(C&: Ctx), V: 1))};
75
76 llvm::Metadata *BlockMDVals[] = {
77 llvm::ConstantAsMetadata::get(C: GV),
78 llvm::MDString::get(Context&: Ctx, Str: "maxclusterrank"),
79 llvm::ConstantAsMetadata::get(
80 C: llvm::ConstantInt::get(Ty: llvm::Type::getInt32Ty(C&: Ctx), V: 1))};
81
82 // Append metadata to nvvm.annotations.
83 MD->addOperand(M: llvm::MDNode::get(Context&: Ctx, MDs: KernelMDVals));
84 MD->addOperand(M: llvm::MDNode::get(Context&: Ctx, MDs: ThreadXMDVals));
85 MD->addOperand(M: llvm::MDNode::get(Context&: Ctx, MDs: ThreadYMDVals));
86 MD->addOperand(M: llvm::MDNode::get(Context&: Ctx, MDs: ThreadZMDVals));
87 MD->addOperand(M: llvm::MDNode::get(Context&: Ctx, MDs: BlockMDVals));
88}
89
90static Function *createInitOrFiniKernelFunction(Module &M, bool IsCtor) {
91 StringRef InitOrFiniKernelName =
92 IsCtor ? "nvptx$device$init" : "nvptx$device$fini";
93 if (M.getFunction(Name: InitOrFiniKernelName))
94 return nullptr;
95
96 Function *InitOrFiniKernel = Function::createWithDefaultAttr(
97 Ty: FunctionType::get(Result: Type::getVoidTy(C&: M.getContext()), isVarArg: false),
98 Linkage: GlobalValue::WeakODRLinkage, AddrSpace: 0, N: InitOrFiniKernelName, M: &M);
99 addKernelMetadata(M, GV: InitOrFiniKernel);
100
101 return InitOrFiniKernel;
102}
103
104// We create the IR required to call each callback in this section. This is
105// equivalent to the following code. Normally, the linker would provide us with
106// the definitions of the init and fini array sections. The 'nvlink' linker does
107// not do this so initializing these values is done by the runtime.
108//
109// extern "C" void **__init_array_start = nullptr;
110// extern "C" void **__init_array_end = nullptr;
111// extern "C" void **__fini_array_start = nullptr;
112// extern "C" void **__fini_array_end = nullptr;
113//
114// using InitCallback = void();
115// using FiniCallback = void();
116//
117// void call_init_array_callbacks() {
118// for (auto start = __init_array_start; start != __init_array_end; ++start)
119// reinterpret_cast<InitCallback *>(*start)();
120// }
121//
122// void call_init_array_callbacks() {
123// size_t fini_array_size = __fini_array_end - __fini_array_start;
124// for (size_t i = fini_array_size; i > 0; --i)
125// reinterpret_cast<FiniCallback *>(__fini_array_start[i - 1])();
126// }
127static void createInitOrFiniCalls(Function &F, bool IsCtor) {
128 Module &M = *F.getParent();
129 LLVMContext &C = M.getContext();
130
131 IRBuilder<> IRB(BasicBlock::Create(Context&: C, Name: "entry", Parent: &F));
132 auto *LoopBB = BasicBlock::Create(Context&: C, Name: "while.entry", Parent: &F);
133 auto *ExitBB = BasicBlock::Create(Context&: C, Name: "while.end", Parent: &F);
134 Type *PtrTy = IRB.getPtrTy(AddrSpace: llvm::ADDRESS_SPACE_GLOBAL);
135
136 auto *Begin = M.getOrInsertGlobal(
137 Name: IsCtor ? "__init_array_start" : "__fini_array_start",
138 Ty: PointerType::get(C, AddressSpace: 0), CreateGlobalCallback: [&]() {
139 auto *GV = new GlobalVariable(
140 M, PointerType::get(C, AddressSpace: 0),
141 /*isConstant=*/false, GlobalValue::WeakAnyLinkage,
142 Constant::getNullValue(Ty: PointerType::get(C, AddressSpace: 0)),
143 IsCtor ? "__init_array_start" : "__fini_array_start",
144 /*InsertBefore=*/nullptr, GlobalVariable::NotThreadLocal,
145 /*AddressSpace=*/llvm::ADDRESS_SPACE_GLOBAL);
146 GV->setVisibility(GlobalVariable::ProtectedVisibility);
147 return GV;
148 });
149 auto *End = M.getOrInsertGlobal(
150 Name: IsCtor ? "__init_array_end" : "__fini_array_end", Ty: PointerType::get(C, AddressSpace: 0),
151 CreateGlobalCallback: [&]() {
152 auto *GV = new GlobalVariable(
153 M, PointerType::get(C, AddressSpace: 0),
154 /*isConstant=*/false, GlobalValue::WeakAnyLinkage,
155 Constant::getNullValue(Ty: PointerType::get(C, AddressSpace: 0)),
156 IsCtor ? "__init_array_end" : "__fini_array_end",
157 /*InsertBefore=*/nullptr, GlobalVariable::NotThreadLocal,
158 /*AddressSpace=*/llvm::ADDRESS_SPACE_GLOBAL);
159 GV->setVisibility(GlobalVariable::ProtectedVisibility);
160 return GV;
161 });
162
163 // The constructor type is suppoed to allow using the argument vectors, but
164 // for now we just call them with no arguments.
165 auto *CallBackTy = FunctionType::get(Result: IRB.getVoidTy(), isVarArg: {});
166
167 // The destructor array must be called in reverse order. Get an expression to
168 // the end of the array and iterate backwards in that case.
169 Value *BeginVal = IRB.CreateLoad(Ty: Begin->getType(), Ptr: Begin, Name: "begin");
170 Value *EndVal = IRB.CreateLoad(Ty: Begin->getType(), Ptr: End, Name: "stop");
171 if (!IsCtor) {
172 auto *BeginInt = IRB.CreatePtrToInt(V: BeginVal, DestTy: IntegerType::getInt64Ty(C));
173 auto *EndInt = IRB.CreatePtrToInt(V: EndVal, DestTy: IntegerType::getInt64Ty(C));
174 auto *SubInst = IRB.CreateSub(LHS: EndInt, RHS: BeginInt);
175 auto *Offset = IRB.CreateAShr(
176 LHS: SubInst, RHS: ConstantInt::get(Ty: IntegerType::getInt64Ty(C), V: 3), Name: "offset",
177 /*IsExact=*/isExact: true);
178 auto *ValuePtr = IRB.CreateGEP(Ty: PointerType::get(C, AddressSpace: 0), Ptr: BeginVal,
179 IdxList: ArrayRef<Value *>({Offset}));
180 EndVal = BeginVal;
181 BeginVal = IRB.CreateInBoundsGEP(
182 Ty: PointerType::get(C, AddressSpace: 0), Ptr: ValuePtr,
183 IdxList: ArrayRef<Value *>(ConstantInt::get(Ty: IntegerType::getInt64Ty(C), V: -1)),
184 Name: "start");
185 }
186 IRB.CreateCondBr(
187 Cond: IRB.CreateCmp(Pred: IsCtor ? ICmpInst::ICMP_NE : ICmpInst::ICMP_UGT, LHS: BeginVal,
188 RHS: EndVal),
189 True: LoopBB, False: ExitBB);
190 IRB.SetInsertPoint(LoopBB);
191 auto *CallBackPHI = IRB.CreatePHI(Ty: PtrTy, NumReservedValues: 2, Name: "ptr");
192 auto *CallBack = IRB.CreateLoad(Ty: IRB.getPtrTy(AddrSpace: F.getAddressSpace()),
193 Ptr: CallBackPHI, Name: "callback");
194 IRB.CreateCall(FTy: CallBackTy, Callee: CallBack);
195 auto *NewCallBack =
196 IRB.CreateConstGEP1_64(Ty: PtrTy, Ptr: CallBackPHI, Idx0: IsCtor ? 1 : -1, Name: "next");
197 auto *EndCmp = IRB.CreateCmp(Pred: IsCtor ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_ULT,
198 LHS: NewCallBack, RHS: EndVal, Name: "end");
199 CallBackPHI->addIncoming(V: BeginVal, BB: &F.getEntryBlock());
200 CallBackPHI->addIncoming(V: NewCallBack, BB: LoopBB);
201 IRB.CreateCondBr(Cond: EndCmp, True: ExitBB, False: LoopBB);
202 IRB.SetInsertPoint(ExitBB);
203 IRB.CreateRetVoid();
204}
205
206static bool createInitOrFiniGlobals(Module &M, GlobalVariable *GV,
207 bool IsCtor) {
208 ConstantArray *GA = dyn_cast<ConstantArray>(Val: GV->getInitializer());
209 if (!GA || GA->getNumOperands() == 0)
210 return false;
211
212 // NVPTX has no way to emit variables at specific sections or support for
213 // the traditional constructor sections. Instead, we emit mangled global
214 // names so the runtime can build the list manually.
215 for (Value *V : GA->operands()) {
216 auto *CS = cast<ConstantStruct>(Val: V);
217 auto *F = cast<Constant>(Val: CS->getOperand(i_nocapture: 1));
218 uint64_t Priority = cast<ConstantInt>(Val: CS->getOperand(i_nocapture: 0))->getSExtValue();
219 std::string PriorityStr = "." + std::to_string(val: Priority);
220 // We append a semi-unique hash and the priority to the global name.
221 std::string GlobalID =
222 !GlobalStr.empty() ? GlobalStr : getHash(Str: M.getSourceFileName());
223 std::string NameStr =
224 ((IsCtor ? "__init_array_object_" : "__fini_array_object_") +
225 F->getName() + "_" + GlobalID + "_" + std::to_string(val: Priority))
226 .str();
227 // PTX does not support exported names with '.' in them.
228 llvm::transform(Range&: NameStr, d_first: NameStr.begin(),
229 F: [](char c) { return c == '.' ? '_' : c; });
230
231 auto *GV = new GlobalVariable(M, F->getType(), /*IsConstant=*/true,
232 GlobalValue::ExternalLinkage, F, NameStr,
233 nullptr, GlobalValue::NotThreadLocal,
234 /*AddressSpace=*/4);
235 // This isn't respected by Nvidia, simply put here for clarity.
236 GV->setSection(IsCtor ? ".init_array" + PriorityStr
237 : ".fini_array" + PriorityStr);
238 GV->setVisibility(GlobalVariable::ProtectedVisibility);
239 appendToUsed(M, Values: {GV});
240 }
241
242 return true;
243}
244
245static bool createInitOrFiniKernel(Module &M, StringRef GlobalName,
246 bool IsCtor) {
247 GlobalVariable *GV = M.getGlobalVariable(Name: GlobalName);
248 if (!GV || !GV->hasInitializer())
249 return false;
250
251 if (!createInitOrFiniGlobals(M, GV, IsCtor))
252 return false;
253
254 if (!CreateKernels)
255 return true;
256
257 Function *InitOrFiniKernel = createInitOrFiniKernelFunction(M, IsCtor);
258 if (!InitOrFiniKernel)
259 return false;
260
261 createInitOrFiniCalls(F&: *InitOrFiniKernel, IsCtor);
262
263 GV->eraseFromParent();
264 return true;
265}
266
267static bool lowerCtorsAndDtors(Module &M) {
268 bool Modified = false;
269 Modified |= createInitOrFiniKernel(M, GlobalName: "llvm.global_ctors", /*IsCtor =*/true);
270 Modified |= createInitOrFiniKernel(M, GlobalName: "llvm.global_dtors", /*IsCtor =*/false);
271 return Modified;
272}
273
274class NVPTXCtorDtorLoweringLegacy final : public ModulePass {
275public:
276 static char ID;
277 NVPTXCtorDtorLoweringLegacy() : ModulePass(ID) {}
278 bool runOnModule(Module &M) override { return lowerCtorsAndDtors(M); }
279};
280
281} // End anonymous namespace
282
283PreservedAnalyses NVPTXCtorDtorLoweringPass::run(Module &M,
284 ModuleAnalysisManager &AM) {
285 return lowerCtorsAndDtors(M) ? PreservedAnalyses::none()
286 : PreservedAnalyses::all();
287}
288
289char NVPTXCtorDtorLoweringLegacy::ID = 0;
290char &llvm::NVPTXCtorDtorLoweringLegacyPassID = NVPTXCtorDtorLoweringLegacy::ID;
291INITIALIZE_PASS(NVPTXCtorDtorLoweringLegacy, DEBUG_TYPE,
292 "Lower ctors and dtors for NVPTX", false, false)
293
294ModulePass *llvm::createNVPTXCtorDtorLoweringLegacyPass() {
295 return new NVPTXCtorDtorLoweringLegacy();
296}
297

source code of llvm/lib/Target/NVPTX/NVPTXCtorDtorLowering.cpp