1//===-- PluginManager.cpp - Plugin loading and communication API ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Functionality for handling plugins.
10//
11//===----------------------------------------------------------------------===//
12
13#include "PluginManager.h"
14#include "Shared/Debug.h"
15#include "Shared/Profile.h"
16
17#include "llvm/Support/Error.h"
18#include "llvm/Support/ErrorHandling.h"
19#include <memory>
20
21using namespace llvm;
22using namespace llvm::sys;
23
24PluginManager *PM = nullptr;
25
26// List of all plugins that can support offloading.
27static const char *RTLNames[] = {ENABLED_OFFLOAD_PLUGINS};
28
29Expected<std::unique_ptr<PluginAdaptorTy>>
30PluginAdaptorTy::create(const std::string &Name) {
31 DP("Attempting to load library '%s'...\n", Name.c_str());
32 TIMESCOPE_WITH_NAME_AND_IDENT(Name, (const ident_t *)nullptr);
33
34 std::string ErrMsg;
35 auto LibraryHandler = std::make_unique<DynamicLibrary>(
36 DynamicLibrary::getPermanentLibrary(Name.c_str(), &ErrMsg));
37
38 if (!LibraryHandler->isValid()) {
39 // Library does not exist or cannot be found.
40 return createStringError(inconvertibleErrorCode(),
41 "Unable to load library '%s': %s!\n", Name.c_str(),
42 ErrMsg.c_str());
43 }
44
45 DP("Successfully loaded library '%s'!\n", Name.c_str());
46 auto PluginAdaptor = std::unique_ptr<PluginAdaptorTy>(
47 new PluginAdaptorTy(Name, std::move(LibraryHandler)));
48 if (auto Err = PluginAdaptor->init())
49 return Err;
50 return std::move(PluginAdaptor);
51}
52
53PluginAdaptorTy::PluginAdaptorTy(const std::string &Name,
54 std::unique_ptr<llvm::sys::DynamicLibrary> DL)
55 : Name(Name), LibraryHandler(std::move(DL)) {}
56
57Error PluginAdaptorTy::init() {
58
59#define PLUGIN_API_HANDLE(NAME) \
60 NAME = reinterpret_cast<decltype(NAME)>( \
61 LibraryHandler->getAddressOfSymbol(GETNAME(__tgt_rtl_##NAME))); \
62 if (!NAME) { \
63 return createStringError(inconvertibleErrorCode(), \
64 "Invalid plugin as necessary interface function " \
65 "(%s) was not found.\n", \
66 std::string(#NAME).c_str()); \
67 }
68
69#include "Shared/PluginAPI.inc"
70#undef PLUGIN_API_HANDLE
71
72 // Remove plugin on failure to call optional init_plugin
73 int32_t Rc = init_plugin();
74 if (Rc != OFFLOAD_SUCCESS) {
75 return createStringError(inconvertibleErrorCode(),
76 "Unable to initialize library '%s': %u!\n",
77 Name.c_str(), Rc);
78 }
79
80 // No devices are supported by this RTL?
81 int32_t NumberOfPluginDevices = number_of_devices();
82 if (!NumberOfPluginDevices) {
83 return createStringError(inconvertibleErrorCode(),
84 "No devices supported in this RTL\n");
85 }
86
87 DP("Registered '%s' with %d plugin visible devices!\n", Name.c_str(),
88 NumberOfPluginDevices);
89 return Error::success();
90}
91
92void PluginManager::init() {
93 TIMESCOPE();
94 DP("Loading RTLs...\n");
95
96 // Attempt to open all the plugins and, if they exist, check if the interface
97 // is correct and if they are supporting any devices.
98 for (const char *Name : RTLNames) {
99 auto PluginAdaptorOrErr =
100 PluginAdaptorTy::create(std::string(Name) + ".so");
101 if (!PluginAdaptorOrErr) {
102 [[maybe_unused]] std::string InfoMsg =
103 toString(PluginAdaptorOrErr.takeError());
104 DP("%s", InfoMsg.c_str());
105 } else {
106 PluginAdaptors.push_back(std::move(*PluginAdaptorOrErr));
107 }
108 }
109
110 DP("RTLs loaded!\n");
111}
112
113void PluginManager::initDevices(PluginAdaptorTy &RTL) {
114 // If this RTL has already been initialized.
115 if (PM->DeviceOffsets.contains(&RTL))
116 return;
117 TIMESCOPE();
118
119 // If this RTL is not already in use, initialize it.
120 assert(RTL.number_of_devices() > 0 &&
121 "Tried to initialize useless plugin adaptor");
122
123 // Initialize the device information for the RTL we are about to use.
124 auto ExclusiveDevicesAccessor = getExclusiveDevicesAccessor();
125
126 // Initialize the index of this RTL and save it in the used RTLs.
127 int32_t DeviceOffset = ExclusiveDevicesAccessor->size();
128
129 // Set the device identifier offset in the plugin.
130 RTL.set_device_offset(DeviceOffset);
131
132 int32_t NumberOfUserDevices = 0;
133 int32_t NumPD = RTL.number_of_devices();
134 ExclusiveDevicesAccessor->reserve(DeviceOffset + NumPD);
135 // Auto zero-copy is a per-device property. We need to ensure
136 // that all devices are suggesting to use it.
137 bool UseAutoZeroCopy = !(NumPD == 0);
138 for (int32_t PDevI = 0, UserDevId = DeviceOffset; PDevI < NumPD; PDevI++) {
139 auto Device = std::make_unique<DeviceTy>(&RTL, UserDevId, PDevI);
140 if (auto Err = Device->init()) {
141 DP("Skip plugin known device %d: %s\n", PDevI,
142 toString(std::move(Err)).c_str());
143 continue;
144 }
145 UseAutoZeroCopy = UseAutoZeroCopy && Device->useAutoZeroCopy();
146
147 ExclusiveDevicesAccessor->push_back(std::move(Device));
148 ++NumberOfUserDevices;
149 ++UserDevId;
150 }
151
152 // Auto Zero-Copy can only be currently triggered when the system is an
153 // homogeneous APU architecture without attached discrete GPUs.
154 // If all devices suggest to use it, change requirment flags to trigger
155 // zero-copy behavior when mapping memory.
156 if (UseAutoZeroCopy)
157 addRequirements(OMPX_REQ_AUTO_ZERO_COPY);
158
159 DeviceOffsets[&RTL] = DeviceOffset;
160 DeviceUsed[&RTL] = NumberOfUserDevices;
161 DP("Plugin adaptor " DPxMOD " has index %d, exposes %d out of %d devices!\n",
162 DPxPTR(RTL.LibraryHandler.get()), DeviceOffset, NumberOfUserDevices,
163 RTL.number_of_devices());
164}
165
166void PluginManager::initAllPlugins() {
167 for (auto &R : PluginAdaptors)
168 initDevices(*R);
169}
170
171static void registerImageIntoTranslationTable(TranslationTable &TT,
172 int32_t DeviceOffset,
173 int32_t NumberOfUserDevices,
174 __tgt_device_image *Image) {
175
176 // same size, as when we increase one, we also increase the other.
177 assert(TT.TargetsTable.size() == TT.TargetsImages.size() &&
178 "We should have as many images as we have tables!");
179
180 // Resize the Targets Table and Images to accommodate the new targets if
181 // required
182 unsigned TargetsTableMinimumSize = DeviceOffset + NumberOfUserDevices;
183
184 if (TT.TargetsTable.size() < TargetsTableMinimumSize) {
185 TT.DeviceTables.resize(TargetsTableMinimumSize, {});
186 TT.TargetsImages.resize(TargetsTableMinimumSize, 0);
187 TT.TargetsEntries.resize(TargetsTableMinimumSize, {});
188 TT.TargetsTable.resize(TargetsTableMinimumSize, 0);
189 }
190
191 // Register the image in all devices for this target type.
192 for (int32_t I = 0; I < NumberOfUserDevices; ++I) {
193 // If we are changing the image we are also invalidating the target table.
194 if (TT.TargetsImages[DeviceOffset + I] != Image) {
195 TT.TargetsImages[DeviceOffset + I] = Image;
196 TT.TargetsTable[DeviceOffset + I] =
197 0; // lazy initialization of target table.
198 }
199 }
200}
201
202void PluginManager::registerLib(__tgt_bin_desc *Desc) {
203 PM->RTLsMtx.lock();
204
205 // Add in all the OpenMP requirements associated with this binary.
206 for (__tgt_offload_entry &Entry :
207 llvm::make_range(Desc->HostEntriesBegin, Desc->HostEntriesEnd))
208 if (Entry.flags == OMP_REGISTER_REQUIRES)
209 PM->addRequirements(Entry.data);
210
211 // Extract the exectuable image and extra information if availible.
212 for (int32_t i = 0; i < Desc->NumDeviceImages; ++i)
213 PM->addDeviceImage(*Desc, Desc->DeviceImages[i]);
214
215 // Register the images with the RTLs that understand them, if any.
216 for (DeviceImageTy &DI : PM->deviceImages()) {
217 // Obtain the image and information that was previously extracted.
218 __tgt_device_image *Img = &DI.getExecutableImage();
219
220 PluginAdaptorTy *FoundRTL = nullptr;
221
222 // Scan the RTLs that have associated images until we find one that supports
223 // the current image.
224 for (auto &R : PM->pluginAdaptors()) {
225 if (!R.is_valid_binary(Img)) {
226 DP("Image " DPxMOD " is NOT compatible with RTL %s!\n",
227 DPxPTR(Img->ImageStart), R.Name.c_str());
228 continue;
229 }
230
231 DP("Image " DPxMOD " is compatible with RTL %s!\n",
232 DPxPTR(Img->ImageStart), R.Name.c_str());
233
234 PM->initDevices(R);
235
236 // Initialize (if necessary) translation table for this library.
237 PM->TrlTblMtx.lock();
238 if (!PM->HostEntriesBeginToTransTable.count(Desc->HostEntriesBegin)) {
239 PM->HostEntriesBeginRegistrationOrder.push_back(Desc->HostEntriesBegin);
240 TranslationTable &TransTable =
241 (PM->HostEntriesBeginToTransTable)[Desc->HostEntriesBegin];
242 TransTable.HostTable.EntriesBegin = Desc->HostEntriesBegin;
243 TransTable.HostTable.EntriesEnd = Desc->HostEntriesEnd;
244 }
245
246 // Retrieve translation table for this library.
247 TranslationTable &TransTable =
248 (PM->HostEntriesBeginToTransTable)[Desc->HostEntriesBegin];
249
250 DP("Registering image " DPxMOD " with RTL %s!\n", DPxPTR(Img->ImageStart),
251 R.Name.c_str());
252
253 registerImageIntoTranslationTable(TransTable, PM->DeviceOffsets[&R],
254 PM->DeviceUsed[&R], Img);
255 PM->UsedImages.insert(Img);
256
257 PM->TrlTblMtx.unlock();
258 FoundRTL = &R;
259
260 // if an RTL was found we are done - proceed to register the next image
261 break;
262 }
263
264 if (!FoundRTL) {
265 DP("No RTL found for image " DPxMOD "!\n", DPxPTR(Img->ImageStart));
266 }
267 }
268 PM->RTLsMtx.unlock();
269
270 DP("Done registering entries!\n");
271}
272
273// Temporary forward declaration, old style CTor/DTor handling is going away.
274int target(ident_t *Loc, DeviceTy &Device, void *HostPtr,
275 KernelArgsTy &KernelArgs, AsyncInfoTy &AsyncInfo);
276
277void PluginManager::unregisterLib(__tgt_bin_desc *Desc) {
278 DP("Unloading target library!\n");
279
280 PM->RTLsMtx.lock();
281 // Find which RTL understands each image, if any.
282 for (DeviceImageTy &DI : PM->deviceImages()) {
283 // Obtain the image and information that was previously extracted.
284 __tgt_device_image *Img = &DI.getExecutableImage();
285
286 PluginAdaptorTy *FoundRTL = NULL;
287
288 // Scan the RTLs that have associated images until we find one that supports
289 // the current image. We only need to scan RTLs that are already being used.
290 for (auto &R : PM->pluginAdaptors()) {
291 if (!DeviceOffsets.contains(&R))
292 continue;
293
294 // Ensure that we do not use any unused images associated with this RTL.
295 if (!UsedImages.contains(Img))
296 continue;
297
298 FoundRTL = &R;
299
300 DP("Unregistered image " DPxMOD " from RTL " DPxMOD "!\n",
301 DPxPTR(Img->ImageStart), DPxPTR(R.LibraryHandler.get()));
302
303 break;
304 }
305
306 // if no RTL was found proceed to unregister the next image
307 if (!FoundRTL) {
308 DP("No RTLs in use support the image " DPxMOD "!\n",
309 DPxPTR(Img->ImageStart));
310 }
311 }
312 PM->RTLsMtx.unlock();
313 DP("Done unregistering images!\n");
314
315 // Remove entries from PM->HostPtrToTableMap
316 PM->TblMapMtx.lock();
317 for (__tgt_offload_entry *Cur = Desc->HostEntriesBegin;
318 Cur < Desc->HostEntriesEnd; ++Cur) {
319 PM->HostPtrToTableMap.erase(Cur->addr);
320 }
321
322 // Remove translation table for this descriptor.
323 auto TransTable =
324 PM->HostEntriesBeginToTransTable.find(Desc->HostEntriesBegin);
325 if (TransTable != PM->HostEntriesBeginToTransTable.end()) {
326 DP("Removing translation table for descriptor " DPxMOD "\n",
327 DPxPTR(Desc->HostEntriesBegin));
328 PM->HostEntriesBeginToTransTable.erase(TransTable);
329 } else {
330 DP("Translation table for descriptor " DPxMOD " cannot be found, probably "
331 "it has been already removed.\n",
332 DPxPTR(Desc->HostEntriesBegin));
333 }
334
335 PM->TblMapMtx.unlock();
336
337 DP("Done unregistering library!\n");
338}
339
340Expected<DeviceTy &> PluginManager::getDevice(uint32_t DeviceNo) {
341 auto ExclusiveDevicesAccessor = getExclusiveDevicesAccessor();
342 if (DeviceNo >= ExclusiveDevicesAccessor->size())
343 return createStringError(
344 inconvertibleErrorCode(),
345 "Device number '%i' out of range, only %i devices available", DeviceNo,
346 ExclusiveDevicesAccessor->size());
347
348 return *(*ExclusiveDevicesAccessor)[DeviceNo];
349}
350

source code of offload/src/PluginManager.cpp