1//===-RTLs/generic-64bit/src/rtl.cpp - Target RTLs Implementation - C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// RTL NextGen for generic 64-bit machine
10//
11//===----------------------------------------------------------------------===//
12
13#include <cassert>
14#include <cstddef>
15#include <ffi.h>
16#include <string>
17#include <unordered_map>
18
19#include "Shared/Debug.h"
20#include "Shared/Environment.h"
21#include "Utils/ELF.h"
22
23#include "GlobalHandler.h"
24#include "OpenMP/OMPT/Callback.h"
25#include "PluginInterface.h"
26#include "omptarget.h"
27
28#include "llvm/ADT/SmallVector.h"
29#include "llvm/Frontend/OpenMP/OMPConstants.h"
30#include "llvm/Frontend/OpenMP/OMPDeviceConstants.h"
31#include "llvm/Frontend/OpenMP/OMPGridValues.h"
32#include "llvm/Support/DynamicLibrary.h"
33
34#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \
35 !defined(__ORDER_BIG_ENDIAN__)
36#error "Missing preprocessor definitions for endianness detection."
37#endif
38
39#if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
40#define LITTLEENDIAN_CPU
41#elif defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
42#define BIGENDIAN_CPU
43#endif
44
45// The number of devices in this plugin.
46#define NUM_DEVICES 4
47
48namespace llvm {
49namespace omp {
50namespace target {
51namespace plugin {
52
53/// Forward declarations for all specialized data structures.
54struct GenELF64KernelTy;
55struct GenELF64DeviceTy;
56struct GenELF64PluginTy;
57
58using llvm::sys::DynamicLibrary;
59using namespace error;
60
61/// Class implementing kernel functionalities for GenELF64.
62struct GenELF64KernelTy : public GenericKernelTy {
63 /// Construct the kernel with a name and an execution mode.
64 GenELF64KernelTy(const char *Name) : GenericKernelTy(Name), Func(nullptr) {}
65
66 /// Initialize the kernel.
67 Error initImpl(GenericDeviceTy &Device, DeviceImageTy &Image) override {
68 // Functions have zero size.
69 GlobalTy Global(getName(), 0);
70
71 // Get the metadata (address) of the kernel function.
72 GenericGlobalHandlerTy &GHandler = Device.Plugin.getGlobalHandler();
73 if (auto Err = GHandler.getGlobalMetadataFromDevice(Device, Image, Global))
74 return Err;
75
76 // Check that the function pointer is valid.
77 if (!Global.getPtr())
78 return Plugin::error(ErrorCode::INVALID_BINARY,
79 "invalid function for kernel %s", getName());
80
81 // Save the function pointer.
82 Func = (void (*)())Global.getPtr();
83
84 KernelEnvironment.Configuration.ExecMode = OMP_TGT_EXEC_MODE_GENERIC;
85 KernelEnvironment.Configuration.MayUseNestedParallelism = /*Unknown=*/2;
86 KernelEnvironment.Configuration.UseGenericStateMachine = /*Unknown=*/2;
87
88 // Set the maximum number of threads to a single.
89 MaxNumThreads = 1;
90 return Plugin::success();
91 }
92
93 /// Launch the kernel using the libffi.
94 Error launchImpl(GenericDeviceTy &GenericDevice, uint32_t NumThreads[3],
95 uint32_t NumBlocks[3], KernelArgsTy &KernelArgs,
96 KernelLaunchParamsTy LaunchParams,
97 AsyncInfoWrapperTy &AsyncInfoWrapper) const override {
98 // Create a vector of ffi_types, one per argument.
99 SmallVector<ffi_type *, 16> ArgTypes(KernelArgs.NumArgs, &ffi_type_pointer);
100 ffi_type **ArgTypesPtr = (ArgTypes.size()) ? &ArgTypes[0] : nullptr;
101
102 // Prepare the cif structure before running the kernel function.
103 ffi_cif Cif;
104 ffi_status Status = ffi_prep_cif(&Cif, FFI_DEFAULT_ABI, KernelArgs.NumArgs,
105 &ffi_type_void, ArgTypesPtr);
106 if (Status != FFI_OK)
107 return Plugin::error(ErrorCode::UNKNOWN, "error in ffi_prep_cif: %d",
108 Status);
109
110 // Call the kernel function through libffi.
111 long Return;
112 ffi_call(cif: &Cif, fn: Func, rvalue: &Return, avalue: (void **)LaunchParams.Ptrs);
113
114 return Plugin::success();
115 }
116
117private:
118 /// The kernel function to execute.
119 void (*Func)(void);
120};
121
122/// Class implementing the GenELF64 device images properties.
123struct GenELF64DeviceImageTy : public DeviceImageTy {
124 /// Create the GenELF64 image with the id and the target image pointer.
125 GenELF64DeviceImageTy(int32_t ImageId, GenericDeviceTy &Device,
126 const __tgt_device_image *TgtImage)
127 : DeviceImageTy(ImageId, Device, TgtImage), DynLib() {}
128
129 /// Getter and setter for the dynamic library.
130 DynamicLibrary &getDynamicLibrary() { return DynLib; }
131 void setDynamicLibrary(const DynamicLibrary &Lib) { DynLib = Lib; }
132
133private:
134 /// The dynamic library that loaded the image.
135 DynamicLibrary DynLib;
136};
137
138/// Class implementing the device functionalities for GenELF64.
139struct GenELF64DeviceTy : public GenericDeviceTy {
140 /// Create the device with a specific id.
141 GenELF64DeviceTy(GenericPluginTy &Plugin, int32_t DeviceId,
142 int32_t NumDevices)
143 : GenericDeviceTy(Plugin, DeviceId, NumDevices, GenELF64GridValues) {}
144
145 ~GenELF64DeviceTy() {}
146
147 /// Initialize the device, which is a no-op
148 Error initImpl(GenericPluginTy &Plugin) override { return Plugin::success(); }
149
150 /// Unload the binary image
151 ///
152 /// TODO: This currently does nothing, and should be implemented as part of
153 /// broader memory handling logic for this plugin
154 Error unloadBinaryImpl(DeviceImageTy *Image) override {
155 auto Elf = reinterpret_cast<GenELF64DeviceImageTy *>(Image);
156 DynamicLibrary::closeLibrary(Lib&: Elf->getDynamicLibrary());
157 Plugin.free(Elf);
158 return Plugin::success();
159 }
160
161 /// Deinitialize the device, which is a no-op
162 Error deinitImpl() override { return Plugin::success(); }
163
164 /// See GenericDeviceTy::getComputeUnitKind().
165 std::string getComputeUnitKind() const override { return "generic-64bit"; }
166
167 /// Construct the kernel for a specific image on the device.
168 Expected<GenericKernelTy &> constructKernel(const char *Name) override {
169 // Allocate and construct the kernel.
170 GenELF64KernelTy *GenELF64Kernel = Plugin.allocate<GenELF64KernelTy>();
171 if (!GenELF64Kernel)
172 return Plugin::error(ErrorCode::OUT_OF_RESOURCES,
173 "failed to allocate memory for GenELF64 kernel");
174
175 new (GenELF64Kernel) GenELF64KernelTy(Name);
176
177 return *GenELF64Kernel;
178 }
179
180 /// Set the current context to this device, which is a no-op.
181 Error setContext() override { return Plugin::success(); }
182
183 /// Load the binary image into the device and allocate an image object.
184 Expected<DeviceImageTy *> loadBinaryImpl(const __tgt_device_image *TgtImage,
185 int32_t ImageId) override {
186 // Allocate and initialize the image object.
187 GenELF64DeviceImageTy *Image = Plugin.allocate<GenELF64DeviceImageTy>();
188 new (Image) GenELF64DeviceImageTy(ImageId, *this, TgtImage);
189
190 // Create a temporary file.
191 char TmpFileName[] = "/tmp/tmpfile_XXXXXX";
192 int TmpFileFd = mkstemp(template: TmpFileName);
193 if (TmpFileFd == -1)
194 return Plugin::error(ErrorCode::HOST_IO,
195 "failed to create tmpfile for loading target image");
196
197 // Open the temporary file.
198 FILE *TmpFile = fdopen(fd: TmpFileFd, modes: "wb");
199 if (!TmpFile)
200 return Plugin::error(ErrorCode::HOST_IO,
201 "failed to open tmpfile %s for loading target image",
202 TmpFileName);
203
204 // Write the image into the temporary file.
205 size_t Written = fwrite(Image->getStart(), Image->getSize(), 1, TmpFile);
206 if (Written != 1)
207 return Plugin::error(ErrorCode::HOST_IO,
208 "failed to write target image to tmpfile %s",
209 TmpFileName);
210
211 // Close the temporary file.
212 int Ret = fclose(stream: TmpFile);
213 if (Ret)
214 return Plugin::error(ErrorCode::HOST_IO,
215 "failed to close tmpfile %s with the target image",
216 TmpFileName);
217
218 // Load the temporary file as a dynamic library.
219 std::string ErrMsg;
220 DynamicLibrary DynLib = DynamicLibrary::getLibrary(FileName: TmpFileName, Err: &ErrMsg);
221
222 // Check if the loaded library is valid.
223 if (!DynLib.isValid())
224 return Plugin::error(ErrorCode::INVALID_BINARY,
225 "failed to load target image: %s", ErrMsg.c_str());
226
227 // Save a reference of the image's dynamic library.
228 Image->setDynamicLibrary(DynLib);
229
230 return Image;
231 }
232
233 /// Allocate memory. Use std::malloc in all cases.
234 void *allocate(size_t Size, void *, TargetAllocTy Kind) override {
235 if (Size == 0)
236 return nullptr;
237
238 void *MemAlloc = nullptr;
239 switch (Kind) {
240 case TARGET_ALLOC_DEFAULT:
241 case TARGET_ALLOC_DEVICE:
242 case TARGET_ALLOC_HOST:
243 case TARGET_ALLOC_SHARED:
244 case TARGET_ALLOC_DEVICE_NON_BLOCKING:
245 MemAlloc = std::malloc(size: Size);
246 break;
247 }
248 return MemAlloc;
249 }
250
251 /// Free the memory. Use std::free in all cases.
252 int free(void *TgtPtr, TargetAllocTy Kind) override {
253 std::free(ptr: TgtPtr);
254 return OFFLOAD_SUCCESS;
255 }
256
257 /// This plugin does nothing to lock buffers. Do not return an error, just
258 /// return the same pointer as the device pointer.
259 Expected<void *> dataLockImpl(void *HstPtr, int64_t Size) override {
260 return HstPtr;
261 }
262
263 /// Nothing to do when unlocking the buffer.
264 Error dataUnlockImpl(void *HstPtr) override { return Plugin::success(); }
265
266 /// Indicate that the buffer is not pinned.
267 Expected<bool> isPinnedPtrImpl(void *HstPtr, void *&BaseHstPtr,
268 void *&BaseDevAccessiblePtr,
269 size_t &BaseSize) const override {
270 return false;
271 }
272
273 /// Submit data to the device (host to device transfer).
274 Error dataSubmitImpl(void *TgtPtr, const void *HstPtr, int64_t Size,
275 AsyncInfoWrapperTy &AsyncInfoWrapper) override {
276 std::memcpy(dest: TgtPtr, src: HstPtr, n: Size);
277 return Plugin::success();
278 }
279
280 /// Retrieve data from the device (device to host transfer).
281 Error dataRetrieveImpl(void *HstPtr, const void *TgtPtr, int64_t Size,
282 AsyncInfoWrapperTy &AsyncInfoWrapper) override {
283 std::memcpy(dest: HstPtr, src: TgtPtr, n: Size);
284 return Plugin::success();
285 }
286
287 /// Exchange data between two devices within the plugin. This function is not
288 /// supported in this plugin.
289 Error dataExchangeImpl(const void *SrcPtr, GenericDeviceTy &DstGenericDevice,
290 void *DstPtr, int64_t Size,
291 AsyncInfoWrapperTy &AsyncInfoWrapper) override {
292 // This function should never be called because the function
293 // GenELF64PluginTy::isDataExchangable() returns false.
294 return Plugin::error(ErrorCode::UNSUPPORTED,
295 "dataExchangeImpl not supported");
296 }
297
298 /// All functions are already synchronous. No need to do anything on this
299 /// synchronization function.
300 Error synchronizeImpl(__tgt_async_info &AsyncInfo) override {
301 return Plugin::success();
302 }
303
304 /// All functions are already synchronous. No need to do anything on this
305 /// query function.
306 Error queryAsyncImpl(__tgt_async_info &AsyncInfo) override {
307 return Plugin::success();
308 }
309
310 /// This plugin does not support interoperability
311 Error initAsyncInfoImpl(AsyncInfoWrapperTy &AsyncInfoWrapper) override {
312 return Plugin::error(ErrorCode::UNSUPPORTED,
313 "initAsyncInfoImpl not supported");
314 }
315
316 /// This plugin does not support interoperability
317 Error initDeviceInfoImpl(__tgt_device_info *DeviceInfo) override {
318 return Plugin::error(ErrorCode::UNSUPPORTED,
319 "initDeviceInfoImpl not supported");
320 }
321
322 /// This plugin does not support the event API. Do nothing without failing.
323 Error createEventImpl(void **EventPtrStorage) override {
324 *EventPtrStorage = nullptr;
325 return Plugin::success();
326 }
327 Error destroyEventImpl(void *EventPtr) override { return Plugin::success(); }
328 Error recordEventImpl(void *EventPtr,
329 AsyncInfoWrapperTy &AsyncInfoWrapper) override {
330 return Plugin::success();
331 }
332 Error waitEventImpl(void *EventPtr,
333 AsyncInfoWrapperTy &AsyncInfoWrapper) override {
334 return Plugin::success();
335 }
336 Error syncEventImpl(void *EventPtr) override { return Plugin::success(); }
337
338 /// Print information about the device.
339 Expected<InfoTreeNode> obtainInfoImpl() override {
340 InfoTreeNode Info;
341 Info.add("Device Type", "Generic-elf-64bit");
342 return Info;
343 }
344
345 /// This plugin should not setup the device environment or memory pool.
346 virtual bool shouldSetupDeviceEnvironment() const override { return false; };
347 virtual bool shouldSetupDeviceMemoryPool() const override { return false; };
348
349 /// Getters and setters for stack size and heap size not relevant.
350 Error getDeviceStackSize(uint64_t &Value) override {
351 Value = 0;
352 return Plugin::success();
353 }
354 Error setDeviceStackSize(uint64_t Value) override {
355 return Plugin::success();
356 }
357 Error getDeviceHeapSize(uint64_t &Value) override {
358 Value = 0;
359 return Plugin::success();
360 }
361 Error setDeviceHeapSize(uint64_t Value) override { return Plugin::success(); }
362
363private:
364 /// Grid values for Generic ELF64 plugins.
365 static constexpr GV GenELF64GridValues = {
366 .GV_Slot_Size: 1, // GV_Slot_Size
367 .GV_Warp_Size: 1, // GV_Warp_Size
368 .GV_Max_Teams: 1, // GV_Max_Teams
369 .GV_Default_Num_Teams: 1, // GV_Default_Num_Teams
370 .GV_SimpleBufferSize: 1, // GV_SimpleBufferSize
371 .GV_Max_WG_Size: 1, // GV_Max_WG_Size
372 .GV_Default_WG_Size: 1, // GV_Default_WG_Size
373 };
374};
375
376class GenELF64GlobalHandlerTy final : public GenericGlobalHandlerTy {
377public:
378 Error getGlobalMetadataFromDevice(GenericDeviceTy &GenericDevice,
379 DeviceImageTy &Image,
380 GlobalTy &DeviceGlobal) override {
381 const char *GlobalName = DeviceGlobal.getName().data();
382 GenELF64DeviceImageTy &GenELF64Image =
383 static_cast<GenELF64DeviceImageTy &>(Image);
384
385 // Get dynamic library that has loaded the device image.
386 DynamicLibrary &DynLib = GenELF64Image.getDynamicLibrary();
387
388 // Get the address of the symbol.
389 void *Addr = DynLib.getAddressOfSymbol(symbolName: GlobalName);
390 if (Addr == nullptr) {
391 return Plugin::error(ErrorCode::NOT_FOUND, "failed to load global '%s'",
392 GlobalName);
393 }
394
395 // Save the pointer to the symbol.
396 DeviceGlobal.setPtr(Addr);
397
398 return Plugin::success();
399 }
400};
401
402/// Class implementing the plugin functionalities for GenELF64.
403struct GenELF64PluginTy final : public GenericPluginTy {
404 /// Create the GenELF64 plugin.
405 GenELF64PluginTy() : GenericPluginTy(getTripleArch()) {}
406
407 /// This class should not be copied.
408 GenELF64PluginTy(const GenELF64PluginTy &) = delete;
409 GenELF64PluginTy(GenELF64PluginTy &&) = delete;
410
411 /// Initialize the plugin and return the number of devices.
412 Expected<int32_t> initImpl() override {
413#ifdef USES_DYNAMIC_FFI
414 if (auto Err = Plugin::check(ffi_init(), "failed to initialize libffi"))
415 return std::move(Err);
416#endif
417
418 return NUM_DEVICES;
419 }
420
421 /// Deinitialize the plugin.
422 Error deinitImpl() override { return Plugin::success(); }
423
424 /// Creates a generic ELF device.
425 GenericDeviceTy *createDevice(GenericPluginTy &Plugin, int32_t DeviceId,
426 int32_t NumDevices) override {
427 return new GenELF64DeviceTy(Plugin, DeviceId, NumDevices);
428 }
429
430 /// Creates a generic global handler.
431 GenericGlobalHandlerTy *createGlobalHandler() override {
432 return new GenELF64GlobalHandlerTy();
433 }
434
435 /// Get the ELF code to recognize the compatible binary images.
436 uint16_t getMagicElfBits() const override {
437 return utils::elf::getTargetMachine();
438 }
439
440 /// This plugin does not support exchanging data between two devices.
441 bool isDataExchangable(int32_t SrcDeviceId, int32_t DstDeviceId) override {
442 return false;
443 }
444
445 /// All images (ELF-compatible) should be compatible with this plugin.
446 Expected<bool> isELFCompatible(uint32_t, StringRef) const override {
447 return true;
448 }
449
450 Triple::ArchType getTripleArch() const override {
451#if defined(__x86_64__)
452 return llvm::Triple::x86_64;
453#elif defined(__s390x__)
454 return llvm::Triple::systemz;
455#elif defined(__aarch64__)
456#ifdef LITTLEENDIAN_CPU
457 return llvm::Triple::aarch64;
458#else
459 return llvm::Triple::aarch64_be;
460#endif
461#elif defined(__powerpc64__)
462#ifdef LITTLEENDIAN_CPU
463 return llvm::Triple::ppc64le;
464#else
465 return llvm::Triple::ppc64;
466#endif
467#elif defined(__riscv) && (__riscv_xlen == 64)
468 return llvm::Triple::riscv64;
469#elif defined(__loongarch__) && (__loongarch_grlen == 64)
470 return llvm::Triple::loongarch64;
471#else
472 return llvm::Triple::UnknownArch;
473#endif
474 }
475
476 const char *getName() const override { return GETNAME(TARGET_NAME); }
477};
478
479template <typename... ArgsTy>
480static Error Plugin::check(int32_t Code, const char *ErrMsg, ArgsTy... Args) {
481 if (Code == 0)
482 return Plugin::success();
483
484 return Plugin::error(ErrorCode::UNKNOWN, ErrMsg, Args...,
485 std::to_string(Code).data());
486}
487
488} // namespace plugin
489} // namespace target
490} // namespace omp
491} // namespace llvm
492
493extern "C" {
494llvm::omp::target::plugin::GenericPluginTy *createPlugin_host() {
495 return new llvm::omp::target::plugin::GenELF64PluginTy();
496}
497}
498

source code of offload/plugins-nextgen/host/src/rtl.cpp