1//===- GlobalHandler.cpp - Target independent global & env. var handling --===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Target independent global handler and environment manager.
10//
11//===----------------------------------------------------------------------===//
12
13#include "GlobalHandler.h"
14#include "PluginInterface.h"
15#include "Utils/ELF.h"
16
17#include "Shared/Utils.h"
18
19#include "llvm/ProfileData/InstrProfData.inc"
20#include "llvm/Support/Error.h"
21
22#include <cstring>
23#include <string>
24
25using namespace llvm;
26using namespace omp;
27using namespace target;
28using namespace plugin;
29using namespace error;
30
31Expected<std::unique_ptr<ObjectFile>>
32GenericGlobalHandlerTy::getELFObjectFile(DeviceImageTy &Image) {
33 assert(utils::elf::isELF(Image.getMemoryBuffer().getBuffer()) &&
34 "Input is not an ELF file");
35
36 auto Expected =
37 ELFObjectFileBase::createELFObjectFile(Image.getMemoryBuffer());
38 if (!Expected) {
39 return Plugin::error(ErrorCode::INVALID_BINARY, Expected.takeError(),
40 "error parsing binary");
41 }
42 return Expected;
43}
44
45Error GenericGlobalHandlerTy::moveGlobalBetweenDeviceAndHost(
46 GenericDeviceTy &Device, DeviceImageTy &Image, const GlobalTy &HostGlobal,
47 bool Device2Host) {
48
49 GlobalTy DeviceGlobal(HostGlobal.getName(), HostGlobal.getSize());
50
51 // Get the metadata from the global on the device.
52 if (auto Err = getGlobalMetadataFromDevice(Device, Image, DeviceGlobal))
53 return Err;
54
55 // Perform the actual transfer.
56 return moveGlobalBetweenDeviceAndHost(Device, HostGlobal, DeviceGlobal,
57 Device2Host);
58}
59
60/// Actually move memory between host and device. See readGlobalFromDevice and
61/// writeGlobalToDevice for the interface description.
62Error GenericGlobalHandlerTy::moveGlobalBetweenDeviceAndHost(
63 GenericDeviceTy &Device, const GlobalTy &HostGlobal,
64 const GlobalTy &DeviceGlobal, bool Device2Host) {
65
66 // Transfer the data from the source to the destination.
67 if (Device2Host) {
68 if (auto Err =
69 Device.dataRetrieve(HostGlobal.getPtr(), DeviceGlobal.getPtr(),
70 HostGlobal.getSize(), nullptr))
71 return Err;
72 } else {
73 if (auto Err = Device.dataSubmit(DeviceGlobal.getPtr(), HostGlobal.getPtr(),
74 HostGlobal.getSize(), nullptr))
75 return Err;
76 }
77
78 DP("Successfully %s %u bytes associated with global symbol '%s' %s the "
79 "device "
80 "(%p -> %p).\n",
81 Device2Host ? "read" : "write", HostGlobal.getSize(),
82 HostGlobal.getName().data(), Device2Host ? "from" : "to",
83 DeviceGlobal.getPtr(), HostGlobal.getPtr());
84
85 return Plugin::success();
86}
87
88bool GenericGlobalHandlerTy::isSymbolInImage(GenericDeviceTy &Device,
89 DeviceImageTy &Image,
90 StringRef SymName) {
91 // Get the ELF object file for the image. Notice the ELF object may already
92 // be created in previous calls, so we can reuse it. If this is unsuccessful
93 // just return false as we couldn't find it.
94 auto ELFObjOrErr = getELFObjectFile(Image);
95 if (!ELFObjOrErr) {
96 consumeError(ELFObjOrErr.takeError());
97 return false;
98 }
99
100 // Search the ELF symbol using the symbol name.
101 auto SymOrErr = utils::elf::getSymbol(**ELFObjOrErr, SymName);
102 if (!SymOrErr) {
103 consumeError(SymOrErr.takeError());
104 return false;
105 }
106
107 return SymOrErr->has_value();
108}
109
110Error GenericGlobalHandlerTy::getGlobalMetadataFromImage(
111 GenericDeviceTy &Device, DeviceImageTy &Image, GlobalTy &ImageGlobal) {
112
113 // Get the ELF object file for the image. Notice the ELF object may already
114 // be created in previous calls, so we can reuse it.
115 auto ELFObj = getELFObjectFile(Image);
116 if (!ELFObj)
117 return ELFObj.takeError();
118
119 // Search the ELF symbol using the symbol name.
120 auto SymOrErr = utils::elf::getSymbol(**ELFObj, ImageGlobal.getName());
121 if (!SymOrErr)
122 return Plugin::error(
123 ErrorCode::NOT_FOUND, "failed ELF lookup of global '%s': %s",
124 ImageGlobal.getName().data(), toString(SymOrErr.takeError()).data());
125
126 if (!SymOrErr->has_value())
127 return Plugin::error(ErrorCode::NOT_FOUND,
128 "failed to find global symbol '%s' in the ELF image",
129 ImageGlobal.getName().data());
130
131 auto AddrOrErr = utils::elf::getSymbolAddress(**SymOrErr);
132 // Get the section to which the symbol belongs.
133 if (!AddrOrErr)
134 return Plugin::error(
135 ErrorCode::NOT_FOUND, "failed to get ELF symbol from global '%s': %s",
136 ImageGlobal.getName().data(), toString(AddrOrErr.takeError()).data());
137
138 // Setup the global symbol's address and size.
139 ImageGlobal.setPtr(const_cast<void *>(*AddrOrErr));
140 ImageGlobal.setSize((*SymOrErr)->getSize());
141
142 return Plugin::success();
143}
144
145Error GenericGlobalHandlerTy::readGlobalFromImage(GenericDeviceTy &Device,
146 DeviceImageTy &Image,
147 const GlobalTy &HostGlobal) {
148
149 GlobalTy ImageGlobal(HostGlobal.getName(), -1);
150 if (auto Err = getGlobalMetadataFromImage(Device, Image, ImageGlobal))
151 return Err;
152
153 if (ImageGlobal.getSize() != HostGlobal.getSize())
154 return Plugin::error(ErrorCode::INVALID_BINARY,
155 "transfer failed because global symbol '%s' has "
156 "%u bytes in the ELF image but %u bytes on the host",
157 HostGlobal.getName().data(), ImageGlobal.getSize(),
158 HostGlobal.getSize());
159
160 DP("Global symbol '%s' was found in the ELF image and %u bytes will copied "
161 "from %p to %p.\n",
162 HostGlobal.getName().data(), HostGlobal.getSize(), ImageGlobal.getPtr(),
163 HostGlobal.getPtr());
164
165 assert(Image.getStart() <= ImageGlobal.getPtr() &&
166 utils::advancePtr(ImageGlobal.getPtr(), ImageGlobal.getSize()) <
167 utils::advancePtr(Image.getStart(), Image.getSize()) &&
168 "Attempting to read outside the image!");
169
170 // Perform the copy from the image to the host memory.
171 std::memcpy(HostGlobal.getPtr(), ImageGlobal.getPtr(), HostGlobal.getSize());
172
173 return Plugin::success();
174}
175
176Expected<GPUProfGlobals>
177GenericGlobalHandlerTy::readProfilingGlobals(GenericDeviceTy &Device,
178 DeviceImageTy &Image) {
179 GPUProfGlobals DeviceProfileData;
180 auto ObjFile = getELFObjectFile(Image);
181 if (!ObjFile)
182 return ObjFile.takeError();
183
184 std::unique_ptr<ELFObjectFileBase> ELFObj(
185 static_cast<ELFObjectFileBase *>(ObjFile->release()));
186 DeviceProfileData.TargetTriple = ELFObj->makeTriple();
187
188 // Iterate through elf symbols
189 for (auto &Sym : ELFObj->symbols()) {
190 auto NameOrErr = Sym.getName();
191 if (!NameOrErr)
192 return NameOrErr.takeError();
193
194 // Check if given current global is a profiling global based
195 // on name
196 if (*NameOrErr == getInstrProfNamesVarName()) {
197 // Read in profiled function names from ELF
198 auto SectionOrErr = Sym.getSection();
199 if (!SectionOrErr)
200 return SectionOrErr.takeError();
201
202 auto ContentsOrErr = (*SectionOrErr)->getContents();
203 if (!ContentsOrErr)
204 return ContentsOrErr.takeError();
205
206 SmallVector<uint8_t> NameBytes(ContentsOrErr->bytes());
207 DeviceProfileData.NamesData = NameBytes;
208 } else if (NameOrErr->starts_with(getInstrProfCountersVarPrefix())) {
209 // Read global variable profiling counts
210 SmallVector<int64_t> Counts(Sym.getSize() / sizeof(int64_t), 0);
211 GlobalTy CountGlobal(NameOrErr->str(), Sym.getSize(), Counts.data());
212 if (auto Err = readGlobalFromDevice(Device, Image, CountGlobal))
213 return Err;
214 DeviceProfileData.Counts.append(std::move(Counts));
215 } else if (NameOrErr->starts_with(getInstrProfDataVarPrefix())) {
216 // Read profiling data for this global variable
217 __llvm_profile_data Data{};
218 GlobalTy DataGlobal(NameOrErr->str(), Sym.getSize(), &Data);
219 if (auto Err = readGlobalFromDevice(Device, Image, DataGlobal))
220 return Err;
221 DeviceProfileData.Data.push_back(std::move(Data));
222 } else if (*NameOrErr == INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR)) {
223 uint64_t RawVersionData;
224 GlobalTy RawVersionGlobal(NameOrErr->str(), Sym.getSize(),
225 &RawVersionData);
226 if (auto Err = readGlobalFromDevice(Device, Image, RawVersionGlobal))
227 return Err;
228 DeviceProfileData.Version = RawVersionData;
229 }
230 }
231 return DeviceProfileData;
232}
233
234void GPUProfGlobals::dump() const {
235 outs() << "======= GPU Profile =======\nTarget: " << TargetTriple.str()
236 << "\n";
237
238 outs() << "======== Counters =========\n";
239 for (size_t i = 0; i < Counts.size(); i++) {
240 if (i > 0 && i % 10 == 0)
241 outs() << "\n";
242 else if (i != 0)
243 outs() << " ";
244 outs() << Counts[i];
245 }
246 outs() << "\n";
247
248 outs() << "========== Data ===========\n";
249 for (const auto &ProfData : Data) {
250 outs() << "{ ";
251// The ProfData.Name maybe array, eg: NumValueSites[IPVK_Last+1] .
252// If we print out it directly, we are accessing out of bound data.
253// Skip dumping the array for now.
254#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
255 if (sizeof(#Name) > 2 && #Name[sizeof(#Name) - 2] == ']') { \
256 outs() << "[...] "; \
257 } else { \
258 outs() << ProfData.Name << " "; \
259 }
260#include "llvm/ProfileData/InstrProfData.inc"
261 outs() << "}\n";
262 }
263
264 outs() << "======== Functions ========\n";
265 std::string s;
266 s.reserve(NamesData.size());
267 for (uint8_t Name : NamesData) {
268 s.push_back((char)Name);
269 }
270
271 InstrProfSymtab Symtab;
272 if (Error Err = Symtab.create(StringRef(s))) {
273 consumeError(std::move(Err));
274 }
275 Symtab.dumpNames(outs());
276 outs() << "===========================\n";
277}
278
279Error GPUProfGlobals::write() const {
280 if (!__llvm_write_custom_profile)
281 return Plugin::error(ErrorCode::INVALID_BINARY,
282 "could not find symbol __llvm_write_custom_profile. "
283 "The compiler-rt profiling library must be linked for "
284 "GPU PGO to work.");
285
286 size_t DataSize = Data.size() * sizeof(__llvm_profile_data),
287 CountsSize = Counts.size() * sizeof(int64_t);
288 __llvm_profile_data *DataBegin, *DataEnd;
289 char *CountersBegin, *CountersEnd, *NamesBegin, *NamesEnd;
290
291 // Initialize array of contiguous data. We need to make sure each section is
292 // contiguous so that the PGO library can compute deltas properly
293 SmallVector<uint8_t> ContiguousData(NamesData.size() + DataSize + CountsSize);
294
295 // Compute region pointers
296 DataBegin = (__llvm_profile_data *)(ContiguousData.data() + CountsSize);
297 DataEnd =
298 (__llvm_profile_data *)(ContiguousData.data() + CountsSize + DataSize);
299 CountersBegin = (char *)ContiguousData.data();
300 CountersEnd = (char *)(ContiguousData.data() + CountsSize);
301 NamesBegin = (char *)(ContiguousData.data() + CountsSize + DataSize);
302 NamesEnd = (char *)(ContiguousData.data() + CountsSize + DataSize +
303 NamesData.size());
304
305 // Copy data to contiguous buffer
306 memcpy(DataBegin, Data.data(), DataSize);
307 memcpy(CountersBegin, Counts.data(), CountsSize);
308 memcpy(NamesBegin, NamesData.data(), NamesData.size());
309
310 // Invoke compiler-rt entrypoint
311 int result = __llvm_write_custom_profile(
312 TargetTriple.str().c_str(), DataBegin, DataEnd, CountersBegin,
313 CountersEnd, NamesBegin, NamesEnd, &Version);
314 if (result != 0)
315 return Plugin::error(ErrorCode::HOST_IO,
316 "error writing GPU PGO data to file");
317
318 return Plugin::success();
319}
320
321bool GPUProfGlobals::empty() const {
322 return Counts.empty() && Data.empty() && NamesData.empty();
323}
324

source code of offload/plugins-nextgen/common/src/GlobalHandler.cpp