1//===-- OpenMP/OMPT/Callback.cpp - OpenMP Tooling Callback implementation -===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Implementation of OMPT callback interfaces for target independent layer
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef OMPT_SUPPORT
14
15extern "C" {
16/// Dummy definition when OMPT is disabled
17void ompt_libomptarget_connect() {}
18}
19
20#else // OMPT_SUPPORT is set
21
22#include <cstdlib>
23#include <cstring>
24#include <memory>
25
26#include "Shared/Debug.h"
27
28#include "OpenMP/OMPT/Callback.h"
29#include "OpenMP/OMPT/Connector.h"
30#include "OpenMP/OMPT/Interface.h"
31
32#include "llvm/Support/DynamicLibrary.h"
33
34#undef DEBUG_PREFIX
35#define DEBUG_PREFIX "OMPT"
36
37using namespace llvm::omp::target::ompt;
38
39// Define OMPT callback functions (bound to actual callbacks later on)
40#define defineOmptCallback(Name, Type, Code) \
41 Name##_t llvm::omp::target::ompt::Name##_fn = nullptr;
42FOREACH_OMPT_NOEMI_EVENT(defineOmptCallback)
43FOREACH_OMPT_EMI_EVENT(defineOmptCallback)
44#undef defineOmptCallback
45
46/// Forward declaration
47class LibomptargetRtlFinalizer;
48
49/// Object that will maintain the RTL finalizer from the plugin
50LibomptargetRtlFinalizer *LibraryFinalizer = nullptr;
51
52thread_local Interface llvm::omp::target::ompt::RegionInterface;
53
54thread_local void *llvm::omp::target::ompt::ReturnAddress = nullptr;
55
56bool llvm::omp::target::ompt::Initialized = false;
57
58ompt_get_callback_t llvm::omp::target::ompt::lookupCallbackByCode = nullptr;
59ompt_function_lookup_t llvm::omp::target::ompt::lookupCallbackByName = nullptr;
60ompt_get_target_task_data_t ompt_get_target_task_data_fn = nullptr;
61ompt_get_task_data_t ompt_get_task_data_fn = nullptr;
62
63/// Unique correlation id
64static std::atomic<uint64_t> IdCounter(1);
65
66/// Used to create a new correlation id
67static uint64_t createId() { return IdCounter.fetch_add(1); }
68
69/// Create a new correlation id and update the operations id
70static uint64_t createOpId() {
71 uint64_t NewId = createId();
72 RegionInterface.setHostOpId(NewId);
73 return NewId;
74}
75
76/// Create a new correlation id and update the target region id
77static uint64_t createRegionId() {
78 uint64_t NewId = createId();
79 RegionInterface.setTargetDataValue(NewId);
80 return NewId;
81}
82
83void Interface::beginTargetDataAlloc(int64_t DeviceId, void *HstPtrBegin,
84 void **TgtPtrBegin, size_t Size,
85 void *Code) {
86 beginTargetDataOperation();
87 if (ompt_callback_target_data_op_emi_fn) {
88 // HostOpId will be set by the tool. Invoke the tool supplied data op EMI
89 // callback
90 ompt_callback_target_data_op_emi_fn(
91 ompt_scope_begin, TargetTaskData, &TargetData, &HostOpId,
92 ompt_target_data_alloc, HstPtrBegin,
93 /*SrcDeviceNum=*/omp_get_initial_device(), *TgtPtrBegin,
94 /*TgtDeviceNum=*/DeviceId, Size, Code);
95 } else if (ompt_callback_target_data_op_fn) {
96 // HostOpId is set by the runtime
97 HostOpId = createOpId();
98 // Invoke the tool supplied data op callback
99 ompt_callback_target_data_op_fn(
100 TargetData.value, HostOpId, ompt_target_data_alloc, HstPtrBegin,
101 /*SrcDeviceNum=*/omp_get_initial_device(), *TgtPtrBegin,
102 /*TgtDeviceNum=*/DeviceId, Size, Code);
103 }
104}
105
106void Interface::endTargetDataAlloc(int64_t DeviceId, void *HstPtrBegin,
107 void **TgtPtrBegin, size_t Size,
108 void *Code) {
109 // Only EMI callback handles end scope
110 if (ompt_callback_target_data_op_emi_fn) {
111 // HostOpId will be set by the tool. Invoke the tool supplied data op EMI
112 // callback
113 ompt_callback_target_data_op_emi_fn(
114 ompt_scope_end, TargetTaskData, &TargetData, &HostOpId,
115 ompt_target_data_alloc, HstPtrBegin,
116 /*SrcDeviceNum=*/omp_get_initial_device(), *TgtPtrBegin,
117 /*TgtDeviceNum=*/DeviceId, Size, Code);
118 }
119 endTargetDataOperation();
120}
121
122void Interface::beginTargetDataSubmit(int64_t SrcDeviceId, void *SrcPtrBegin,
123 int64_t DstDeviceId, void *DstPtrBegin,
124 size_t Size, void *Code) {
125 beginTargetDataOperation();
126 if (ompt_callback_target_data_op_emi_fn) {
127 // HostOpId will be set by the tool. Invoke the tool supplied data op EMI
128 // callback
129 ompt_callback_target_data_op_emi_fn(
130 ompt_scope_begin, TargetTaskData, &TargetData, &HostOpId,
131 ompt_target_data_transfer_to_device, SrcPtrBegin, SrcDeviceId,
132 DstPtrBegin, DstDeviceId, Size, Code);
133 } else if (ompt_callback_target_data_op_fn) {
134 // HostOpId is set by the runtime
135 HostOpId = createOpId();
136 // Invoke the tool supplied data op callback
137 ompt_callback_target_data_op_fn(
138 TargetData.value, HostOpId, ompt_target_data_transfer_to_device,
139 SrcPtrBegin, SrcDeviceId, DstPtrBegin, DstDeviceId, Size, Code);
140 }
141}
142
143void Interface::endTargetDataSubmit(int64_t SrcDeviceId, void *SrcPtrBegin,
144 int64_t DstDeviceId, void *DstPtrBegin,
145 size_t Size, void *Code) {
146 // Only EMI callback handles end scope
147 if (ompt_callback_target_data_op_emi_fn) {
148 // HostOpId will be set by the tool. Invoke the tool supplied data op EMI
149 // callback
150 ompt_callback_target_data_op_emi_fn(
151 ompt_scope_end, TargetTaskData, &TargetData, &HostOpId,
152 ompt_target_data_transfer_to_device, SrcPtrBegin, SrcDeviceId,
153 DstPtrBegin, DstDeviceId, Size, Code);
154 }
155 endTargetDataOperation();
156}
157
158void Interface::beginTargetDataDelete(int64_t DeviceId, void *TgtPtrBegin,
159 void *Code) {
160 beginTargetDataOperation();
161 if (ompt_callback_target_data_op_emi_fn) {
162 // HostOpId will be set by the tool. Invoke the tool supplied data op EMI
163 // callback
164 ompt_callback_target_data_op_emi_fn(
165 ompt_scope_begin, TargetTaskData, &TargetData, &HostOpId,
166 ompt_target_data_delete, TgtPtrBegin, DeviceId,
167 /*TgtPtrBegin=*/nullptr, /*TgtDeviceNum=*/-1, /*Bytes=*/0, Code);
168 } else if (ompt_callback_target_data_op_fn) {
169 // HostOpId is set by the runtime
170 HostOpId = createOpId();
171 // Invoke the tool supplied data op callback
172 ompt_callback_target_data_op_fn(TargetData.value, HostOpId,
173 ompt_target_data_delete, TgtPtrBegin,
174 DeviceId, /*TgtPtrBegin=*/nullptr,
175 /*TgtDeviceNum=*/-1, /*Bytes=*/0, Code);
176 }
177}
178
179void Interface::endTargetDataDelete(int64_t DeviceId, void *TgtPtrBegin,
180 void *Code) {
181 // Only EMI callback handles end scope
182 if (ompt_callback_target_data_op_emi_fn) {
183 // HostOpId will be set by the tool. Invoke the tool supplied data op EMI
184 // callback
185 ompt_callback_target_data_op_emi_fn(
186 ompt_scope_end, TargetTaskData, &TargetData, &HostOpId,
187 ompt_target_data_delete, TgtPtrBegin, DeviceId,
188 /*TgtPtrBegin=*/nullptr, /*TgtDeviceNum=*/-1, /*Bytes=*/0, Code);
189 }
190 endTargetDataOperation();
191}
192
193void Interface::beginTargetDataRetrieve(int64_t SrcDeviceId, void *SrcPtrBegin,
194 int64_t DstDeviceId, void *DstPtrBegin,
195 size_t Size, void *Code) {
196 beginTargetDataOperation();
197 if (ompt_callback_target_data_op_emi_fn) {
198 // HostOpId will be set by the tool. Invoke the tool supplied data op EMI
199 // callback
200 ompt_callback_target_data_op_emi_fn(
201 ompt_scope_begin, TargetTaskData, &TargetData, &HostOpId,
202 ompt_target_data_transfer_from_device, SrcPtrBegin, SrcDeviceId,
203 DstPtrBegin, DstDeviceId, Size, Code);
204 } else if (ompt_callback_target_data_op_fn) {
205 // HostOpId is set by the runtime
206 HostOpId = createOpId();
207 // Invoke the tool supplied data op callback
208 ompt_callback_target_data_op_fn(
209 TargetData.value, HostOpId, ompt_target_data_transfer_from_device,
210 SrcPtrBegin, SrcDeviceId, DstPtrBegin, DstDeviceId, Size, Code);
211 }
212}
213
214void Interface::endTargetDataRetrieve(int64_t SrcDeviceId, void *SrcPtrBegin,
215 int64_t DstDeviceId, void *DstPtrBegin,
216 size_t Size, void *Code) {
217 // Only EMI callback handles end scope
218 if (ompt_callback_target_data_op_emi_fn) {
219 // HostOpId will be set by the tool. Invoke the tool supplied data op EMI
220 // callback
221 ompt_callback_target_data_op_emi_fn(
222 ompt_scope_end, TargetTaskData, &TargetData, &HostOpId,
223 ompt_target_data_transfer_from_device, SrcPtrBegin, SrcDeviceId,
224 DstPtrBegin, DstDeviceId, Size, Code);
225 }
226 endTargetDataOperation();
227}
228
229void Interface::beginTargetSubmit(unsigned int numTeams) {
230 if (ompt_callback_target_submit_emi_fn) {
231 // HostOpId is set by the tool. Invoke the tool supplied target submit EMI
232 // callback
233 ompt_callback_target_submit_emi_fn(ompt_scope_begin, &TargetData, &HostOpId,
234 numTeams);
235 } else if (ompt_callback_target_submit_fn) {
236 // HostOpId is set by the runtime
237 HostOpId = createOpId();
238 ompt_callback_target_submit_fn(TargetData.value, HostOpId, numTeams);
239 }
240}
241
242void Interface::endTargetSubmit(unsigned int numTeams) {
243 // Only EMI callback handles end scope
244 if (ompt_callback_target_submit_emi_fn) {
245 // HostOpId is set by the tool. Invoke the tool supplied target submit EMI
246 // callback
247 ompt_callback_target_submit_emi_fn(ompt_scope_end, &TargetData, &HostOpId,
248 numTeams);
249 }
250}
251
252void Interface::beginTargetDataEnter(int64_t DeviceId, void *Code) {
253 beginTargetRegion();
254 if (ompt_callback_target_emi_fn) {
255 // Invoke the tool supplied target EMI callback
256 ompt_callback_target_emi_fn(ompt_target_enter_data, ompt_scope_begin,
257 DeviceId, TaskData, TargetTaskData, &TargetData,
258 Code);
259 } else if (ompt_callback_target_fn) {
260 // Invoke the tool supplied target callback
261 ompt_callback_target_fn(ompt_target_enter_data, ompt_scope_begin, DeviceId,
262 TaskData, TargetData.value, Code);
263 }
264}
265
266void Interface::endTargetDataEnter(int64_t DeviceId, void *Code) {
267 if (ompt_callback_target_emi_fn) {
268 // Invoke the tool supplied target EMI callback
269 ompt_callback_target_emi_fn(ompt_target_enter_data, ompt_scope_end,
270 DeviceId, TaskData, TargetTaskData, &TargetData,
271 Code);
272 } else if (ompt_callback_target_fn) {
273 // Invoke the tool supplied target callback
274 ompt_callback_target_fn(ompt_target_enter_data, ompt_scope_end, DeviceId,
275 TaskData, TargetData.value, Code);
276 }
277 endTargetRegion();
278}
279
280void Interface::beginTargetDataExit(int64_t DeviceId, void *Code) {
281 beginTargetRegion();
282 if (ompt_callback_target_emi_fn) {
283 // Invoke the tool supplied target EMI callback
284 ompt_callback_target_emi_fn(ompt_target_exit_data, ompt_scope_begin,
285 DeviceId, TaskData, TargetTaskData, &TargetData,
286 Code);
287 } else if (ompt_callback_target_fn) {
288 TargetData.value = createRegionId();
289 // Invoke the tool supplied target callback
290 ompt_callback_target_fn(ompt_target_exit_data, ompt_scope_begin, DeviceId,
291 TaskData, TargetData.value, Code);
292 }
293}
294
295void Interface::endTargetDataExit(int64_t DeviceId, void *Code) {
296 if (ompt_callback_target_emi_fn) {
297 // Invoke the tool supplied target EMI callback
298 ompt_callback_target_emi_fn(ompt_target_exit_data, ompt_scope_end, DeviceId,
299 TaskData, TargetTaskData, &TargetData, Code);
300 } else if (ompt_callback_target_fn) {
301 // Invoke the tool supplied target callback
302 ompt_callback_target_fn(ompt_target_exit_data, ompt_scope_end, DeviceId,
303 TaskData, TargetData.value, Code);
304 }
305 endTargetRegion();
306}
307
308void Interface::beginTargetUpdate(int64_t DeviceId, void *Code) {
309 beginTargetRegion();
310 if (ompt_callback_target_emi_fn) {
311 // Invoke the tool supplied target EMI callback
312 ompt_callback_target_emi_fn(ompt_target_update, ompt_scope_begin, DeviceId,
313 TaskData, TargetTaskData, &TargetData, Code);
314 } else if (ompt_callback_target_fn) {
315 TargetData.value = createRegionId();
316 // Invoke the tool supplied target callback
317 ompt_callback_target_fn(ompt_target_update, ompt_scope_begin, DeviceId,
318 TaskData, TargetData.value, Code);
319 }
320}
321
322void Interface::endTargetUpdate(int64_t DeviceId, void *Code) {
323 if (ompt_callback_target_emi_fn) {
324 // Invoke the tool supplied target EMI callback
325 ompt_callback_target_emi_fn(ompt_target_update, ompt_scope_end, DeviceId,
326 TaskData, TargetTaskData, &TargetData, Code);
327 } else if (ompt_callback_target_fn) {
328 // Invoke the tool supplied target callback
329 ompt_callback_target_fn(ompt_target_update, ompt_scope_end, DeviceId,
330 TaskData, TargetData.value, Code);
331 }
332 endTargetRegion();
333}
334
335void Interface::beginTarget(int64_t DeviceId, void *Code) {
336 beginTargetRegion();
337 if (ompt_callback_target_emi_fn) {
338 // Invoke the tool supplied target EMI callback
339 ompt_callback_target_emi_fn(ompt_target, ompt_scope_begin, DeviceId,
340 TaskData, TargetTaskData, &TargetData, Code);
341 } else if (ompt_callback_target_fn) {
342 TargetData.value = createRegionId();
343 // Invoke the tool supplied target callback
344 ompt_callback_target_fn(ompt_target, ompt_scope_begin, DeviceId, TaskData,
345 TargetData.value, Code);
346 }
347}
348
349void Interface::endTarget(int64_t DeviceId, void *Code) {
350 if (ompt_callback_target_emi_fn) {
351 // Invoke the tool supplied target EMI callback
352 ompt_callback_target_emi_fn(ompt_target, ompt_scope_end, DeviceId, TaskData,
353 TargetTaskData, &TargetData, Code);
354 } else if (ompt_callback_target_fn) {
355 // Invoke the tool supplied target callback
356 ompt_callback_target_fn(ompt_target, ompt_scope_end, DeviceId, TaskData,
357 TargetData.value, Code);
358 }
359 endTargetRegion();
360}
361
362void Interface::beginTargetDataOperation() {
363 DP("in ompt_target_region_begin (TargetRegionId = %lu)\n", TargetData.value);
364}
365
366void Interface::endTargetDataOperation() {
367 DP("in ompt_target_region_end (TargetRegionId = %lu)\n", TargetData.value);
368}
369
370void Interface::beginTargetRegion() {
371 // Set up task state
372 assert(ompt_get_task_data_fn && "Calling a null task data function");
373 TaskData = ompt_get_task_data_fn();
374 // Set up target task state
375 assert(ompt_get_target_task_data_fn &&
376 "Calling a null target task data function");
377 TargetTaskData = ompt_get_target_task_data_fn();
378 // Target state will be set later
379 TargetData = ompt_data_none;
380}
381
382void Interface::endTargetRegion() {
383 TaskData = 0;
384 TargetTaskData = 0;
385 TargetData = ompt_data_none;
386}
387
388/// Used to maintain the finalization functions that are received
389/// from the plugins during connect.
390/// Note: Currently, there are no plugin-specific finalizations, so each plugin
391/// will call the same (empty) function.
392class LibomptargetRtlFinalizer {
393public:
394 LibomptargetRtlFinalizer() {}
395
396 void registerRtl(ompt_finalize_t FinalizationFunction) {
397 if (FinalizationFunction) {
398 RtlFinalizationFunctions.emplace_back(FinalizationFunction);
399 }
400 }
401
402 void finalize() {
403 for (auto FinalizationFunction : RtlFinalizationFunctions)
404 FinalizationFunction(/* tool_data */ nullptr);
405 RtlFinalizationFunctions.clear();
406 }
407
408private:
409 llvm::SmallVector<ompt_finalize_t> RtlFinalizationFunctions;
410};
411
412int llvm::omp::target::ompt::initializeLibrary(ompt_function_lookup_t lookup,
413 int initial_device_num,
414 ompt_data_t *tool_data) {
415 DP("Executing initializeLibrary (libomp)\n");
416#define bindOmptFunctionName(OmptFunction, DestinationFunction) \
417 DestinationFunction = (OmptFunction##_t)lookup(#OmptFunction); \
418 DP("initializeLibrary (libomp) bound %s=%p\n", #DestinationFunction, \
419 ((void *)(uint64_t)DestinationFunction));
420
421 bindOmptFunctionName(ompt_get_callback, lookupCallbackByCode);
422 bindOmptFunctionName(ompt_get_task_data, ompt_get_task_data_fn);
423 bindOmptFunctionName(ompt_get_target_task_data, ompt_get_target_task_data_fn);
424#undef bindOmptFunctionName
425
426 // Store pointer of 'ompt_libomp_target_fn_lookup' for use by libomptarget
427 lookupCallbackByName = lookup;
428
429 assert(lookupCallbackByCode && "lookupCallbackByCode should be non-null");
430 assert(lookupCallbackByName && "lookupCallbackByName should be non-null");
431 assert(ompt_get_task_data_fn && "ompt_get_task_data_fn should be non-null");
432 assert(ompt_get_target_task_data_fn &&
433 "ompt_get_target_task_data_fn should be non-null");
434 assert(LibraryFinalizer == nullptr &&
435 "LibraryFinalizer should not be initialized yet");
436
437 LibraryFinalizer = new LibomptargetRtlFinalizer();
438
439 Initialized = true;
440
441 return 0;
442}
443
444void llvm::omp::target::ompt::finalizeLibrary(ompt_data_t *data) {
445 DP("Executing finalizeLibrary (libomp)\n");
446 // Before disabling OMPT, call the (plugin) finalizations that were registered
447 // with this library
448 LibraryFinalizer->finalize();
449 delete LibraryFinalizer;
450 Initialized = false;
451}
452
453void llvm::omp::target::ompt::connectLibrary() {
454 DP("Entering connectLibrary (libomp)\n");
455 // Connect with libomp
456 static OmptLibraryConnectorTy LibompConnector("libomp");
457 static ompt_start_tool_result_t OmptResult;
458
459 // Initialize OmptResult with the init and fini functions that will be
460 // called by the connector
461 OmptResult.initialize = ompt::initializeLibrary;
462 OmptResult.finalize = ompt::finalizeLibrary;
463 OmptResult.tool_data.value = 0;
464
465 // Now call connect that causes the above init/fini functions to be called
466 LibompConnector.connect(&OmptResult);
467
468#define bindOmptCallback(Name, Type, Code) \
469 if (lookupCallbackByCode) \
470 lookupCallbackByCode( \
471 (ompt_callbacks_t)(Code), \
472 (ompt_callback_t *)&(llvm::omp::target::ompt::Name##_fn));
473 FOREACH_OMPT_NOEMI_EVENT(bindOmptCallback)
474 FOREACH_OMPT_EMI_EVENT(bindOmptCallback)
475#undef bindOmptCallback
476
477 DP("Exiting connectLibrary (libomp)\n");
478}
479
480extern "C" {
481/// Used for connecting libomptarget with a plugin
482void ompt_libomptarget_connect(ompt_start_tool_result_t *result) {
483 DP("Enter ompt_libomptarget_connect\n");
484 if (Initialized && result && LibraryFinalizer) {
485 // Cache each fini function, so that they can be invoked on exit
486 LibraryFinalizer->registerRtl(result->finalize);
487 // Invoke the provided init function with the lookup function maintained
488 // in this library so that callbacks maintained by this library are
489 // retrieved.
490 result->initialize(lookupCallbackByName,
491 /* initial_device_num */ 0, /* tool_data */ nullptr);
492 }
493 DP("Leave ompt_libomptarget_connect\n");
494}
495}
496#endif // OMPT_SUPPORT
497

source code of offload/src/OpenMP/OMPT/Callback.cpp