1//===-- OpenMP/OMPT/Callback.cpp - OpenMP Tooling Callback implementation -===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Implementation of OMPT callback interfaces for target independent layer
10//
11//===----------------------------------------------------------------------===//
12
13#ifdef OMPT_SUPPORT
14
15#include <cstdlib>
16#include <cstring>
17#include <memory>
18
19#include "Shared/Debug.h"
20
21#include "OpenMP/OMPT/Callback.h"
22#include "OpenMP/OMPT/Connector.h"
23#include "OpenMP/OMPT/Interface.h"
24
25#include "llvm/Support/DynamicLibrary.h"
26
27#undef DEBUG_PREFIX
28#define DEBUG_PREFIX "OMPT"
29
30// Define OMPT callback functions (bound to actual callbacks later on)
31#define defineOmptCallback(Name, Type, Code) \
32 Name##_t llvm::omp::target::ompt::Name##_fn = nullptr;
33FOREACH_OMPT_NOEMI_EVENT(defineOmptCallback)
34FOREACH_OMPT_EMI_EVENT(defineOmptCallback)
35#undef defineOmptCallback
36
37using namespace llvm::omp::target::ompt;
38
39/// Forward declaration
40class LibomptargetRtlFinalizer;
41
42/// Object that will maintain the RTL finalizer from the plugin
43LibomptargetRtlFinalizer *LibraryFinalizer = nullptr;
44
45thread_local Interface llvm::omp::target::ompt::RegionInterface;
46
47thread_local void *llvm::omp::target::ompt::ReturnAddress = nullptr;
48
49bool llvm::omp::target::ompt::Initialized = false;
50
51ompt_get_callback_t llvm::omp::target::ompt::lookupCallbackByCode = nullptr;
52ompt_function_lookup_t llvm::omp::target::ompt::lookupCallbackByName = nullptr;
53ompt_get_target_task_data_t ompt_get_target_task_data_fn = nullptr;
54ompt_get_task_data_t ompt_get_task_data_fn = nullptr;
55
56/// Unique correlation id
57static std::atomic<uint64_t> IdCounter(1);
58
59/// Used to create a new correlation id
60static uint64_t createId() { return IdCounter.fetch_add(1); }
61
62/// Create a new correlation id and update the operations id
63static uint64_t createOpId() {
64 uint64_t NewId = createId();
65 RegionInterface.setHostOpId(NewId);
66 return NewId;
67}
68
69/// Create a new correlation id and update the target region id
70static uint64_t createRegionId() {
71 uint64_t NewId = createId();
72 RegionInterface.setTargetDataValue(NewId);
73 return NewId;
74}
75
76void Interface::beginTargetDataAlloc(int64_t DeviceId, void *HstPtrBegin,
77 void **TgtPtrBegin, size_t Size,
78 void *Code) {
79 beginTargetDataOperation();
80 if (ompt_callback_target_data_op_emi_fn) {
81 // HostOpId will be set by the tool. Invoke the tool supplied data op EMI
82 // callback
83 ompt_callback_target_data_op_emi_fn(
84 ompt_scope_begin, TargetTaskData, &TargetData, &HostOpId,
85 ompt_target_data_alloc, HstPtrBegin,
86 /*SrcDeviceNum=*/omp_get_initial_device(), *TgtPtrBegin,
87 /*TgtDeviceNum=*/DeviceId, Size, Code);
88 } else if (ompt_callback_target_data_op_fn) {
89 // HostOpId is set by the runtime
90 HostOpId = createOpId();
91 // Invoke the tool supplied data op callback
92 ompt_callback_target_data_op_fn(
93 TargetData.value, HostOpId, ompt_target_data_alloc, HstPtrBegin,
94 /*SrcDeviceNum=*/omp_get_initial_device(), *TgtPtrBegin,
95 /*TgtDeviceNum=*/DeviceId, Size, Code);
96 }
97}
98
99void Interface::endTargetDataAlloc(int64_t DeviceId, void *HstPtrBegin,
100 void **TgtPtrBegin, size_t Size,
101 void *Code) {
102 // Only EMI callback handles end scope
103 if (ompt_callback_target_data_op_emi_fn) {
104 // HostOpId will be set by the tool. Invoke the tool supplied data op EMI
105 // callback
106 ompt_callback_target_data_op_emi_fn(
107 ompt_scope_end, TargetTaskData, &TargetData, &HostOpId,
108 ompt_target_data_alloc, HstPtrBegin,
109 /*SrcDeviceNum=*/omp_get_initial_device(), *TgtPtrBegin,
110 /*TgtDeviceNum=*/DeviceId, Size, Code);
111 }
112 endTargetDataOperation();
113}
114
115void Interface::beginTargetDataSubmit(int64_t SrcDeviceId, void *SrcPtrBegin,
116 int64_t DstDeviceId, void *DstPtrBegin,
117 size_t Size, void *Code) {
118 beginTargetDataOperation();
119 if (ompt_callback_target_data_op_emi_fn) {
120 // HostOpId will be set by the tool. Invoke the tool supplied data op EMI
121 // callback
122 ompt_callback_target_data_op_emi_fn(
123 ompt_scope_begin, TargetTaskData, &TargetData, &HostOpId,
124 ompt_target_data_transfer_to_device, SrcPtrBegin, SrcDeviceId,
125 DstPtrBegin, DstDeviceId, Size, Code);
126 } else if (ompt_callback_target_data_op_fn) {
127 // HostOpId is set by the runtime
128 HostOpId = createOpId();
129 // Invoke the tool supplied data op callback
130 ompt_callback_target_data_op_fn(
131 TargetData.value, HostOpId, ompt_target_data_transfer_to_device,
132 SrcPtrBegin, SrcDeviceId, DstPtrBegin, DstDeviceId, Size, Code);
133 }
134}
135
136void Interface::endTargetDataSubmit(int64_t SrcDeviceId, void *SrcPtrBegin,
137 int64_t DstDeviceId, void *DstPtrBegin,
138 size_t Size, void *Code) {
139 // Only EMI callback handles end scope
140 if (ompt_callback_target_data_op_emi_fn) {
141 // HostOpId will be set by the tool. Invoke the tool supplied data op EMI
142 // callback
143 ompt_callback_target_data_op_emi_fn(
144 ompt_scope_end, TargetTaskData, &TargetData, &HostOpId,
145 ompt_target_data_transfer_to_device, SrcPtrBegin, SrcDeviceId,
146 DstPtrBegin, DstDeviceId, Size, Code);
147 }
148 endTargetDataOperation();
149}
150
151void Interface::beginTargetDataDelete(int64_t DeviceId, void *TgtPtrBegin,
152 void *Code) {
153 beginTargetDataOperation();
154 if (ompt_callback_target_data_op_emi_fn) {
155 // HostOpId will be set by the tool. Invoke the tool supplied data op EMI
156 // callback
157 ompt_callback_target_data_op_emi_fn(
158 ompt_scope_begin, TargetTaskData, &TargetData, &HostOpId,
159 ompt_target_data_delete, TgtPtrBegin, DeviceId,
160 /*TgtPtrBegin=*/nullptr, /*TgtDeviceNum=*/-1, /*Bytes=*/0, Code);
161 } else if (ompt_callback_target_data_op_fn) {
162 // HostOpId is set by the runtime
163 HostOpId = createOpId();
164 // Invoke the tool supplied data op callback
165 ompt_callback_target_data_op_fn(TargetData.value, HostOpId,
166 ompt_target_data_delete, TgtPtrBegin,
167 DeviceId, /*TgtPtrBegin=*/nullptr,
168 /*TgtDeviceNum=*/-1, /*Bytes=*/0, Code);
169 }
170}
171
172void Interface::endTargetDataDelete(int64_t DeviceId, void *TgtPtrBegin,
173 void *Code) {
174 // Only EMI callback handles end scope
175 if (ompt_callback_target_data_op_emi_fn) {
176 // HostOpId will be set by the tool. Invoke the tool supplied data op EMI
177 // callback
178 ompt_callback_target_data_op_emi_fn(
179 ompt_scope_end, TargetTaskData, &TargetData, &HostOpId,
180 ompt_target_data_delete, TgtPtrBegin, DeviceId,
181 /*TgtPtrBegin=*/nullptr, /*TgtDeviceNum=*/-1, /*Bytes=*/0, Code);
182 }
183 endTargetDataOperation();
184}
185
186void Interface::beginTargetDataRetrieve(int64_t SrcDeviceId, void *SrcPtrBegin,
187 int64_t DstDeviceId, void *DstPtrBegin,
188 size_t Size, void *Code) {
189 beginTargetDataOperation();
190 if (ompt_callback_target_data_op_emi_fn) {
191 // HostOpId will be set by the tool. Invoke the tool supplied data op EMI
192 // callback
193 ompt_callback_target_data_op_emi_fn(
194 ompt_scope_begin, TargetTaskData, &TargetData, &HostOpId,
195 ompt_target_data_transfer_from_device, SrcPtrBegin, SrcDeviceId,
196 DstPtrBegin, DstDeviceId, Size, Code);
197 } else if (ompt_callback_target_data_op_fn) {
198 // HostOpId is set by the runtime
199 HostOpId = createOpId();
200 // Invoke the tool supplied data op callback
201 ompt_callback_target_data_op_fn(
202 TargetData.value, HostOpId, ompt_target_data_transfer_from_device,
203 SrcPtrBegin, SrcDeviceId, DstPtrBegin, DstDeviceId, Size, Code);
204 }
205}
206
207void Interface::endTargetDataRetrieve(int64_t SrcDeviceId, void *SrcPtrBegin,
208 int64_t DstDeviceId, void *DstPtrBegin,
209 size_t Size, void *Code) {
210 // Only EMI callback handles end scope
211 if (ompt_callback_target_data_op_emi_fn) {
212 // HostOpId will be set by the tool. Invoke the tool supplied data op EMI
213 // callback
214 ompt_callback_target_data_op_emi_fn(
215 ompt_scope_end, TargetTaskData, &TargetData, &HostOpId,
216 ompt_target_data_transfer_from_device, SrcPtrBegin, SrcDeviceId,
217 DstPtrBegin, DstDeviceId, Size, Code);
218 }
219 endTargetDataOperation();
220}
221
222void Interface::beginTargetSubmit(unsigned int NumTeams) {
223 if (ompt_callback_target_submit_emi_fn) {
224 // HostOpId is set by the tool. Invoke the tool supplied target submit EMI
225 // callback
226 ompt_callback_target_submit_emi_fn(ompt_scope_begin, &TargetData, &HostOpId,
227 NumTeams);
228 } else if (ompt_callback_target_submit_fn) {
229 // HostOpId is set by the runtime
230 HostOpId = createOpId();
231 ompt_callback_target_submit_fn(TargetData.value, HostOpId, NumTeams);
232 }
233}
234
235void Interface::endTargetSubmit(unsigned int NumTeams) {
236 // Only EMI callback handles end scope
237 if (ompt_callback_target_submit_emi_fn) {
238 // HostOpId is set by the tool. Invoke the tool supplied target submit EMI
239 // callback
240 ompt_callback_target_submit_emi_fn(ompt_scope_end, &TargetData, &HostOpId,
241 NumTeams);
242 }
243}
244
245void Interface::beginTargetDataEnter(int64_t DeviceId, void *Code) {
246 beginTargetRegion();
247 if (ompt_callback_target_emi_fn) {
248 // Invoke the tool supplied target EMI callback
249 ompt_callback_target_emi_fn(ompt_target_enter_data, ompt_scope_begin,
250 DeviceId, TaskData, TargetTaskData, &TargetData,
251 Code);
252 } else if (ompt_callback_target_fn) {
253 // Invoke the tool supplied target callback
254 ompt_callback_target_fn(ompt_target_enter_data, ompt_scope_begin, DeviceId,
255 TaskData, TargetData.value, Code);
256 }
257}
258
259void Interface::endTargetDataEnter(int64_t DeviceId, void *Code) {
260 if (ompt_callback_target_emi_fn) {
261 // Invoke the tool supplied target EMI callback
262 ompt_callback_target_emi_fn(ompt_target_enter_data, ompt_scope_end,
263 DeviceId, TaskData, TargetTaskData, &TargetData,
264 Code);
265 } else if (ompt_callback_target_fn) {
266 // Invoke the tool supplied target callback
267 ompt_callback_target_fn(ompt_target_enter_data, ompt_scope_end, DeviceId,
268 TaskData, TargetData.value, Code);
269 }
270 endTargetRegion();
271}
272
273void Interface::beginTargetDataExit(int64_t DeviceId, void *Code) {
274 beginTargetRegion();
275 if (ompt_callback_target_emi_fn) {
276 // Invoke the tool supplied target EMI callback
277 ompt_callback_target_emi_fn(ompt_target_exit_data, ompt_scope_begin,
278 DeviceId, TaskData, TargetTaskData, &TargetData,
279 Code);
280 } else if (ompt_callback_target_fn) {
281 TargetData.value = createRegionId();
282 // Invoke the tool supplied target callback
283 ompt_callback_target_fn(ompt_target_exit_data, ompt_scope_begin, DeviceId,
284 TaskData, TargetData.value, Code);
285 }
286}
287
288void Interface::endTargetDataExit(int64_t DeviceId, void *Code) {
289 if (ompt_callback_target_emi_fn) {
290 // Invoke the tool supplied target EMI callback
291 ompt_callback_target_emi_fn(ompt_target_exit_data, ompt_scope_end, DeviceId,
292 TaskData, TargetTaskData, &TargetData, Code);
293 } else if (ompt_callback_target_fn) {
294 // Invoke the tool supplied target callback
295 ompt_callback_target_fn(ompt_target_exit_data, ompt_scope_end, DeviceId,
296 TaskData, TargetData.value, Code);
297 }
298 endTargetRegion();
299}
300
301void Interface::beginTargetUpdate(int64_t DeviceId, void *Code) {
302 beginTargetRegion();
303 if (ompt_callback_target_emi_fn) {
304 // Invoke the tool supplied target EMI callback
305 ompt_callback_target_emi_fn(ompt_target_update, ompt_scope_begin, DeviceId,
306 TaskData, TargetTaskData, &TargetData, Code);
307 } else if (ompt_callback_target_fn) {
308 TargetData.value = createRegionId();
309 // Invoke the tool supplied target callback
310 ompt_callback_target_fn(ompt_target_update, ompt_scope_begin, DeviceId,
311 TaskData, TargetData.value, Code);
312 }
313}
314
315void Interface::endTargetUpdate(int64_t DeviceId, void *Code) {
316 if (ompt_callback_target_emi_fn) {
317 // Invoke the tool supplied target EMI callback
318 ompt_callback_target_emi_fn(ompt_target_update, ompt_scope_end, DeviceId,
319 TaskData, TargetTaskData, &TargetData, Code);
320 } else if (ompt_callback_target_fn) {
321 // Invoke the tool supplied target callback
322 ompt_callback_target_fn(ompt_target_update, ompt_scope_end, DeviceId,
323 TaskData, TargetData.value, Code);
324 }
325 endTargetRegion();
326}
327
328void Interface::beginTargetAssociatePointer(int64_t DeviceId, void *HstPtrBegin,
329 void *TgtPtrBegin, size_t Size,
330 void *Code) {
331 beginTargetDataOperation();
332 if (ompt_callback_target_data_op_emi_fn) {
333 ompt_callback_target_data_op_emi_fn(
334 ompt_scope_begin, TargetTaskData, &TargetData, &HostOpId,
335 ompt_target_data_associate, HstPtrBegin, omp_get_initial_device(),
336 TgtPtrBegin, DeviceId, Size, Code);
337 } else if (ompt_callback_target_data_op_fn) {
338 HostOpId = createOpId();
339 ompt_callback_target_data_op_fn(
340 TargetData.value, HostOpId, ompt_target_data_associate, HstPtrBegin,
341 omp_get_initial_device(), TgtPtrBegin, DeviceId, Size, Code);
342 }
343}
344
345void Interface::endTargetAssociatePointer(int64_t DeviceId, void *HstPtrBegin,
346 void *TgtPtrBegin, size_t Size,
347 void *Code) {
348 if (ompt_callback_target_data_op_emi_fn) {
349 ompt_callback_target_data_op_emi_fn(
350 ompt_scope_end, TargetTaskData, &TargetData, &HostOpId,
351 ompt_target_data_associate, HstPtrBegin, omp_get_initial_device(),
352 TgtPtrBegin, DeviceId, Size, Code);
353 }
354}
355
356void Interface::beginTargetDisassociatePointer(int64_t DeviceId,
357 void *HstPtrBegin,
358 void *TgtPtrBegin, size_t Size,
359 void *Code) {
360 beginTargetDataOperation();
361 if (ompt_callback_target_data_op_emi_fn) {
362 ompt_callback_target_data_op_emi_fn(
363 ompt_scope_begin, TargetTaskData, &TargetData, &HostOpId,
364 ompt_target_data_disassociate, HstPtrBegin, omp_get_initial_device(),
365 TgtPtrBegin, DeviceId, Size, Code);
366 } else if (ompt_callback_target_data_op_fn) {
367 HostOpId = createOpId();
368 ompt_callback_target_data_op_fn(
369 TargetData.value, HostOpId, ompt_target_data_disassociate, HstPtrBegin,
370 omp_get_initial_device(), TgtPtrBegin, DeviceId, Size, Code);
371 }
372}
373void Interface::endTargetDisassociatePointer(int64_t DeviceId,
374 void *HstPtrBegin,
375 void *TgtPtrBegin, size_t Size,
376 void *Code) {
377 if (ompt_callback_target_data_op_emi_fn) {
378 ompt_callback_target_data_op_emi_fn(
379 ompt_scope_end, TargetTaskData, &TargetData, &HostOpId,
380 ompt_target_data_disassociate, HstPtrBegin, omp_get_initial_device(),
381 TgtPtrBegin, DeviceId, Size, Code);
382 }
383}
384
385void Interface::beginTarget(int64_t DeviceId, void *Code) {
386 beginTargetRegion();
387 if (ompt_callback_target_emi_fn) {
388 // Invoke the tool supplied target EMI callback
389 ompt_callback_target_emi_fn(ompt_target, ompt_scope_begin, DeviceId,
390 TaskData, TargetTaskData, &TargetData, Code);
391 } else if (ompt_callback_target_fn) {
392 TargetData.value = createRegionId();
393 // Invoke the tool supplied target callback
394 ompt_callback_target_fn(ompt_target, ompt_scope_begin, DeviceId, TaskData,
395 TargetData.value, Code);
396 }
397}
398
399void Interface::endTarget(int64_t DeviceId, void *Code) {
400 if (ompt_callback_target_emi_fn) {
401 // Invoke the tool supplied target EMI callback
402 ompt_callback_target_emi_fn(ompt_target, ompt_scope_end, DeviceId, TaskData,
403 TargetTaskData, &TargetData, Code);
404 } else if (ompt_callback_target_fn) {
405 // Invoke the tool supplied target callback
406 ompt_callback_target_fn(ompt_target, ompt_scope_end, DeviceId, TaskData,
407 TargetData.value, Code);
408 }
409 endTargetRegion();
410}
411
412void Interface::beginTargetDataOperation() {
413 DP("in ompt_target_region_begin (TargetRegionId = %lu)\n", TargetData.value);
414}
415
416void Interface::endTargetDataOperation() {
417 DP("in ompt_target_region_end (TargetRegionId = %lu)\n", TargetData.value);
418}
419
420void Interface::beginTargetRegion() {
421 // Set up task state
422 assert(ompt_get_task_data_fn && "Calling a null task data function");
423 TaskData = ompt_get_task_data_fn();
424 // Set up target task state
425 assert(ompt_get_target_task_data_fn &&
426 "Calling a null target task data function");
427 TargetTaskData = ompt_get_target_task_data_fn();
428 // Target state will be set later
429 TargetData = ompt_data_none;
430}
431
432void Interface::endTargetRegion() {
433 TaskData = 0;
434 TargetTaskData = 0;
435 TargetData = ompt_data_none;
436}
437
438/// Used to maintain the finalization functions that are received
439/// from the plugins during connect.
440/// Note: Currently, there are no plugin-specific finalizations, so each plugin
441/// will call the same (empty) function.
442class LibomptargetRtlFinalizer {
443public:
444 LibomptargetRtlFinalizer() {}
445
446 void registerRtl(ompt_finalize_t FinalizationFunction) {
447 if (FinalizationFunction) {
448 RtlFinalizationFunctions.emplace_back(FinalizationFunction);
449 }
450 }
451
452 void finalize() {
453 for (auto FinalizationFunction : RtlFinalizationFunctions)
454 FinalizationFunction(/*tool_data=*/nullptr);
455 RtlFinalizationFunctions.clear();
456 }
457
458private:
459 llvm::SmallVector<ompt_finalize_t> RtlFinalizationFunctions;
460};
461
462int llvm::omp::target::ompt::initializeLibrary(ompt_function_lookup_t lookup,
463 int initial_device_num,
464 ompt_data_t *tool_data) {
465 DP("Executing initializeLibrary\n");
466#define bindOmptFunctionName(OmptFunction, DestinationFunction) \
467 if (lookup) \
468 DestinationFunction = (OmptFunction##_t)lookup(#OmptFunction); \
469 DP("initializeLibrary bound %s=%p\n", #DestinationFunction, \
470 ((void *)(uint64_t)DestinationFunction));
471
472 bindOmptFunctionName(ompt_get_callback, lookupCallbackByCode);
473 bindOmptFunctionName(ompt_get_task_data, ompt_get_task_data_fn);
474 bindOmptFunctionName(ompt_get_target_task_data, ompt_get_target_task_data_fn);
475#undef bindOmptFunctionName
476
477 // Store pointer of 'ompt_libomp_target_fn_lookup' for use by libomptarget
478 lookupCallbackByName = lookup;
479
480 assert(lookupCallbackByCode && "lookupCallbackByCode should be non-null");
481 assert(lookupCallbackByName && "lookupCallbackByName should be non-null");
482 assert(ompt_get_task_data_fn && "ompt_get_task_data_fn should be non-null");
483 assert(ompt_get_target_task_data_fn &&
484 "ompt_get_target_task_data_fn should be non-null");
485 assert(LibraryFinalizer == nullptr &&
486 "LibraryFinalizer should not be initialized yet");
487
488 LibraryFinalizer = new LibomptargetRtlFinalizer();
489
490 Initialized = true;
491
492 return 0;
493}
494
495void llvm::omp::target::ompt::finalizeLibrary(ompt_data_t *data) {
496 DP("Executing finalizeLibrary\n");
497 // Before disabling OMPT, call the (plugin) finalizations that were registered
498 // with this library
499 LibraryFinalizer->finalize();
500 delete LibraryFinalizer;
501 Initialized = false;
502}
503
504void llvm::omp::target::ompt::connectLibrary() {
505 DP("Entering connectLibrary\n");
506 // Connect with libomp
507 static OmptLibraryConnectorTy LibompConnector("libomp");
508 static ompt_start_tool_result_t OmptResult;
509
510 // Initialize OmptResult with the init and fini functions that will be
511 // called by the connector
512 OmptResult.initialize = ompt::initializeLibrary;
513 OmptResult.finalize = ompt::finalizeLibrary;
514 OmptResult.tool_data.value = 0;
515
516 // Now call connect that causes the above init/fini functions to be called
517 LibompConnector.connect(&OmptResult);
518
519#define bindOmptCallback(Name, Type, Code) \
520 if (lookupCallbackByCode) \
521 lookupCallbackByCode( \
522 (ompt_callbacks_t)(Code), \
523 (ompt_callback_t *)&(llvm::omp::target::ompt::Name##_fn));
524 FOREACH_OMPT_NOEMI_EVENT(bindOmptCallback)
525 FOREACH_OMPT_EMI_EVENT(bindOmptCallback)
526#undef bindOmptCallback
527
528 DP("Exiting connectLibrary\n");
529}
530
531#endif // OMPT_SUPPORT
532

source code of offload/libomptarget/OpenMP/OMPT/Callback.cpp