1	/*! \file */
2	/*
3	* kmp.h -- KPTS runtime header file.
4	*/
5
6	//===----------------------------------------------------------------------===//
7	//
8	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
9	// See https://llvm.org/LICENSE.txt for license information.
10	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
11	//
12	//===----------------------------------------------------------------------===//
13
14	#ifndef KMP_H
15	#define KMP_H
16
17	#include "kmp_config.h"
18
19	/* #define BUILD_PARALLEL_ORDERED 1 */
20
21	/* This fix replaces gettimeofday with clock_gettime for better scalability on
22	the Altix. Requires user code to be linked with -lrt. */
23	//#define FIX_SGI_CLOCK
24
25	/* Defines for OpenMP 3.0 tasking and auto scheduling */
26
27	#ifndef KMP_STATIC_STEAL_ENABLED
28	#define KMP_STATIC_STEAL_ENABLED 1
29	#endif
30	#define KMP_WEIGHTED_ITERATIONS_SUPPORTED \
31	(KMP_AFFINITY_SUPPORTED && KMP_STATIC_STEAL_ENABLED && \
32	(KMP_ARCH_X86 || KMP_ARCH_X86_64))
33
34	#define TASK_CURRENT_NOT_QUEUED 0
35	#define TASK_CURRENT_QUEUED 1
36
37	#ifdef BUILD_TIED_TASK_STACK
38	#define TASK_STACK_EMPTY 0 // entries when the stack is empty
39	#define TASK_STACK_BLOCK_BITS 5 // Used in TASK_STACK_SIZE and TASK_STACK_MASK
40	// Number of entries in each task stack array
41	#define TASK_STACK_BLOCK_SIZE (1 << TASK_STACK_BLOCK_BITS)
42	// Mask for determining index into stack block
43	#define TASK_STACK_INDEX_MASK (TASK_STACK_BLOCK_SIZE - 1)
44	#endif // BUILD_TIED_TASK_STACK
45
46	#define TASK_NOT_PUSHED 1
47	#define TASK_SUCCESSFULLY_PUSHED 0
48	#define TASK_TIED 1
49	#define TASK_UNTIED 0
50	#define TASK_EXPLICIT 1
51	#define TASK_IMPLICIT 0
52	#define TASK_PROXY 1
53	#define TASK_FULL 0
54	#define TASK_DETACHABLE 1
55	#define TASK_UNDETACHABLE 0
56
57	#define KMP_CANCEL_THREADS
58	#define KMP_THREAD_ATTR
59
60	// Android does not have pthread_cancel. Undefine KMP_CANCEL_THREADS if being
61	// built on Android
62	#if defined(__ANDROID__)
63	#undef KMP_CANCEL_THREADS
64	#endif
65
66	// Some WASI targets (e.g., wasm32-wasi-threads) do not support thread
67	// cancellation.
68	#if KMP_OS_WASI
69	#undef KMP_CANCEL_THREADS
70	#endif
71
72	#if !KMP_OS_WASI
73	#include <signal.h>
74	#endif
75	#include <stdarg.h>
76	#include <stddef.h>
77	#include <stdio.h>
78	#include <stdlib.h>
79	#include <string.h>
80	#include <limits>
81	#include <type_traits>
82	/* include <ctype.h> don't use; problems with /MD on Windows* OS NT due to bad
83	Microsoft library. Some macros provided below to replace these functions */
84	#ifndef __ABSOFT_WIN
85	#include <sys/types.h>
86	#endif
87	#include <limits.h>
88	#include <time.h>
89
90	#include <errno.h>
91
92	#include "kmp_os.h"
93
94	#include "kmp_safe_c_api.h"
95
96	#if KMP_STATS_ENABLED
97	class kmp_stats_list;
98	#endif
99
100	#if KMP_USE_HIER_SCHED
101	// Only include hierarchical scheduling if affinity is supported
102	#undef KMP_USE_HIER_SCHED
103	#define KMP_USE_HIER_SCHED KMP_AFFINITY_SUPPORTED
104	#endif
105
106	// OMPD_SKIP_HWLOC used in libompd/omp-icv.cpp to avoid OMPD depending on hwloc
107	#if KMP_USE_HWLOC && KMP_AFFINITY_SUPPORTED && !defined(OMPD_SKIP_HWLOC)
108	#include "hwloc.h"
109	#define KMP_HWLOC_ENABLED 1
110	#ifndef HWLOC_OBJ_NUMANODE
111	#define HWLOC_OBJ_NUMANODE HWLOC_OBJ_NODE
112	#endif
113	#ifndef HWLOC_OBJ_PACKAGE
114	#define HWLOC_OBJ_PACKAGE HWLOC_OBJ_SOCKET
115	#endif
116	#else
117	#define KMP_HWLOC_ENABLED 0
118	#endif
119
120	#if KMP_ARCH_X86 || KMP_ARCH_X86_64
121	#include <xmmintrin.h>
122	#endif
123
124	// The below has to be defined before including "kmp_barrier.h".
125	#define KMP_INTERNAL_MALLOC(sz) malloc(sz)
126	#define KMP_INTERNAL_FREE(p) free(p)
127	#define KMP_INTERNAL_REALLOC(p, sz) realloc((p), (sz))
128	#define KMP_INTERNAL_CALLOC(n, sz) calloc((n), (sz))
129
130	#include "kmp_debug.h"
131	#include "kmp_lock.h"
132	#include "kmp_version.h"
133	#include "kmp_barrier.h"
134	#if USE_DEBUGGER
135	#include "kmp_debugger.h"
136	#endif
137	#include "kmp_i18n.h"
138
139	#define KMP_HANDLE_SIGNALS ((KMP_OS_UNIX && !KMP_OS_WASI) || KMP_OS_WINDOWS)
140
141	#include "kmp_wrapper_malloc.h"
142	#if KMP_OS_UNIX
143	#include <unistd.h>
144	#if !defined NSIG && defined _NSIG
145	#define NSIG _NSIG
146	#endif
147	#endif
148
149	#if KMP_OS_LINUX
150	#pragma weak clock_gettime
151	#endif
152
153	#if OMPT_SUPPORT
154	#include "ompt-internal.h"
155	#endif
156
157	#if OMPD_SUPPORT
158	#include "ompd-specific.h"
159	#endif
160
161	#ifndef UNLIKELY
162	#define UNLIKELY(x) (x)
163	#endif
164
165	// Affinity format function
166	#include "kmp_str.h"
167
168	// 0 - no fast memory allocation, alignment: 8-byte on x86, 16-byte on x64.
169	// 3 - fast allocation using sync, non-sync free lists of any size, non-self
170	// free lists of limited size.
171	#ifndef USE_FAST_MEMORY
172	#define USE_FAST_MEMORY 3
173	#endif
174
175	#ifndef KMP_NESTED_HOT_TEAMS
176	#define KMP_NESTED_HOT_TEAMS 0
177	#define USE_NESTED_HOT_ARG(x)
178	#else
179	#if KMP_NESTED_HOT_TEAMS
180	#define USE_NESTED_HOT_ARG(x) , x
181	#else
182	#define USE_NESTED_HOT_ARG(x)
183	#endif
184	#endif
185
186	// Assume using BGET compare_exchange instruction instead of lock by default.
187	#ifndef USE_CMP_XCHG_FOR_BGET
188	#define USE_CMP_XCHG_FOR_BGET 1
189	#endif
190
191	// Test to see if queuing lock is better than bootstrap lock for bget
192	// #ifndef USE_QUEUING_LOCK_FOR_BGET
193	// #define USE_QUEUING_LOCK_FOR_BGET
194	// #endif
195
196	#define KMP_NSEC_PER_SEC 1000000000L
197	#define KMP_USEC_PER_SEC 1000000L
198	#define KMP_NSEC_PER_USEC 1000L
199
200	/*!
201	@ingroup BASIC_TYPES
202	@{
203	*/
204
205	/*!
206	Values for bit flags used in the ident_t to describe the fields.
207	*/
208	enum {
209	/*! Use trampoline for internal microtasks */
210	KMP_IDENT_IMB = 0x01,
211	/*! Use c-style ident structure */
212	KMP_IDENT_KMPC = 0x02,
213	/* 0x04 is no longer used */
214	/*! Entry point generated by auto-parallelization */
215	KMP_IDENT_AUTOPAR = 0x08,
216	/*! Compiler generates atomic reduction option for kmpc_reduce* */
217	KMP_IDENT_ATOMIC_REDUCE = 0x10,
218	/*! To mark a 'barrier' directive in user code */
219	KMP_IDENT_BARRIER_EXPL = 0x20,
220	/*! To Mark implicit barriers. */
221	KMP_IDENT_BARRIER_IMPL = 0x0040,
222	KMP_IDENT_BARRIER_IMPL_MASK = 0x01C0,
223	KMP_IDENT_BARRIER_IMPL_FOR = 0x0040,
224	KMP_IDENT_BARRIER_IMPL_SECTIONS = 0x00C0,
225
226	KMP_IDENT_BARRIER_IMPL_SINGLE = 0x0140,
227	KMP_IDENT_BARRIER_IMPL_WORKSHARE = 0x01C0,
228
229	/*! To mark a static loop in OMPT callbacks */
230	KMP_IDENT_WORK_LOOP = 0x200,
231	/*! To mark a sections directive in OMPT callbacks */
232	KMP_IDENT_WORK_SECTIONS = 0x400,
233	/*! To mark a distribute construct in OMPT callbacks */
234	KMP_IDENT_WORK_DISTRIBUTE = 0x800,
235	/*! Atomic hint; bottom four bits as omp_sync_hint_t. Top four reserved and
236	not currently used. If one day we need more bits, then we can use
237	an invalid combination of hints to mean that another, larger field
238	should be used in a different flag. */
239	KMP_IDENT_ATOMIC_HINT_MASK = 0xFF0000,
240	KMP_IDENT_ATOMIC_HINT_UNCONTENDED = 0x010000,
241	KMP_IDENT_ATOMIC_HINT_CONTENDED = 0x020000,
242	KMP_IDENT_ATOMIC_HINT_NONSPECULATIVE = 0x040000,
243	KMP_IDENT_ATOMIC_HINT_SPECULATIVE = 0x080000,
244	KMP_IDENT_OPENMP_SPEC_VERSION_MASK = 0xFF000000
245	};
246
247	/*!
248	* The ident structure that describes a source location.
249	*/
250	typedef struct ident {
251	kmp_int32 reserved_1; /**< might be used in Fortran; see above */
252	kmp_int32 flags; /**< also f.flags; KMP_IDENT_xxx flags; KMP_IDENT_KMPC
253	identifies this union member */
254	kmp_int32 reserved_2; /**< not really used in Fortran any more; see above */
255	#if USE_ITT_BUILD
256	/* but currently used for storing region-specific ITT */
257	/* contextual information. */
258	#endif /* USE_ITT_BUILD */
259	kmp_int32 reserved_3; /**< source[4] in Fortran, do not use for C++ */
260	char const *psource; /**< String describing the source location.
261	The string is composed of semi-colon separated fields
262	which describe the source file, the function and a pair
263	of line numbers that delimit the construct. */
264	// Returns the OpenMP version in form major*10+minor (e.g., 50 for 5.0)
265	kmp_int32 get_openmp_version() {
266	return (((flags & KMP_IDENT_OPENMP_SPEC_VERSION_MASK) >> 24) & 0xFF);
267	}
268	} ident_t;
269	/*!
270	@}
271	*/
272
273	// Some forward declarations.
274	typedef union kmp_team kmp_team_t;
275	typedef struct kmp_taskdata kmp_taskdata_t;
276	typedef union kmp_task_team kmp_task_team_t;
277	typedef union kmp_team kmp_team_p;
278	typedef union kmp_info kmp_info_p;
279	typedef union kmp_root kmp_root_p;
280
281	template <bool C = false, bool S = true> class kmp_flag_32;
282	template <bool C = false, bool S = true> class kmp_flag_64;
283	template <bool C = false, bool S = true> class kmp_atomic_flag_64;
284	class kmp_flag_oncore;
285
286	#ifdef __cplusplus
287	extern "C" {
288	#endif
289
290	/* ------------------------------------------------------------------------ */
291
292	/* Pack two 32-bit signed integers into a 64-bit signed integer */
293	/* ToDo: Fix word ordering for big-endian machines. */
294	#define KMP_PACK_64(HIGH_32, LOW_32) \
295	((kmp_int64)((((kmp_uint64)(HIGH_32)) << 32) | (kmp_uint64)(LOW_32)))
296
297	// Generic string manipulation macros. Assume that _x is of type char *
298	#define SKIP_WS(_x) \
299	{ \
300	while (*(_x) == ' ' || *(_x) == '\t') \
301	(_x)++; \
302	}
303	#define SKIP_DIGITS(_x) \
304	{ \
305	while (*(_x) >= '0' && *(_x) <= '9') \
306	(_x)++; \
307	}
308	#define SKIP_TOKEN(_x) \
309	{ \
310	while ((*(_x) >= '0' && *(_x) <= '9') || (*(_x) >= 'a' && *(_x) <= 'z') || \
311	(*(_x) >= 'A' && *(_x) <= 'Z') || *(_x) == '_') \
312	(_x)++; \
313	}
314	#define SKIP_TO(_x, _c) \
315	{ \
316	while (*(_x) != '\0' && *(_x) != (_c)) \
317	(_x)++; \
318	}
319
320	/* ------------------------------------------------------------------------ */
321
322	#define KMP_MAX(x, y) ((x) > (y) ? (x) : (y))
323	#define KMP_MIN(x, y) ((x) < (y) ? (x) : (y))
324
325	/* ------------------------------------------------------------------------ */
326	/* Enumeration types */
327
328	enum kmp_state_timer {
329	ts_stop,
330	ts_start,
331	ts_pause,
332
333	ts_last_state
334	};
335
336	enum dynamic_mode {
337	dynamic_default,
338	#ifdef USE_LOAD_BALANCE
339	dynamic_load_balance,
340	#endif /* USE_LOAD_BALANCE */
341	dynamic_random,
342	dynamic_thread_limit,
343	dynamic_max
344	};
345
346	/* external schedule constants, duplicate enum omp_sched in omp.h in order to
347	* not include it here */
348	#ifndef KMP_SCHED_TYPE_DEFINED
349	#define KMP_SCHED_TYPE_DEFINED
350	typedef enum kmp_sched {
351	kmp_sched_lower = 0, // lower and upper bounds are for routine parameter check
352	// Note: need to adjust __kmp_sch_map global array in case enum is changed
353	kmp_sched_static = 1, // mapped to kmp_sch_static_chunked (33)
354	kmp_sched_dynamic = 2, // mapped to kmp_sch_dynamic_chunked (35)
355	kmp_sched_guided = 3, // mapped to kmp_sch_guided_chunked (36)
356	kmp_sched_auto = 4, // mapped to kmp_sch_auto (38)
357	kmp_sched_upper_std = 5, // upper bound for standard schedules
358	kmp_sched_lower_ext = 100, // lower bound of Intel extension schedules
359	kmp_sched_trapezoidal = 101, // mapped to kmp_sch_trapezoidal (39)
360	#if KMP_STATIC_STEAL_ENABLED
361	kmp_sched_static_steal = 102, // mapped to kmp_sch_static_steal (44)
362	#endif
363	kmp_sched_upper,
364	kmp_sched_default = kmp_sched_static, // default scheduling
365	kmp_sched_monotonic = 0x80000000
366	} kmp_sched_t;
367	#endif
368
369	/*!
370	@ingroup WORK_SHARING
371	* Describes the loop schedule to be used for a parallel for loop.
372	*/
373	enum sched_type : kmp_int32 {
374	kmp_sch_lower = 32, /**< lower bound for unordered values */
375	kmp_sch_static_chunked = 33,
376	kmp_sch_static = 34, /**< static unspecialized */
377	kmp_sch_dynamic_chunked = 35,
378	kmp_sch_guided_chunked = 36, /**< guided unspecialized */
379	kmp_sch_runtime = 37,
380	kmp_sch_auto = 38, /**< auto */
381	kmp_sch_trapezoidal = 39,
382
383	/* accessible only through KMP_SCHEDULE environment variable */
384	kmp_sch_static_greedy = 40,
385	kmp_sch_static_balanced = 41,
386	/* accessible only through KMP_SCHEDULE environment variable */
387	kmp_sch_guided_iterative_chunked = 42,
388	kmp_sch_guided_analytical_chunked = 43,
389	/* accessible only through KMP_SCHEDULE environment variable */
390	kmp_sch_static_steal = 44,
391
392	/* static with chunk adjustment (e.g., simd) */
393	kmp_sch_static_balanced_chunked = 45,
394	kmp_sch_guided_simd = 46, /**< guided with chunk adjustment */
395	kmp_sch_runtime_simd = 47, /**< runtime with chunk adjustment */
396
397	/* accessible only through KMP_SCHEDULE environment variable */
398	kmp_sch_upper, /**< upper bound for unordered values */
399
400	kmp_ord_lower = 64, /**< lower bound for ordered values, must be power of 2 */
401	kmp_ord_static_chunked = 65,
402	kmp_ord_static = 66, /**< ordered static unspecialized */
403	kmp_ord_dynamic_chunked = 67,
404	kmp_ord_guided_chunked = 68,
405	kmp_ord_runtime = 69,
406	kmp_ord_auto = 70, /**< ordered auto */
407	kmp_ord_trapezoidal = 71,
408	kmp_ord_upper, /**< upper bound for ordered values */
409
410	/* Schedules for Distribute construct */
411	kmp_distribute_static_chunked = 91, /**< distribute static chunked */
412	kmp_distribute_static = 92, /**< distribute static unspecialized */
413
414	/* For the "nomerge" versions, kmp_dispatch_next*() will always return a
415	single iteration/chunk, even if the loop is serialized. For the schedule
416	types listed above, the entire iteration vector is returned if the loop is
417	serialized. This doesn't work for gcc/gcomp sections. */
418	kmp_nm_lower = 160, /**< lower bound for nomerge values */
419
420	kmp_nm_static_chunked =
421	(kmp_sch_static_chunked - kmp_sch_lower + kmp_nm_lower),
422	kmp_nm_static = 162, /**< static unspecialized */
423	kmp_nm_dynamic_chunked = 163,
424	kmp_nm_guided_chunked = 164, /**< guided unspecialized */
425	kmp_nm_runtime = 165,
426	kmp_nm_auto = 166, /**< auto */
427	kmp_nm_trapezoidal = 167,
428
429	/* accessible only through KMP_SCHEDULE environment variable */
430	kmp_nm_static_greedy = 168,
431	kmp_nm_static_balanced = 169,
432	/* accessible only through KMP_SCHEDULE environment variable */
433	kmp_nm_guided_iterative_chunked = 170,
434	kmp_nm_guided_analytical_chunked = 171,
435	kmp_nm_static_steal =
436	172, /* accessible only through OMP_SCHEDULE environment variable */
437
438	kmp_nm_ord_static_chunked = 193,
439	kmp_nm_ord_static = 194, /**< ordered static unspecialized */
440	kmp_nm_ord_dynamic_chunked = 195,
441	kmp_nm_ord_guided_chunked = 196,
442	kmp_nm_ord_runtime = 197,
443	kmp_nm_ord_auto = 198, /**< auto */
444	kmp_nm_ord_trapezoidal = 199,
445	kmp_nm_upper, /**< upper bound for nomerge values */
446
447	/* Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers. Since
448	we need to distinguish the three possible cases (no modifier, monotonic
449	modifier, nonmonotonic modifier), we need separate bits for each modifier.
450	The absence of monotonic does not imply nonmonotonic, especially since 4.5
451	says that the behaviour of the "no modifier" case is implementation defined
452	in 4.5, but will become "nonmonotonic" in 5.0.
453
454	Since we're passing a full 32 bit value, we can use a couple of high bits
455	for these flags; out of paranoia we avoid the sign bit.
456
457	These modifiers can be or-ed into non-static schedules by the compiler to
458	pass the additional information. They will be stripped early in the
459	processing in __kmp_dispatch_init when setting up schedules, so most of the
460	code won't ever see schedules with these bits set. */
461	kmp_sch_modifier_monotonic =
462	(1 << 29), /**< Set if the monotonic schedule modifier was present */
463	kmp_sch_modifier_nonmonotonic =
464	(1 << 30), /**< Set if the nonmonotonic schedule modifier was present */
465
466	#define SCHEDULE_WITHOUT_MODIFIERS(s) \
467	(enum sched_type)( \
468	(s) & ~(kmp_sch_modifier_nonmonotonic | kmp_sch_modifier_monotonic))
469	#define SCHEDULE_HAS_MONOTONIC(s) (((s)&kmp_sch_modifier_monotonic) != 0)
470	#define SCHEDULE_HAS_NONMONOTONIC(s) (((s)&kmp_sch_modifier_nonmonotonic) != 0)
471	#define SCHEDULE_HAS_NO_MODIFIERS(s) \
472	(((s) & (kmp_sch_modifier_nonmonotonic | kmp_sch_modifier_monotonic)) == 0)
473	#define SCHEDULE_GET_MODIFIERS(s) \
474	((enum sched_type)( \
475	(s) & (kmp_sch_modifier_nonmonotonic | kmp_sch_modifier_monotonic)))
476	#define SCHEDULE_SET_MODIFIERS(s, m) \
477	(s = (enum sched_type)((kmp_int32)s | (kmp_int32)m))
478	#define SCHEDULE_NONMONOTONIC 0
479	#define SCHEDULE_MONOTONIC 1
480
481	kmp_sch_default = kmp_sch_static /**< default scheduling algorithm */
482	};
483
484	// Apply modifiers on internal kind to standard kind
485	static inline void
486	__kmp_sched_apply_mods_stdkind(kmp_sched_t *kind,
487	enum sched_type internal_kind) {
488	if (SCHEDULE_HAS_MONOTONIC(internal_kind)) {
489	*kind = (kmp_sched_t)((int)*kind | (int)kmp_sched_monotonic);
490	}
491	}
492
493	// Apply modifiers on standard kind to internal kind
494	static inline void
495	__kmp_sched_apply_mods_intkind(kmp_sched_t kind,
496	enum sched_type *internal_kind) {
497	if ((int)kind & (int)kmp_sched_monotonic) {
498	*internal_kind = (enum sched_type)((int)*internal_kind |
499	(int)kmp_sch_modifier_monotonic);
500	}
501	}
502
503	// Get standard schedule without modifiers
504	static inline kmp_sched_t __kmp_sched_without_mods(kmp_sched_t kind) {
505	return (kmp_sched_t)((int)kind & ~((int)kmp_sched_monotonic));
506	}
507
508	/* Type to keep runtime schedule set via OMP_SCHEDULE or omp_set_schedule() */
509	typedef union kmp_r_sched {
510	struct {
511	enum sched_type r_sched_type;
512	int chunk;
513	};
514	kmp_int64 sched;
515	} kmp_r_sched_t;
516
517	extern enum sched_type __kmp_sch_map[]; // map OMP 3.0 schedule types with our
518	// internal schedule types
519
520	enum library_type {
521	library_none,
522	library_serial,
523	library_turnaround,
524	library_throughput
525	};
526
527	#if KMP_MIC_SUPPORTED
528	enum mic_type { non_mic, mic1, mic2, mic3, dummy };
529	#endif
530
531	// OpenMP 3.1 - Nested num threads array
532	typedef struct kmp_nested_nthreads_t {
533	int *nth;
534	int size;
535	int used;
536	} kmp_nested_nthreads_t;
537
538	extern kmp_nested_nthreads_t __kmp_nested_nth;
539
540	/* -- fast reduction stuff ------------------------------------------------ */
541
542	#undef KMP_FAST_REDUCTION_BARRIER
543	#define KMP_FAST_REDUCTION_BARRIER 1
544
545	#undef KMP_FAST_REDUCTION_CORE_DUO
546	#if KMP_ARCH_X86 || KMP_ARCH_X86_64
547	#define KMP_FAST_REDUCTION_CORE_DUO 1
548	#endif
549
550	enum _reduction_method {
551	reduction_method_not_defined = 0,
552	critical_reduce_block = (1 << 8),
553	atomic_reduce_block = (2 << 8),
554	tree_reduce_block = (3 << 8),
555	empty_reduce_block = (4 << 8)
556	};
557
558	// Description of the packed_reduction_method variable:
559	// The packed_reduction_method variable consists of two enum types variables
560	// that are packed together into 0-th byte and 1-st byte:
561	// 0: (packed_reduction_method & 0x000000FF) is a 'enum barrier_type' value of
562	// barrier that will be used in fast reduction: bs_plain_barrier or
563	// bs_reduction_barrier
564	// 1: (packed_reduction_method & 0x0000FF00) is a reduction method that will
565	// be used in fast reduction;
566	// Reduction method is of 'enum _reduction_method' type and it's defined the way
567	// so that the bits of 0-th byte are empty, so no need to execute a shift
568	// instruction while packing/unpacking
569
570	#if KMP_FAST_REDUCTION_BARRIER
571	#define PACK_REDUCTION_METHOD_AND_BARRIER(reduction_method, barrier_type) \
572	((reduction_method) | (barrier_type))
573
574	#define UNPACK_REDUCTION_METHOD(packed_reduction_method) \
575	((enum _reduction_method)((packed_reduction_method) & (0x0000FF00)))
576
577	#define UNPACK_REDUCTION_BARRIER(packed_reduction_method) \
578	((enum barrier_type)((packed_reduction_method) & (0x000000FF)))
579	#else
580	#define PACK_REDUCTION_METHOD_AND_BARRIER(reduction_method, barrier_type) \
581	(reduction_method)
582
583	#define UNPACK_REDUCTION_METHOD(packed_reduction_method) \
584	(packed_reduction_method)
585
586	#define UNPACK_REDUCTION_BARRIER(packed_reduction_method) (bs_plain_barrier)
587	#endif
588
589	#define TEST_REDUCTION_METHOD(packed_reduction_method, which_reduction_block) \
590	((UNPACK_REDUCTION_METHOD(packed_reduction_method)) == \
591	(which_reduction_block))
592
593	#if KMP_FAST_REDUCTION_BARRIER
594	#define TREE_REDUCE_BLOCK_WITH_REDUCTION_BARRIER \
595	(PACK_REDUCTION_METHOD_AND_BARRIER(tree_reduce_block, bs_reduction_barrier))
596
597	#define TREE_REDUCE_BLOCK_WITH_PLAIN_BARRIER \
598	(PACK_REDUCTION_METHOD_AND_BARRIER(tree_reduce_block, bs_plain_barrier))
599	#endif
600
601	typedef int PACKED_REDUCTION_METHOD_T;
602
603	/* -- end of fast reduction stuff ----------------------------------------- */
604
605	#if KMP_OS_WINDOWS
606	#define USE_CBLKDATA
607	#if KMP_MSVC_COMPAT
608	#pragma warning(push)
609	#pragma warning(disable : 271 310)
610	#endif
611	#include <windows.h>
612	#if KMP_MSVC_COMPAT
613	#pragma warning(pop)
614	#endif
615	#endif
616
617	#if KMP_OS_UNIX
618	#if !KMP_OS_WASI
619	#include <dlfcn.h>
620	#endif
621	#include <pthread.h>
622	#endif
623
624	enum kmp_hw_t : int {
625	KMP_HW_UNKNOWN = -1,
626	KMP_HW_SOCKET = 0,
627	KMP_HW_PROC_GROUP,
628	KMP_HW_NUMA,
629	KMP_HW_DIE,
630	KMP_HW_LLC,
631	KMP_HW_L3,
632	KMP_HW_TILE,
633	KMP_HW_MODULE,
634	KMP_HW_L2,
635	KMP_HW_L1,
636	KMP_HW_CORE,
637	KMP_HW_THREAD,
638	KMP_HW_LAST
639	};
640
641	typedef enum kmp_hw_core_type_t {
642	KMP_HW_CORE_TYPE_UNKNOWN = 0x0,
643	#if KMP_ARCH_X86 || KMP_ARCH_X86_64
644	KMP_HW_CORE_TYPE_ATOM = 0x20,
645	KMP_HW_CORE_TYPE_CORE = 0x40,
646	KMP_HW_MAX_NUM_CORE_TYPES = 3,
647	#else
648	KMP_HW_MAX_NUM_CORE_TYPES = 1,
649	#endif
650	} kmp_hw_core_type_t;
651
652	#define KMP_HW_MAX_NUM_CORE_EFFS 8
653
654	#define KMP_DEBUG_ASSERT_VALID_HW_TYPE(type) \
655	KMP_DEBUG_ASSERT(type >= (kmp_hw_t)0 && type < KMP_HW_LAST)
656	#define KMP_ASSERT_VALID_HW_TYPE(type) \
657	KMP_ASSERT(type >= (kmp_hw_t)0 && type < KMP_HW_LAST)
658
659	#define KMP_FOREACH_HW_TYPE(type) \
660	for (kmp_hw_t type = (kmp_hw_t)0; type < KMP_HW_LAST; \
661	type = (kmp_hw_t)((int)type + 1))
662
663	const char *__kmp_hw_get_keyword(kmp_hw_t type, bool plural = false);
664	const char *__kmp_hw_get_catalog_string(kmp_hw_t type, bool plural = false);
665	const char *__kmp_hw_get_core_type_string(kmp_hw_core_type_t type);
666
667	/* Only Linux* OS and Windows* OS support thread affinity. */
668	#if KMP_AFFINITY_SUPPORTED
669
670	// GROUP_AFFINITY is already defined for _MSC_VER>=1600 (VS2010 and later).
671	#if KMP_OS_WINDOWS
672	#if _MSC_VER < 1600 && KMP_MSVC_COMPAT
673	typedef struct GROUP_AFFINITY {
674	KAFFINITY Mask;
675	WORD Group;
676	WORD Reserved[3];
677	} GROUP_AFFINITY;
678	#endif /* _MSC_VER < 1600 */
679	#if KMP_GROUP_AFFINITY
680	extern int __kmp_num_proc_groups;
681	#else
682	static const int __kmp_num_proc_groups = 1;
683	#endif /* KMP_GROUP_AFFINITY */
684	typedef DWORD (*kmp_GetActiveProcessorCount_t)(WORD);
685	extern kmp_GetActiveProcessorCount_t __kmp_GetActiveProcessorCount;
686
687	typedef WORD (*kmp_GetActiveProcessorGroupCount_t)(void);
688	extern kmp_GetActiveProcessorGroupCount_t __kmp_GetActiveProcessorGroupCount;
689
690	typedef BOOL (*kmp_GetThreadGroupAffinity_t)(HANDLE, GROUP_AFFINITY *);
691	extern kmp_GetThreadGroupAffinity_t __kmp_GetThreadGroupAffinity;
692
693	typedef BOOL (*kmp_SetThreadGroupAffinity_t)(HANDLE, const GROUP_AFFINITY *,
694	GROUP_AFFINITY *);
695	extern kmp_SetThreadGroupAffinity_t __kmp_SetThreadGroupAffinity;
696	#endif /* KMP_OS_WINDOWS */
697
698	#if KMP_HWLOC_ENABLED
699	extern hwloc_topology_t __kmp_hwloc_topology;
700	extern int __kmp_hwloc_error;
701	#endif // KMP_HWLOC_ENABLED
702
703	extern size_t __kmp_affin_mask_size;
704	#define KMP_AFFINITY_CAPABLE() (__kmp_affin_mask_size > 0)
705	#define KMP_AFFINITY_DISABLE() (__kmp_affin_mask_size = 0)
706	#define KMP_AFFINITY_ENABLE(mask_size) (__kmp_affin_mask_size = mask_size)
707	#define KMP_CPU_SET_ITERATE(i, mask) \
708	for (i = (mask)->begin(); (int)i != (mask)->end(); i = (mask)->next(i))
709	#define KMP_CPU_SET(i, mask) (mask)->set(i)
710	#define KMP_CPU_ISSET(i, mask) (mask)->is_set(i)
711	#define KMP_CPU_CLR(i, mask) (mask)->clear(i)
712	#define KMP_CPU_ZERO(mask) (mask)->zero()
713	#define KMP_CPU_ISEMPTY(mask) (mask)->empty()
714	#define KMP_CPU_COPY(dest, src) (dest)->copy(src)
715	#define KMP_CPU_AND(dest, src) (dest)->bitwise_and(src)
716	#define KMP_CPU_COMPLEMENT(max_bit_number, mask) (mask)->bitwise_not()
717	#define KMP_CPU_UNION(dest, src) (dest)->bitwise_or(src)
718	#define KMP_CPU_EQUAL(dest, src) (dest)->is_equal(src)
719	#define KMP_CPU_ALLOC(ptr) (ptr = __kmp_affinity_dispatch->allocate_mask())
720	#define KMP_CPU_FREE(ptr) __kmp_affinity_dispatch->deallocate_mask(ptr)
721	#define KMP_CPU_ALLOC_ON_STACK(ptr) KMP_CPU_ALLOC(ptr)
722	#define KMP_CPU_FREE_FROM_STACK(ptr) KMP_CPU_FREE(ptr)
723	#define KMP_CPU_INTERNAL_ALLOC(ptr) KMP_CPU_ALLOC(ptr)
724	#define KMP_CPU_INTERNAL_FREE(ptr) KMP_CPU_FREE(ptr)
725	#define KMP_CPU_INDEX(arr, i) __kmp_affinity_dispatch->index_mask_array(arr, i)
726	#define KMP_CPU_ALLOC_ARRAY(arr, n) \
727	(arr = __kmp_affinity_dispatch->allocate_mask_array(n))
728	#define KMP_CPU_FREE_ARRAY(arr, n) \
729	__kmp_affinity_dispatch->deallocate_mask_array(arr)
730	#define KMP_CPU_INTERNAL_ALLOC_ARRAY(arr, n) KMP_CPU_ALLOC_ARRAY(arr, n)
731	#define KMP_CPU_INTERNAL_FREE_ARRAY(arr, n) KMP_CPU_FREE_ARRAY(arr, n)
732	#define __kmp_get_system_affinity(mask, abort_bool) \
733	(mask)->get_system_affinity(abort_bool)
734	#define __kmp_set_system_affinity(mask, abort_bool) \
735	(mask)->set_system_affinity(abort_bool)
736	#define __kmp_get_proc_group(mask) (mask)->get_proc_group()
737
738	class KMPAffinity {
739	public:
740	class Mask {
741	public:
742	void *operator new(size_t n);
743	void operator delete(void *p);
744	void *operator new[](size_t n);
745	void operator delete[](void *p);
746	virtual ~Mask() {}
747	// Set bit i to 1
748	virtual void set(int i) {}
749	// Return bit i
750	virtual bool is_set(int i) const { return false; }
751	// Set bit i to 0
752	virtual void clear(int i) {}
753	// Zero out entire mask
754	virtual void zero() {}
755	// Check whether mask is empty
756	virtual bool empty() const { return true; }
757	// Copy src into this mask
758	virtual void copy(const Mask *src) {}
759	// this &= rhs
760	virtual void bitwise_and(const Mask *rhs) {}
761	// this |= rhs
762	virtual void bitwise_or(const Mask *rhs) {}
763	// this = ~this
764	virtual void bitwise_not() {}
765	// this == rhs
766	virtual bool is_equal(const Mask *rhs) const { return false; }
767	// API for iterating over an affinity mask
768	// for (int i = mask->begin(); i != mask->end(); i = mask->next(i))
769	virtual int begin() const { return 0; }
770	virtual int end() const { return 0; }
771	virtual int next(int previous) const { return 0; }
772	#if KMP_OS_WINDOWS
773	virtual int set_process_affinity(bool abort_on_error) const { return -1; }
774	#endif
775	// Set the system's affinity to this affinity mask's value
776	virtual int set_system_affinity(bool abort_on_error) const { return -1; }
777	// Set this affinity mask to the current system affinity
778	virtual int get_system_affinity(bool abort_on_error) { return -1; }
779	// Only 1 DWORD in the mask should have any procs set.
780	// Return the appropriate index, or -1 for an invalid mask.
781	virtual int get_proc_group() const { return -1; }
782	int get_max_cpu() const {
783	int cpu;
784	int max_cpu = -1;
785	KMP_CPU_SET_ITERATE(cpu, this) {
786	if (cpu > max_cpu)
787	max_cpu = cpu;
788	}
789	return max_cpu;
790	}
791	};
792	void *operator new(size_t n);
793	void operator delete(void *p);
794	// Need virtual destructor
795	virtual ~KMPAffinity() = default;
796	// Determine if affinity is capable
797	virtual void determine_capable(const char *env_var) {}
798	// Bind the current thread to os proc
799	virtual void bind_thread(int proc) {}
800	// Factory functions to allocate/deallocate a mask
801	virtual Mask *allocate_mask() { return nullptr; }
802	virtual void deallocate_mask(Mask *m) {}
803	virtual Mask *allocate_mask_array(int num) { return nullptr; }
804	virtual void deallocate_mask_array(Mask *m) {}
805	virtual Mask *index_mask_array(Mask *m, int index) { return nullptr; }
806	static void pick_api();
807	static void destroy_api();
808	enum api_type {
809	NATIVE_OS
810	#if KMP_HWLOC_ENABLED
811	,
812	HWLOC
813	#endif // KMP_HWLOC_ENABLED
814	};
815	virtual api_type get_api_type() const {
816	KMP_ASSERT(0);
817	return NATIVE_OS;
818	}
819
820	private:
821	static bool picked_api;
822	};
823
824	typedef KMPAffinity::Mask kmp_affin_mask_t;
825	extern KMPAffinity *__kmp_affinity_dispatch;
826
827	#if !KMP_OS_AIX
828	class kmp_affinity_raii_t {
829	kmp_affin_mask_t *mask;
830	bool restored;
831
832	public:
833	kmp_affinity_raii_t(const kmp_affin_mask_t *new_mask = nullptr)
834	: mask(nullptr), restored(false) {
835	if (KMP_AFFINITY_CAPABLE()) {
836	KMP_CPU_ALLOC(mask);
837	KMP_ASSERT(mask != NULL);
838	__kmp_get_system_affinity(mask, /*abort_on_error=*/true);
839	if (new_mask)
840	__kmp_set_system_affinity(new_mask, /*abort_on_error=*/true);
841	}
842	}
843	void restore() {
844	if (mask && KMP_AFFINITY_CAPABLE() && !restored) {
845	__kmp_set_system_affinity(mask, /*abort_on_error=*/true);
846	KMP_CPU_FREE(mask);
847	}
848	restored = true;
849	}
850	~kmp_affinity_raii_t() { restore(); }
851	};
852	#endif // !KMP_OS_AIX
853
854	// Declare local char buffers with this size for printing debug and info
855	// messages, using __kmp_affinity_print_mask().
856	#define KMP_AFFIN_MASK_PRINT_LEN 1024
857
858	enum affinity_type {
859	affinity_none = 0,
860	affinity_physical,
861	affinity_logical,
862	affinity_compact,
863	affinity_scatter,
864	affinity_explicit,
865	affinity_balanced,
866	affinity_disabled, // not used outsize the env var parser
867	affinity_default
868	};
869
870	enum affinity_top_method {
871	affinity_top_method_all = 0, // try all (supported) methods, in order
872	#if KMP_ARCH_X86 || KMP_ARCH_X86_64
873	affinity_top_method_apicid,
874	affinity_top_method_x2apicid,
875	affinity_top_method_x2apicid_1f,
876	#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
877	affinity_top_method_cpuinfo, // KMP_CPUINFO_FILE is usable on Windows* OS, too
878	#if KMP_GROUP_AFFINITY
879	affinity_top_method_group,
880	#endif /* KMP_GROUP_AFFINITY */
881	affinity_top_method_flat,
882	#if KMP_HWLOC_ENABLED
883	affinity_top_method_hwloc,
884	#endif // KMP_HWLOC_ENABLED
885	affinity_top_method_default
886	};
887
888	#define affinity_respect_mask_default (2)
889
890	typedef struct kmp_affinity_flags_t {
891	unsigned dups : 1;
892	unsigned verbose : 1;
893	unsigned warnings : 1;
894	unsigned respect : 2;
895	unsigned reset : 1;
896	unsigned initialized : 1;
897	unsigned core_types_gran : 1;
898	unsigned core_effs_gran : 1;
899	unsigned omp_places : 1;
900	unsigned reserved : 22;
901	} kmp_affinity_flags_t;
902	KMP_BUILD_ASSERT(sizeof(kmp_affinity_flags_t) == 4);
903
904	typedef struct kmp_affinity_ids_t {
905	int os_id;
906	int ids[KMP_HW_LAST];
907	} kmp_affinity_ids_t;
908
909	typedef struct kmp_affinity_attrs_t {
910	int core_type : 8;
911	int core_eff : 8;
912	unsigned valid : 1;
913	unsigned reserved : 15;
914	} kmp_affinity_attrs_t;
915	#define KMP_AFFINITY_ATTRS_UNKNOWN \
916	{ KMP_HW_CORE_TYPE_UNKNOWN, kmp_hw_attr_t::UNKNOWN_CORE_EFF, 0, 0 }
917
918	typedef struct kmp_affinity_t {
919	char *proclist;
920	enum affinity_type type;
921	kmp_hw_t gran;
922	int gran_levels;
923	kmp_affinity_attrs_t core_attr_gran;
924	int compact;
925	int offset;
926	kmp_affinity_flags_t flags;
927	unsigned num_masks;
928	kmp_affin_mask_t *masks;
929	kmp_affinity_ids_t *ids;
930	kmp_affinity_attrs_t *attrs;
931	unsigned num_os_id_masks;
932	kmp_affin_mask_t *os_id_masks;
933	const char *env_var;
934	} kmp_affinity_t;
935
936	#define KMP_AFFINITY_INIT(env) \
937	{ \
938	nullptr, affinity_default, KMP_HW_UNKNOWN, -1, KMP_AFFINITY_ATTRS_UNKNOWN, \
939	0, 0, \
940	{TRUE, FALSE, TRUE, affinity_respect_mask_default, FALSE, FALSE, \
941	FALSE, FALSE, FALSE}, \
942	0, nullptr, nullptr, nullptr, 0, nullptr, env \
943	}
944
945	extern enum affinity_top_method __kmp_affinity_top_method;
946	extern kmp_affinity_t __kmp_affinity;
947	extern kmp_affinity_t __kmp_hh_affinity;
948	extern kmp_affinity_t *__kmp_affinities[2];
949
950	extern void __kmp_affinity_bind_thread(int which);
951
952	extern kmp_affin_mask_t *__kmp_affin_fullMask;
953	extern kmp_affin_mask_t *__kmp_affin_origMask;
954	extern char *__kmp_cpuinfo_file;
955
956	#if KMP_WEIGHTED_ITERATIONS_SUPPORTED
957	extern int __kmp_first_osid_with_ecore;
958	#endif
959
960	#endif /* KMP_AFFINITY_SUPPORTED */
961
962	// This needs to be kept in sync with the values in omp.h !!!
963	typedef enum kmp_proc_bind_t {
964	proc_bind_false = 0,
965	proc_bind_true,
966	proc_bind_primary,
967	proc_bind_close,
968	proc_bind_spread,
969	proc_bind_intel, // use KMP_AFFINITY interface
970	proc_bind_default
971	} kmp_proc_bind_t;
972
973	typedef struct kmp_nested_proc_bind_t {
974	kmp_proc_bind_t *bind_types;
975	int size;
976	int used;
977	} kmp_nested_proc_bind_t;
978
979	extern kmp_nested_proc_bind_t __kmp_nested_proc_bind;
980	extern kmp_proc_bind_t __kmp_teams_proc_bind;
981
982	extern int __kmp_display_affinity;
983	extern char *__kmp_affinity_format;
984	static const size_t KMP_AFFINITY_FORMAT_SIZE = 512;
985	#if OMPT_SUPPORT
986	extern int __kmp_tool;
987	extern char *__kmp_tool_libraries;
988	#endif // OMPT_SUPPORT
989
990	#if KMP_AFFINITY_SUPPORTED
991	#define KMP_PLACE_ALL (-1)
992	#define KMP_PLACE_UNDEFINED (-2)
993	// Is KMP_AFFINITY is being used instead of OMP_PROC_BIND/OMP_PLACES?
994	#define KMP_AFFINITY_NON_PROC_BIND \
995	((__kmp_nested_proc_bind.bind_types[0] == proc_bind_false || \
996	__kmp_nested_proc_bind.bind_types[0] == proc_bind_intel) && \
997	(__kmp_affinity.num_masks > 0 || __kmp_affinity.type == affinity_balanced))
998	#endif /* KMP_AFFINITY_SUPPORTED */
999
1000	extern int __kmp_affinity_num_places;
1001
1002	typedef enum kmp_cancel_kind_t {
1003	cancel_noreq = 0,
1004	cancel_parallel = 1,
1005	cancel_loop = 2,
1006	cancel_sections = 3,
1007	cancel_taskgroup = 4
1008	} kmp_cancel_kind_t;
1009
1010	// KMP_HW_SUBSET support:
1011	typedef struct kmp_hws_item {
1012	int num;
1013	int offset;
1014	} kmp_hws_item_t;
1015
1016	extern kmp_hws_item_t __kmp_hws_socket;
1017	extern kmp_hws_item_t __kmp_hws_die;
1018	extern kmp_hws_item_t __kmp_hws_node;
1019	extern kmp_hws_item_t __kmp_hws_tile;
1020	extern kmp_hws_item_t __kmp_hws_core;
1021	extern kmp_hws_item_t __kmp_hws_proc;
1022	extern int __kmp_hws_requested;
1023	extern int __kmp_hws_abs_flag; // absolute or per-item number requested
1024
1025	/* ------------------------------------------------------------------------ */
1026
1027	#define KMP_PAD(type, sz) \
1028	(sizeof(type) + (sz - ((sizeof(type) - 1) % (sz)) - 1))
1029
1030	// We need to avoid using -1 as a GTID as +1 is added to the gtid
1031	// when storing it in a lock, and the value 0 is reserved.
1032	#define KMP_GTID_DNE (-2) /* Does not exist */
1033	#define KMP_GTID_SHUTDOWN (-3) /* Library is shutting down */
1034	#define KMP_GTID_MONITOR (-4) /* Monitor thread ID */
1035	#define KMP_GTID_UNKNOWN (-5) /* Is not known */
1036	#define KMP_GTID_MIN (-6) /* Minimal gtid for low bound check in DEBUG */
1037
1038	/* OpenMP 5.0 Memory Management support */
1039
1040	#ifndef __OMP_H
1041	// Duplicate type definitions from omp.h
1042	typedef uintptr_t omp_uintptr_t;
1043
1044	typedef enum {
1045	omp_atk_sync_hint = 1,
1046	omp_atk_alignment = 2,
1047	omp_atk_access = 3,
1048	omp_atk_pool_size = 4,
1049	omp_atk_fallback = 5,
1050	omp_atk_fb_data = 6,
1051	omp_atk_pinned = 7,
1052	omp_atk_partition = 8,
1053	omp_atk_pin_device = 9,
1054	omp_atk_preferred_device = 10,
1055	omp_atk_device_access = 11,
1056	omp_atk_target_access = 12,
1057	omp_atk_atomic_scope = 13,
1058	omp_atk_part_size = 14
1059	} omp_alloctrait_key_t;
1060
1061	typedef enum {
1062	omp_atv_false = 0,
1063	omp_atv_true = 1,
1064	omp_atv_contended = 3,
1065	omp_atv_uncontended = 4,
1066	omp_atv_serialized = 5,
1067	omp_atv_sequential = omp_atv_serialized, // (deprecated)
1068	omp_atv_private = 6,
1069	omp_atv_device = 7,
1070	omp_atv_thread = 8,
1071	omp_atv_pteam = 9,
1072	omp_atv_cgroup = 10,
1073	omp_atv_default_mem_fb = 11,
1074	omp_atv_null_fb = 12,
1075	omp_atv_abort_fb = 13,
1076	omp_atv_allocator_fb = 14,
1077	omp_atv_environment = 15,
1078	omp_atv_nearest = 16,
1079	omp_atv_blocked = 17,
1080	omp_atv_interleaved = 18,
1081	omp_atv_all = 19,
1082	omp_atv_single = 20,
1083	omp_atv_multiple = 21,
1084	omp_atv_memspace = 22
1085	} omp_alloctrait_value_t;
1086	#define omp_atv_default ((omp_uintptr_t)-1)
1087
1088	typedef void *omp_memspace_handle_t;
1089	extern omp_memspace_handle_t const omp_null_mem_space;
1090	extern omp_memspace_handle_t const omp_default_mem_space;
1091	extern omp_memspace_handle_t const omp_large_cap_mem_space;
1092	extern omp_memspace_handle_t const omp_const_mem_space;
1093	extern omp_memspace_handle_t const omp_high_bw_mem_space;
1094	extern omp_memspace_handle_t const omp_low_lat_mem_space;
1095	extern omp_memspace_handle_t const llvm_omp_target_host_mem_space;
1096	extern omp_memspace_handle_t const llvm_omp_target_shared_mem_space;
1097	extern omp_memspace_handle_t const llvm_omp_target_device_mem_space;
1098	extern omp_memspace_handle_t const kmp_max_mem_space;
1099
1100	typedef struct {
1101	omp_alloctrait_key_t key;
1102	omp_uintptr_t value;
1103	} omp_alloctrait_t;
1104
1105	typedef void *omp_allocator_handle_t;
1106	extern omp_allocator_handle_t const omp_null_allocator;
1107	extern omp_allocator_handle_t const omp_default_mem_alloc;
1108	extern omp_allocator_handle_t const omp_large_cap_mem_alloc;
1109	extern omp_allocator_handle_t const omp_const_mem_alloc;
1110	extern omp_allocator_handle_t const omp_high_bw_mem_alloc;
1111	extern omp_allocator_handle_t const omp_low_lat_mem_alloc;
1112	extern omp_allocator_handle_t const omp_cgroup_mem_alloc;
1113	extern omp_allocator_handle_t const omp_pteam_mem_alloc;
1114	extern omp_allocator_handle_t const omp_thread_mem_alloc;
1115	extern omp_allocator_handle_t const llvm_omp_target_host_mem_alloc;
1116	extern omp_allocator_handle_t const llvm_omp_target_shared_mem_alloc;
1117	extern omp_allocator_handle_t const llvm_omp_target_device_mem_alloc;
1118	extern omp_allocator_handle_t const kmp_max_mem_alloc;
1119	extern omp_allocator_handle_t __kmp_def_allocator;
1120
1121	// end of duplicate type definitions from omp.h
1122	#endif
1123
1124	extern int __kmp_memkind_available;
1125	extern bool __kmp_hwloc_available;
1126
1127	/// Memory space informaition is shared with offload runtime.
1128	typedef struct kmp_memspace_t {
1129	omp_memspace_handle_t memspace; // predefined input memory space
1130	int num_resources = 0; // number of available resources
1131	int *resources = nullptr; // available resources
1132	kmp_memspace_t *next = nullptr; // next memory space handle
1133	} kmp_memspace_t;
1134
1135	/// Memory allocator information is shared with offload runtime.
1136	typedef struct kmp_allocator_t {
1137	omp_memspace_handle_t memspace;
1138	void **memkind; // pointer to memkind
1139	size_t alignment;
1140	omp_alloctrait_value_t fb;
1141	kmp_allocator_t *fb_data;
1142	kmp_uint64 pool_size;
1143	kmp_uint64 pool_used;
1144	bool pinned;
1145	omp_alloctrait_value_t partition;
1146	int pin_device;
1147	int preferred_device;
1148	omp_alloctrait_value_t target_access;
1149	omp_alloctrait_value_t atomic_scope;
1150	size_t part_size;
1151	#if KMP_HWLOC_ENABLED
1152	omp_alloctrait_value_t membind;
1153	#endif // KMP_HWLOC_ENABLED
1154	} kmp_allocator_t;
1155
1156	extern omp_allocator_handle_t __kmpc_init_allocator(int gtid,
1157	omp_memspace_handle_t,
1158	int ntraits,
1159	omp_alloctrait_t traits[]);
1160	extern void __kmpc_destroy_allocator(int gtid, omp_allocator_handle_t al);
1161	extern void __kmpc_set_default_allocator(int gtid, omp_allocator_handle_t al);
1162	extern omp_allocator_handle_t __kmpc_get_default_allocator(int gtid);
1163	// external interfaces, may be used by compiler
1164	extern void *__kmpc_alloc(int gtid, size_t sz, omp_allocator_handle_t al);
1165	extern void *__kmpc_aligned_alloc(int gtid, size_t align, size_t sz,
1166	omp_allocator_handle_t al);
1167	extern void *__kmpc_calloc(int gtid, size_t nmemb, size_t sz,
1168	omp_allocator_handle_t al);
1169	extern void *__kmpc_realloc(int gtid, void *ptr, size_t sz,
1170	omp_allocator_handle_t al,
1171	omp_allocator_handle_t free_al);
1172	extern void __kmpc_free(int gtid, void *ptr, omp_allocator_handle_t al);
1173	// internal interfaces, contain real implementation
1174	extern void *__kmp_alloc(int gtid, size_t align, size_t sz,
1175	omp_allocator_handle_t al);
1176	extern void *__kmp_calloc(int gtid, size_t align, size_t nmemb, size_t sz,
1177	omp_allocator_handle_t al);
1178	extern void *__kmp_realloc(int gtid, void *ptr, size_t sz,
1179	omp_allocator_handle_t al,
1180	omp_allocator_handle_t free_al);
1181	extern void ___kmpc_free(int gtid, void *ptr, omp_allocator_handle_t al);
1182
1183	extern void __kmp_init_memkind();
1184	extern void __kmp_fini_memkind();
1185	extern void __kmp_init_target_mem();
1186	extern void __kmp_fini_target_mem();
1187
1188	// OpenMP 6.0 (TR11) Memory Management support
1189	extern omp_memspace_handle_t __kmp_get_devices_memspace(int ndevs,
1190	const int *devs,
1191	omp_memspace_handle_t,
1192	int host);
1193	extern omp_allocator_handle_t __kmp_get_devices_allocator(int ndevs,
1194	const int *devs,
1195	omp_memspace_handle_t,
1196	int host);
1197	extern int __kmp_get_memspace_num_resources(omp_memspace_handle_t memspace);
1198	extern omp_memspace_handle_t
1199	__kmp_get_submemspace(omp_memspace_handle_t memspace, int num_resources,
1200	int *resources);
1201
1202	/* ------------------------------------------------------------------------ */
1203
1204	#if ENABLE_LIBOMPTARGET
1205	extern void __kmp_init_target_task();
1206	#endif
1207
1208	/* ------------------------------------------------------------------------ */
1209
1210	#define KMP_UINT64_MAX \
1211	(~((kmp_uint64)1 << ((sizeof(kmp_uint64) * (1 << 3)) - 1)))
1212
1213	#define KMP_MIN_NTH 1
1214
1215	#ifndef KMP_MAX_NTH
1216	#if defined(PTHREAD_THREADS_MAX) && PTHREAD_THREADS_MAX < INT_MAX
1217	#define KMP_MAX_NTH PTHREAD_THREADS_MAX
1218	#else
1219	#ifdef __ve__
1220	// VE's pthread supports only up to 64 threads per a VE process.
1221	// Please check p. 14 of following documentation for more details.
1222	// https://sxauroratsubasa.sakura.ne.jp/documents/veos/en/VEOS_high_level_design.pdf
1223	#define KMP_MAX_NTH 64
1224	#else
1225	#define KMP_MAX_NTH INT_MAX
1226	#endif
1227	#endif
1228	#endif /* KMP_MAX_NTH */
1229
1230	#ifdef PTHREAD_STACK_MIN
1231	#define KMP_MIN_STKSIZE ((size_t)PTHREAD_STACK_MIN)
1232	#else
1233	#define KMP_MIN_STKSIZE ((size_t)(32 * 1024))
1234	#endif
1235
1236	#if KMP_OS_AIX && KMP_ARCH_PPC
1237	#define KMP_MAX_STKSIZE 0x10000000 /* 256Mb max size on 32-bit AIX */
1238	#else
1239	#define KMP_MAX_STKSIZE (~((size_t)1 << ((sizeof(size_t) * (1 << 3)) - 1)))
1240	#endif
1241
1242	#if KMP_ARCH_X86
1243	#define KMP_DEFAULT_STKSIZE ((size_t)(2 * 1024 * 1024))
1244	#elif KMP_ARCH_X86_64
1245	#define KMP_DEFAULT_STKSIZE ((size_t)(4 * 1024 * 1024))
1246	#define KMP_BACKUP_STKSIZE ((size_t)(2 * 1024 * 1024))
1247	#elif KMP_ARCH_VE
1248	// Minimum stack size for pthread for VE is 4MB.
1249	// https://www.hpc.nec/documents/veos/en/glibc/Difference_Points_glibc.htm
1250	#define KMP_DEFAULT_STKSIZE ((size_t)(4 * 1024 * 1024))
1251	#elif KMP_OS_AIX
1252	// The default stack size for worker threads on AIX is 4MB.
1253	#define KMP_DEFAULT_STKSIZE ((size_t)(4 * 1024 * 1024))
1254	#else
1255	#define KMP_DEFAULT_STKSIZE ((size_t)(1024 * 1024))
1256	#endif
1257
1258	#define KMP_DEFAULT_MALLOC_POOL_INCR ((size_t)(1024 * 1024))
1259	#define KMP_MIN_MALLOC_POOL_INCR ((size_t)(4 * 1024))
1260	#define KMP_MAX_MALLOC_POOL_INCR \
1261	(~((size_t)1 << ((sizeof(size_t) * (1 << 3)) - 1)))
1262
1263	#define KMP_MIN_STKOFFSET (0)
1264	#define KMP_MAX_STKOFFSET KMP_MAX_STKSIZE
1265	#if KMP_OS_DARWIN
1266	#define KMP_DEFAULT_STKOFFSET KMP_MIN_STKOFFSET
1267	#else
1268	#define KMP_DEFAULT_STKOFFSET CACHE_LINE
1269	#endif
1270
1271	#define KMP_MIN_STKPADDING (0)
1272	#define KMP_MAX_STKPADDING (2 * 1024 * 1024)
1273
1274	#define KMP_BLOCKTIME_MULTIPLIER \
1275	(1000000) /* number of blocktime units per second */
1276	#define KMP_MIN_BLOCKTIME (0)
1277	#define KMP_MAX_BLOCKTIME \
1278	(INT_MAX) /* Must be this for "infinite" setting the work */
1279
1280	/* __kmp_blocktime is in microseconds */
1281	#define KMP_DEFAULT_BLOCKTIME (__kmp_is_hybrid_cpu() ? (0) : (200000))
1282
1283	#if KMP_USE_MONITOR
1284	#define KMP_DEFAULT_MONITOR_STKSIZE ((size_t)(64 * 1024))
1285	#define KMP_MIN_MONITOR_WAKEUPS (1) // min times monitor wakes up per second
1286	#define KMP_MAX_MONITOR_WAKEUPS (1000) // max times monitor can wake up per sec
1287
1288	/* Calculate new number of monitor wakeups for a specific block time based on
1289	previous monitor_wakeups. Only allow increasing number of wakeups */
1290	#define KMP_WAKEUPS_FROM_BLOCKTIME(blocktime, monitor_wakeups) \
1291	(((blocktime) == KMP_MAX_BLOCKTIME) ? (monitor_wakeups) \
1292	: ((blocktime) == KMP_MIN_BLOCKTIME) ? KMP_MAX_MONITOR_WAKEUPS \
1293	: ((monitor_wakeups) > (KMP_BLOCKTIME_MULTIPLIER / (blocktime))) \
1294	? (monitor_wakeups) \
1295	: (KMP_BLOCKTIME_MULTIPLIER) / (blocktime))
1296
1297	/* Calculate number of intervals for a specific block time based on
1298	monitor_wakeups */
1299	#define KMP_INTERVALS_FROM_BLOCKTIME(blocktime, monitor_wakeups) \
1300	(((blocktime) + (KMP_BLOCKTIME_MULTIPLIER / (monitor_wakeups)) - 1) / \
1301	(KMP_BLOCKTIME_MULTIPLIER / (monitor_wakeups)))
1302	#else
1303	#define KMP_BLOCKTIME(team, tid) \
1304	(get__bt_set(team, tid) ? get__blocktime(team, tid) : __kmp_dflt_blocktime)
1305	#if KMP_OS_UNIX && (KMP_ARCH_X86 || KMP_ARCH_X86_64)
1306	// HW TSC is used to reduce overhead (clock tick instead of nanosecond).
1307	extern kmp_uint64 __kmp_ticks_per_msec;
1308	extern kmp_uint64 __kmp_ticks_per_usec;
1309	#if KMP_COMPILER_ICC || KMP_COMPILER_ICX
1310	#define KMP_NOW() ((kmp_uint64)_rdtsc())
1311	#else
1312	#define KMP_NOW() __kmp_hardware_timestamp()
1313	#endif
1314	#define KMP_BLOCKTIME_INTERVAL(team, tid) \
1315	((kmp_uint64)KMP_BLOCKTIME(team, tid) * __kmp_ticks_per_usec)
1316	#define KMP_BLOCKING(goal, count) ((goal) > KMP_NOW())
1317	#else
1318	// System time is retrieved sporadically while blocking.
1319	extern kmp_uint64 __kmp_now_nsec();
1320	#define KMP_NOW() __kmp_now_nsec()
1321	#define KMP_BLOCKTIME_INTERVAL(team, tid) \
1322	((kmp_uint64)KMP_BLOCKTIME(team, tid) * (kmp_uint64)KMP_NSEC_PER_USEC)
1323	#define KMP_BLOCKING(goal, count) ((count) % 1000 != 0 || (goal) > KMP_NOW())
1324	#endif
1325	#endif // KMP_USE_MONITOR
1326
1327	#define KMP_MIN_STATSCOLS 40
1328	#define KMP_MAX_STATSCOLS 4096
1329	#define KMP_DEFAULT_STATSCOLS 80
1330
1331	#define KMP_MIN_INTERVAL 0
1332	#define KMP_MAX_INTERVAL (INT_MAX - 1)
1333	#define KMP_DEFAULT_INTERVAL 0
1334
1335	#define KMP_MIN_CHUNK 1
1336	#define KMP_MAX_CHUNK (INT_MAX - 1)
1337	#define KMP_DEFAULT_CHUNK 1
1338
1339	#define KMP_MIN_DISP_NUM_BUFF 1
1340	#define KMP_DFLT_DISP_NUM_BUFF 7
1341	#define KMP_MAX_DISP_NUM_BUFF 4096
1342
1343	#define KMP_MAX_ORDERED 8
1344
1345	#define KMP_MAX_FIELDS 32
1346
1347	#define KMP_MAX_BRANCH_BITS 31
1348
1349	#define KMP_MAX_ACTIVE_LEVELS_LIMIT INT_MAX
1350
1351	#define KMP_MAX_DEFAULT_DEVICE_LIMIT INT_MAX
1352
1353	#define KMP_MAX_TASK_PRIORITY_LIMIT INT_MAX
1354
1355	/* Minimum number of threads before switch to TLS gtid (experimentally
1356	determined) */
1357	/* josh TODO: what about OS X* tuning? */
1358	#if KMP_ARCH_X86 || KMP_ARCH_X86_64
1359	#define KMP_TLS_GTID_MIN 5
1360	#else
1361	#define KMP_TLS_GTID_MIN INT_MAX
1362	#endif
1363
1364	#define KMP_MASTER_TID(tid) (0 == (tid))
1365	#define KMP_WORKER_TID(tid) (0 != (tid))
1366
1367	#define KMP_MASTER_GTID(gtid) (0 == __kmp_tid_from_gtid((gtid)))
1368	#define KMP_WORKER_GTID(gtid) (0 != __kmp_tid_from_gtid((gtid)))
1369	#define KMP_INITIAL_GTID(gtid) (0 == (gtid))
1370
1371	#ifndef TRUE
1372	#define FALSE 0
1373	#define TRUE (!FALSE)
1374	#endif
1375
1376	/* NOTE: all of the following constants must be even */
1377
1378	#if KMP_OS_WINDOWS
1379	#define KMP_INIT_WAIT 64U /* initial number of spin-tests */
1380	#define KMP_NEXT_WAIT 32U /* susequent number of spin-tests */
1381	#elif KMP_OS_LINUX
1382	#define KMP_INIT_WAIT 1024U /* initial number of spin-tests */
1383	#define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */
1384	#elif KMP_OS_DARWIN
1385	/* TODO: tune for KMP_OS_DARWIN */
1386	#define KMP_INIT_WAIT 1024U /* initial number of spin-tests */
1387	#define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */
1388	#elif KMP_OS_DRAGONFLY
1389	/* TODO: tune for KMP_OS_DRAGONFLY */
1390	#define KMP_INIT_WAIT 1024U /* initial number of spin-tests */
1391	#define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */
1392	#elif KMP_OS_FREEBSD
1393	/* TODO: tune for KMP_OS_FREEBSD */
1394	#define KMP_INIT_WAIT 1024U /* initial number of spin-tests */
1395	#define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */
1396	#elif KMP_OS_NETBSD
1397	/* TODO: tune for KMP_OS_NETBSD */
1398	#define KMP_INIT_WAIT 1024U /* initial number of spin-tests */
1399	#define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */
1400	#elif KMP_OS_OPENBSD
1401	/* TODO: tune for KMP_OS_OPENBSD */
1402	#define KMP_INIT_WAIT 1024U /* initial number of spin-tests */
1403	#define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */
1404	#elif KMP_OS_HAIKU
1405	/* TODO: tune for KMP_OS_HAIKU */
1406	#define KMP_INIT_WAIT 1024U /* initial number of spin-tests */
1407	#define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */
1408	#elif KMP_OS_HURD
1409	/* TODO: tune for KMP_OS_HURD */
1410	#define KMP_INIT_WAIT 1024U /* initial number of spin-tests */
1411	#define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */
1412	#elif KMP_OS_SOLARIS
1413	/* TODO: tune for KMP_OS_SOLARIS */
1414	#define KMP_INIT_WAIT 1024U /* initial number of spin-tests */
1415	#define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */
1416	#elif KMP_OS_WASI
1417	/* TODO: tune for KMP_OS_WASI */
1418	#define KMP_INIT_WAIT 1024U /* initial number of spin-tests */
1419	#define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */
1420	#elif KMP_OS_AIX
1421	/* TODO: tune for KMP_OS_AIX */
1422	#define KMP_INIT_WAIT 1024U /* initial number of spin-tests */
1423	#define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */
1424	#endif
1425
1426	#if KMP_ARCH_X86 || KMP_ARCH_X86_64
1427	typedef struct kmp_cpuid {
1428	kmp_uint32 eax;
1429	kmp_uint32 ebx;
1430	kmp_uint32 ecx;
1431	kmp_uint32 edx;
1432	} kmp_cpuid_t;
1433
1434	typedef struct kmp_cpuinfo_flags_t {
1435	unsigned sse2 : 1; // 0 if SSE2 instructions are not supported, 1 otherwise.
1436	unsigned rtm : 1; // 0 if RTM instructions are not supported, 1 otherwise.
1437	unsigned hybrid : 1;
1438	unsigned reserved : 29; // Ensure size of 32 bits
1439	} kmp_cpuinfo_flags_t;
1440
1441	typedef struct kmp_cpuinfo {
1442	int initialized; // If 0, other fields are not initialized.
1443	int signature; // CPUID(1).EAX
1444	int family; // CPUID(1).EAX[27:20]+CPUID(1).EAX[11:8] (Extended Family+Family)
1445	int model; // ( CPUID(1).EAX[19:16] << 4 ) + CPUID(1).EAX[7:4] ( ( Extended
1446	// Model << 4 ) + Model)
1447	int stepping; // CPUID(1).EAX[3:0] ( Stepping )
1448	kmp_cpuinfo_flags_t flags;
1449	int apic_id;
1450	kmp_uint64 frequency; // Nominal CPU frequency in Hz.
1451	char name[3 * sizeof(kmp_cpuid_t)]; // CPUID(0x80000002,0x80000003,0x80000004)
1452	} kmp_cpuinfo_t;
1453
1454	extern void __kmp_query_cpuid(kmp_cpuinfo_t *p);
1455
1456	#if KMP_OS_UNIX
1457	// subleaf is only needed for cache and topology discovery and can be set to
1458	// zero in most cases
1459	static inline void __kmp_x86_cpuid(int leaf, int subleaf, struct kmp_cpuid *p) {
1460	__asm__ __volatile__("cpuid"
1461	: "=a"(p->eax), "=b"(p->ebx), "=c"(p->ecx), "=d"(p->edx)
1462	: "a"(leaf), "c"(subleaf));
1463	}
1464	// Load p into FPU control word
1465	static inline void __kmp_load_x87_fpu_control_word(const kmp_int16 *p) {
1466	__asm__ __volatile__("fldcw %0" : : "m"(*p));
1467	}
1468	// Store FPU control word into p
1469	static inline void __kmp_store_x87_fpu_control_word(kmp_int16 *p) {
1470	__asm__ __volatile__("fstcw %0" : "=m"(*p));
1471	}
1472	static inline void __kmp_clear_x87_fpu_status_word() {
1473	#if KMP_MIC
1474	// 32-bit protected mode x87 FPU state
1475	struct x87_fpu_state {
1476	unsigned cw;
1477	unsigned sw;
1478	unsigned tw;
1479	unsigned fip;
1480	unsigned fips;
1481	unsigned fdp;
1482	unsigned fds;
1483	};
1484	struct x87_fpu_state fpu_state = {0, 0, 0, 0, 0, 0, 0};
1485	__asm__ __volatile__("fstenv %0\n\t" // store FP env
1486	"andw $0x7f00, %1\n\t" // clear 0-7,15 bits of FP SW
1487	"fldenv %0\n\t" // load FP env back
1488	: "+m"(fpu_state), "+m"(fpu_state.sw));
1489	#else
1490	__asm__ __volatile__("fnclex");
1491	#endif // KMP_MIC
1492	}
1493	#if __SSE__
1494	static inline void __kmp_load_mxcsr(const kmp_uint32 *p) { _mm_setcsr(i: *p); }
1495	static inline void __kmp_store_mxcsr(kmp_uint32 *p) { *p = _mm_getcsr(); }
1496	#else
1497	static inline void __kmp_load_mxcsr(const kmp_uint32 *p) {}
1498	static inline void __kmp_store_mxcsr(kmp_uint32 *p) { *p = 0; }
1499	#endif
1500	#else
1501	// Windows still has these as external functions in assembly file
1502	extern void __kmp_x86_cpuid(int mode, int mode2, struct kmp_cpuid *p);
1503	extern void __kmp_load_x87_fpu_control_word(const kmp_int16 *p);
1504	extern void __kmp_store_x87_fpu_control_word(kmp_int16 *p);
1505	extern void __kmp_clear_x87_fpu_status_word();
1506	static inline void __kmp_load_mxcsr(const kmp_uint32 *p) { _mm_setcsr(*p); }
1507	static inline void __kmp_store_mxcsr(kmp_uint32 *p) { *p = _mm_getcsr(); }
1508	#endif // KMP_OS_UNIX
1509
1510	#define KMP_X86_MXCSR_MASK 0xffffffc0 /* ignore status flags (6 lsb) */
1511
1512	// User-level Monitor/Mwait
1513	#if KMP_HAVE_UMWAIT
1514	// We always try for UMWAIT first
1515	#if KMP_HAVE_WAITPKG_INTRINSICS
1516	#if KMP_HAVE_IMMINTRIN_H
1517	#include <immintrin.h>
1518	#elif KMP_HAVE_INTRIN_H
1519	#include <intrin.h>
1520	#endif
1521	#endif // KMP_HAVE_WAITPKG_INTRINSICS
1522
1523	KMP_ATTRIBUTE_TARGET_WAITPKG
1524	static inline int __kmp_tpause(uint32_t hint, uint64_t counter) {
1525	#if !KMP_HAVE_WAITPKG_INTRINSICS
1526	uint32_t timeHi = uint32_t(counter >> 32);
1527	uint32_t timeLo = uint32_t(counter & 0xffffffff);
1528	char flag;
1529	__asm__ volatile("#tpause\n.byte 0x66, 0x0F, 0xAE, 0xF1\n"
1530	"setb %0"
1531	// The "=q" restraint means any register accessible as rl
1532	// in 32-bit mode: a, b, c, and d;
1533	// in 64-bit mode: any integer register
1534	: "=q"(flag)
1535	: "a"(timeLo), "d"(timeHi), "c"(hint)
1536	:);
1537	return flag;
1538	#else
1539	return _tpause(control: hint, counter: counter);
1540	#endif
1541	}
1542	KMP_ATTRIBUTE_TARGET_WAITPKG
1543	static inline void __kmp_umonitor(void *cacheline) {
1544	#if !KMP_HAVE_WAITPKG_INTRINSICS
1545	__asm__ volatile("# umonitor\n.byte 0xF3, 0x0F, 0xAE, 0x01 "
1546	:
1547	: "a"(cacheline)
1548	:);
1549	#else
1550	_umonitor(address: cacheline);
1551	#endif
1552	}
1553	KMP_ATTRIBUTE_TARGET_WAITPKG
1554	static inline int __kmp_umwait(uint32_t hint, uint64_t counter) {
1555	#if !KMP_HAVE_WAITPKG_INTRINSICS
1556	uint32_t timeHi = uint32_t(counter >> 32);
1557	uint32_t timeLo = uint32_t(counter & 0xffffffff);
1558	char flag;
1559	__asm__ volatile("#umwait\n.byte 0xF2, 0x0F, 0xAE, 0xF1\n"
1560	"setb %0"
1561	// The "=q" restraint means any register accessible as rl
1562	// in 32-bit mode: a, b, c, and d;
1563	// in 64-bit mode: any integer register
1564	: "=q"(flag)
1565	: "a"(timeLo), "d"(timeHi), "c"(hint)
1566	:);
1567	return flag;
1568	#else
1569	return _umwait(control: hint, counter: counter);
1570	#endif
1571	}
1572	#elif KMP_HAVE_MWAIT
1573	#if KMP_OS_UNIX
1574	#include <pmmintrin.h>
1575	#else
1576	#include <intrin.h>
1577	#endif
1578	#if KMP_OS_UNIX
1579	__attribute__((target("sse3")))
1580	#endif
1581	static inline void
1582	__kmp_mm_monitor(void *cacheline, unsigned extensions, unsigned hints) {
1583	_mm_monitor(cacheline, extensions, hints);
1584	}
1585	#if KMP_OS_UNIX
1586	__attribute__((target("sse3")))
1587	#endif
1588	static inline void
1589	__kmp_mm_mwait(unsigned extensions, unsigned hints) {
1590	_mm_mwait(extensions, hints);
1591	}
1592	#endif // KMP_HAVE_UMWAIT
1593
1594	#if KMP_ARCH_X86
1595	extern void __kmp_x86_pause(void);
1596	#elif KMP_MIC
1597	// Performance testing on KNC (C0QS-7120 P/A/X/D, 61-core, 16 GB Memory) showed
1598	// regression after removal of extra PAUSE from spin loops. Changing
1599	// the delay from 100 to 300 showed even better performance than double PAUSE
1600	// on Spec OMP2001 and LCPC tasking tests, no regressions on EPCC.
1601	static inline void __kmp_x86_pause(void) { _mm_delay_32(300); }
1602	#else
1603	static inline void __kmp_x86_pause(void) { _mm_pause(); }
1604	#endif
1605	#define KMP_CPU_PAUSE() __kmp_x86_pause()
1606	#elif KMP_ARCH_PPC64
1607	#define KMP_PPC64_PRI_LOW() __asm__ volatile("or 1, 1, 1")
1608	#define KMP_PPC64_PRI_MED() __asm__ volatile("or 2, 2, 2")
1609	#define KMP_PPC64_PRI_LOC_MB() __asm__ volatile("" : : : "memory")
1610	#define KMP_CPU_PAUSE() \
1611	do { \
1612	KMP_PPC64_PRI_LOW(); \
1613	KMP_PPC64_PRI_MED(); \
1614	KMP_PPC64_PRI_LOC_MB(); \
1615	} while (0)
1616	#else
1617	#define KMP_CPU_PAUSE() /* nothing to do */
1618	#endif
1619
1620	#define KMP_INIT_YIELD(count) \
1621	{ (count) = __kmp_yield_init; }
1622
1623	#define KMP_INIT_BACKOFF(time) \
1624	{ (time) = __kmp_pause_init; }
1625
1626	#define KMP_OVERSUBSCRIBED \
1627	(TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc))
1628
1629	#define KMP_TRY_YIELD \
1630	((__kmp_use_yield == 1) || (__kmp_use_yield == 2 && (KMP_OVERSUBSCRIBED)))
1631
1632	#define KMP_TRY_YIELD_OVERSUB \
1633	((__kmp_use_yield == 1 || __kmp_use_yield == 2) && (KMP_OVERSUBSCRIBED))
1634
1635	#define KMP_YIELD(cond) \
1636	{ \
1637	KMP_CPU_PAUSE(); \
1638	if ((cond) && (KMP_TRY_YIELD)) \
1639	__kmp_yield(); \
1640	}
1641
1642	#define KMP_YIELD_OVERSUB() \
1643	{ \
1644	KMP_CPU_PAUSE(); \
1645	if ((KMP_TRY_YIELD_OVERSUB)) \
1646	__kmp_yield(); \
1647	}
1648
1649	// Note the decrement of 2 in the following Macros. With KMP_LIBRARY=turnaround,
1650	// there should be no yielding since initial value from KMP_INIT_YIELD() is odd.
1651	#define KMP_YIELD_SPIN(count) \
1652	{ \
1653	KMP_CPU_PAUSE(); \
1654	if (KMP_TRY_YIELD) { \
1655	(count) -= 2; \
1656	if (!(count)) { \
1657	__kmp_yield(); \
1658	(count) = __kmp_yield_next; \
1659	} \
1660	} \
1661	}
1662
1663	// If TPAUSE is available & enabled, use it. If oversubscribed, use the slower
1664	// (C0.2) state, which improves performance of other SMT threads on the same
1665	// core, otherwise, use the fast (C0.1) default state, or whatever the user has
1666	// requested. Uses a timed TPAUSE, and exponential backoff. If TPAUSE isn't
1667	// available, fall back to the regular CPU pause and yield combination.
1668	#if KMP_HAVE_UMWAIT
1669	#define KMP_TPAUSE_MAX_MASK ((kmp_uint64)0xFFFF)
1670	#define KMP_YIELD_OVERSUB_ELSE_SPIN(count, time) \
1671	{ \
1672	if (__kmp_tpause_enabled) { \
1673	if (KMP_OVERSUBSCRIBED) { \
1674	__kmp_tpause(0, (time)); \
1675	} else { \
1676	__kmp_tpause(__kmp_tpause_hint, (time)); \
1677	} \
1678	(time) = (time << 1 | 1) & KMP_TPAUSE_MAX_MASK; \
1679	} else { \
1680	KMP_CPU_PAUSE(); \
1681	if ((KMP_TRY_YIELD_OVERSUB)) { \
1682	__kmp_yield(); \
1683	} else if (__kmp_use_yield == 1) { \
1684	(count) -= 2; \
1685	if (!(count)) { \
1686	__kmp_yield(); \
1687	(count) = __kmp_yield_next; \
1688	} \
1689	} \
1690	} \
1691	}
1692	#else
1693	#define KMP_YIELD_OVERSUB_ELSE_SPIN(count, time) \
1694	{ \
1695	KMP_CPU_PAUSE(); \
1696	if ((KMP_TRY_YIELD_OVERSUB)) \
1697	__kmp_yield(); \
1698	else if (__kmp_use_yield == 1) { \
1699	(count) -= 2; \
1700	if (!(count)) { \
1701	__kmp_yield(); \
1702	(count) = __kmp_yield_next; \
1703	} \
1704	} \
1705	}
1706	#endif // KMP_HAVE_UMWAIT
1707
1708	/* ------------------------------------------------------------------------ */
1709	/* Support datatypes for the orphaned construct nesting checks. */
1710	/* ------------------------------------------------------------------------ */
1711
1712	/* When adding to this enum, add its corresponding string in cons_text_c[]
1713	* array in kmp_error.cpp */
1714	enum cons_type {
1715	ct_none,
1716	ct_parallel,
1717	ct_pdo,
1718	ct_pdo_ordered,
1719	ct_psections,
1720	ct_psingle,
1721	ct_critical,
1722	ct_ordered_in_parallel,
1723	ct_ordered_in_pdo,
1724	ct_master,
1725	ct_reduce,
1726	ct_barrier,
1727	ct_masked
1728	};
1729
1730	#define IS_CONS_TYPE_ORDERED(ct) ((ct) == ct_pdo_ordered)
1731
1732	struct cons_data {
1733	ident_t const *ident;
1734	enum cons_type type;
1735	int prev;
1736	kmp_user_lock_p
1737	name; /* address exclusively for critical section name comparison */
1738	};
1739
1740	struct cons_header {
1741	int p_top, w_top, s_top;
1742	int stack_size, stack_top;
1743	struct cons_data *stack_data;
1744	};
1745
1746	struct kmp_region_info {
1747	char *text;
1748	int offset[KMP_MAX_FIELDS];
1749	int length[KMP_MAX_FIELDS];
1750	};
1751
1752	/* ---------------------------------------------------------------------- */
1753	/* ---------------------------------------------------------------------- */
1754
1755	#if KMP_OS_WINDOWS
1756	typedef HANDLE kmp_thread_t;
1757	typedef DWORD kmp_key_t;
1758	#endif /* KMP_OS_WINDOWS */
1759
1760	#if KMP_OS_UNIX
1761	typedef pthread_t kmp_thread_t;
1762	typedef pthread_key_t kmp_key_t;
1763	#endif
1764
1765	extern kmp_key_t __kmp_gtid_threadprivate_key;
1766
1767	typedef struct kmp_sys_info {
1768	long maxrss; /* the maximum resident set size utilized (in kilobytes) */
1769	long minflt; /* the number of page faults serviced without any I/O */
1770	long majflt; /* the number of page faults serviced that required I/O */
1771	long nswap; /* the number of times a process was "swapped" out of memory */
1772	long inblock; /* the number of times the file system had to perform input */
1773	long oublock; /* the number of times the file system had to perform output */
1774	long nvcsw; /* the number of times a context switch was voluntarily */
1775	long nivcsw; /* the number of times a context switch was forced */
1776	} kmp_sys_info_t;
1777
1778	#if USE_ITT_BUILD
1779	// We cannot include "kmp_itt.h" due to circular dependency. Declare the only
1780	// required type here. Later we will check the type meets requirements.
1781	typedef int kmp_itt_mark_t;
1782	#define KMP_ITT_DEBUG 0
1783	#endif /* USE_ITT_BUILD */
1784
1785	typedef kmp_int32 kmp_critical_name[8];
1786
1787	/*!
1788	@ingroup PARALLEL
1789	The type for a microtask which gets passed to @ref __kmpc_fork_call().
1790	The arguments to the outlined function are
1791	@param global_tid the global thread identity of the thread executing the
1792	function.
1793	@param bound_tid the local identity of the thread executing the function
1794	@param ... pointers to shared variables accessed by the function.
1795	*/
1796	typedef void (*kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid, ...);
1797	typedef void (*kmpc_micro_bound)(kmp_int32 *bound_tid, kmp_int32 *bound_nth,
1798	...);
1799
1800	/*!
1801	@ingroup THREADPRIVATE
1802	@{
1803	*/
1804	/* ---------------------------------------------------------------------------
1805	*/
1806	/* Threadprivate initialization/finalization function declarations */
1807
1808	/* for non-array objects: __kmpc_threadprivate_register() */
1809
1810	/*!
1811	Pointer to the constructor function.
1812	The first argument is the <tt>this</tt> pointer
1813	*/
1814	typedef void *(*kmpc_ctor)(void *);
1815
1816	/*!
1817	Pointer to the destructor function.
1818	The first argument is the <tt>this</tt> pointer
1819	*/
1820	typedef void (*kmpc_dtor)(
1821	void * /*, size_t */); /* 2nd arg: magic number for KCC unused by Intel
1822	compiler */
1823	/*!
1824	Pointer to an alternate constructor.
1825	The first argument is the <tt>this</tt> pointer.
1826	*/
1827	typedef void *(*kmpc_cctor)(void *, void *);
1828
1829	/* for array objects: __kmpc_threadprivate_register_vec() */
1830	/* First arg: "this" pointer */
1831	/* Last arg: number of array elements */
1832	/*!
1833	Array constructor.
1834	First argument is the <tt>this</tt> pointer
1835	Second argument the number of array elements.
1836	*/
1837	typedef void *(*kmpc_ctor_vec)(void *, size_t);
1838	/*!
1839	Pointer to the array destructor function.
1840	The first argument is the <tt>this</tt> pointer
1841	Second argument the number of array elements.
1842	*/
1843	typedef void (*kmpc_dtor_vec)(void *, size_t);
1844	/*!
1845	Array constructor.
1846	First argument is the <tt>this</tt> pointer
1847	Third argument the number of array elements.
1848	*/
1849	typedef void *(*kmpc_cctor_vec)(void *, void *,
1850	size_t); /* function unused by compiler */
1851
1852	/*!
1853	@}
1854	*/
1855
1856	/* keeps tracked of threadprivate cache allocations for cleanup later */
1857	typedef struct kmp_cached_addr {
1858	void **addr; /* address of allocated cache */
1859	void ***compiler_cache; /* pointer to compiler's cache */
1860	void *data; /* pointer to global data */
1861	struct kmp_cached_addr *next; /* pointer to next cached address */
1862	} kmp_cached_addr_t;
1863
1864	struct private_data {
1865	struct private_data *next; /* The next descriptor in the list */
1866	void *data; /* The data buffer for this descriptor */
1867	int more; /* The repeat count for this descriptor */
1868	size_t size; /* The data size for this descriptor */
1869	};
1870
1871	struct private_common {
1872	struct private_common *next;
1873	struct private_common *link;
1874	void *gbl_addr;
1875	void *par_addr; /* par_addr == gbl_addr for PRIMARY thread */
1876	size_t cmn_size;
1877	};
1878
1879	struct shared_common {
1880	struct shared_common *next;
1881	struct private_data *pod_init;
1882	void *obj_init;
1883	void *gbl_addr;
1884	union {
1885	kmpc_ctor ctor;
1886	kmpc_ctor_vec ctorv;
1887	} ct;
1888	union {
1889	kmpc_cctor cctor;
1890	kmpc_cctor_vec cctorv;
1891	} cct;
1892	union {
1893	kmpc_dtor dtor;
1894	kmpc_dtor_vec dtorv;
1895	} dt;
1896	size_t vec_len;
1897	int is_vec;
1898	size_t cmn_size;
1899	};
1900
1901	#define KMP_HASH_TABLE_LOG2 9 /* log2 of the hash table size */
1902	#define KMP_HASH_TABLE_SIZE \
1903	(1 << KMP_HASH_TABLE_LOG2) /* size of the hash table */
1904	#define KMP_HASH_SHIFT 3 /* throw away this many low bits from the address */
1905	#define KMP_HASH(x) \
1906	((((kmp_uintptr_t)x) >> KMP_HASH_SHIFT) & (KMP_HASH_TABLE_SIZE - 1))
1907
1908	struct common_table {
1909	struct private_common *data[KMP_HASH_TABLE_SIZE];
1910	};
1911
1912	struct shared_table {
1913	struct shared_common *data[KMP_HASH_TABLE_SIZE];
1914	};
1915
1916	/* ------------------------------------------------------------------------ */
1917
1918	#if KMP_USE_HIER_SCHED
1919	// Shared barrier data that exists inside a single unit of the scheduling
1920	// hierarchy
1921	typedef struct kmp_hier_private_bdata_t {
1922	kmp_int32 num_active;
1923	kmp_uint64 index;
1924	kmp_uint64 wait_val[2];
1925	} kmp_hier_private_bdata_t;
1926	#endif
1927
1928	typedef struct kmp_sched_flags {
1929	unsigned ordered : 1;
1930	unsigned nomerge : 1;
1931	unsigned contains_last : 1;
1932	unsigned use_hier : 1; // Used in KMP_USE_HIER_SCHED code
1933	unsigned use_hybrid : 1; // Used in KMP_WEIGHTED_ITERATIONS_SUPPORTED code
1934	unsigned unused : 27;
1935	} kmp_sched_flags_t;
1936
1937	KMP_BUILD_ASSERT(sizeof(kmp_sched_flags_t) == 4);
1938
1939	#if KMP_STATIC_STEAL_ENABLED
1940	typedef struct KMP_ALIGN_CACHE dispatch_private_info32 {
1941	kmp_int32 count;
1942	kmp_int32 ub;
1943	/* Adding KMP_ALIGN_CACHE here doesn't help / can hurt performance */
1944	kmp_int32 lb;
1945	kmp_int32 st;
1946	kmp_int32 tc;
1947	kmp_lock_t *steal_lock; // lock used for chunk stealing
1948
1949	kmp_uint32 ordered_lower;
1950	kmp_uint32 ordered_upper;
1951
1952	// KMP_ALIGN(32) ensures (if the KMP_ALIGN macro is turned on)
1953	// a) parm3 is properly aligned and
1954	// b) all parm1-4 are on the same cache line.
1955	// Because of parm1-4 are used together, performance seems to be better
1956	// if they are on the same cache line (not measured though).
1957
1958	struct KMP_ALIGN(32) {
1959	kmp_int32 parm1;
1960	kmp_int32 parm2;
1961	kmp_int32 parm3;
1962	kmp_int32 parm4;
1963	};
1964
1965	#if KMP_WEIGHTED_ITERATIONS_SUPPORTED
1966	kmp_uint32 pchunks;
1967	kmp_uint32 num_procs_with_pcore;
1968	kmp_int32 first_thread_with_ecore;
1969	#endif
1970	#if KMP_OS_WINDOWS
1971	kmp_int32 last_upper;
1972	#endif /* KMP_OS_WINDOWS */
1973	} dispatch_private_info32_t;
1974
1975	#if CACHE_LINE <= 128
1976	KMP_BUILD_ASSERT(sizeof(dispatch_private_info32_t) <= 128);
1977	#endif
1978
1979	typedef struct KMP_ALIGN_CACHE dispatch_private_info64 {
1980	kmp_int64 count; // current chunk number for static & static-steal scheduling
1981	kmp_int64 ub; /* upper-bound */
1982	/* Adding KMP_ALIGN_CACHE here doesn't help / can hurt performance */
1983	kmp_int64 lb; /* lower-bound */
1984	kmp_int64 st; /* stride */
1985	kmp_int64 tc; /* trip count (number of iterations) */
1986	kmp_lock_t *steal_lock; // lock used for chunk stealing
1987
1988	kmp_uint64 ordered_lower;
1989	kmp_uint64 ordered_upper;
1990	/* parm[1-4] are used in different ways by different scheduling algorithms */
1991
1992	// KMP_ALIGN(32) ensures ( if the KMP_ALIGN macro is turned on )
1993	// a) parm3 is properly aligned and
1994	// b) all parm1-4 are in the same cache line.
1995	// Because of parm1-4 are used together, performance seems to be better
1996	// if they are in the same line (not measured though).
1997	struct KMP_ALIGN(32) {
1998	kmp_int64 parm1;
1999	kmp_int64 parm2;
2000	kmp_int64 parm3;
2001	kmp_int64 parm4;
2002	};
2003
2004	#if KMP_WEIGHTED_ITERATIONS_SUPPORTED
2005	kmp_uint64 pchunks;
2006	kmp_uint64 num_procs_with_pcore;
2007	kmp_int64 first_thread_with_ecore;
2008	#endif
2009
2010	#if KMP_OS_WINDOWS
2011	kmp_int64 last_upper;
2012	#endif /* KMP_OS_WINDOWS */
2013	} dispatch_private_info64_t;
2014
2015	#if CACHE_LINE <= 128
2016	KMP_BUILD_ASSERT(sizeof(dispatch_private_info64_t) <= 128);
2017	#endif
2018
2019	#else /* KMP_STATIC_STEAL_ENABLED */
2020	typedef struct KMP_ALIGN_CACHE dispatch_private_info32 {
2021	kmp_int32 lb;
2022	kmp_int32 ub;
2023	kmp_int32 st;
2024	kmp_int32 tc;
2025
2026	kmp_int32 parm1;
2027	kmp_int32 parm2;
2028	kmp_int32 parm3;
2029	kmp_int32 parm4;
2030
2031	kmp_int32 count;
2032
2033	kmp_uint32 ordered_lower;
2034	kmp_uint32 ordered_upper;
2035	#if KMP_OS_WINDOWS
2036	kmp_int32 last_upper;
2037	#endif /* KMP_OS_WINDOWS */
2038	} dispatch_private_info32_t;
2039
2040	typedef struct KMP_ALIGN_CACHE dispatch_private_info64 {
2041	kmp_int64 lb; /* lower-bound */
2042	kmp_int64 ub; /* upper-bound */
2043	kmp_int64 st; /* stride */
2044	kmp_int64 tc; /* trip count (number of iterations) */
2045
2046	/* parm[1-4] are used in different ways by different scheduling algorithms */
2047	kmp_int64 parm1;
2048	kmp_int64 parm2;
2049	kmp_int64 parm3;
2050	kmp_int64 parm4;
2051
2052	kmp_int64 count; /* current chunk number for static scheduling */
2053
2054	kmp_uint64 ordered_lower;
2055	kmp_uint64 ordered_upper;
2056	#if KMP_OS_WINDOWS
2057	kmp_int64 last_upper;
2058	#endif /* KMP_OS_WINDOWS */
2059	} dispatch_private_info64_t;
2060	#endif /* KMP_STATIC_STEAL_ENABLED */
2061
2062	typedef struct KMP_ALIGN_CACHE dispatch_private_info {
2063	union private_info {
2064	dispatch_private_info32_t p32;
2065	dispatch_private_info64_t p64;
2066	} u;
2067	enum sched_type schedule; /* scheduling algorithm */
2068	kmp_sched_flags_t flags; /* flags (e.g., ordered, nomerge, etc.) */
2069	std::atomic<kmp_uint32> steal_flag; // static_steal only, state of a buffer
2070	kmp_int32 ordered_bumped;
2071	// Stack of buffers for nest of serial regions
2072	struct dispatch_private_info *next;
2073	kmp_int32 type_size; /* the size of types in private_info */
2074	#if KMP_USE_HIER_SCHED
2075	kmp_int32 hier_id;
2076	void *parent; /* hierarchical scheduling parent pointer */
2077	#endif
2078	enum cons_type pushed_ws;
2079	} dispatch_private_info_t;
2080
2081	typedef struct dispatch_shared_info32 {
2082	/* chunk index under dynamic, number of idle threads under static-steal;
2083	iteration index otherwise */
2084	volatile kmp_uint32 iteration;
2085	volatile kmp_int32 num_done;
2086	volatile kmp_uint32 ordered_iteration;
2087	// Dummy to retain the structure size after making ordered_iteration scalar
2088	kmp_int32 ordered_dummy[KMP_MAX_ORDERED - 1];
2089	} dispatch_shared_info32_t;
2090
2091	typedef struct dispatch_shared_info64 {
2092	/* chunk index under dynamic, number of idle threads under static-steal;
2093	iteration index otherwise */
2094	volatile kmp_uint64 iteration;
2095	volatile kmp_int64 num_done;
2096	volatile kmp_uint64 ordered_iteration;
2097	// Dummy to retain the structure size after making ordered_iteration scalar
2098	kmp_int64 ordered_dummy[KMP_MAX_ORDERED - 3];
2099	} dispatch_shared_info64_t;
2100
2101	typedef struct dispatch_shared_info {
2102	union shared_info {
2103	dispatch_shared_info32_t s32;
2104	dispatch_shared_info64_t s64;
2105	} u;
2106	volatile kmp_uint32 buffer_index;
2107	volatile kmp_int32 doacross_buf_idx; // teamwise index
2108	volatile kmp_uint32 *doacross_flags; // shared array of iteration flags (0/1)
2109	kmp_int32 doacross_num_done; // count finished threads
2110	#if KMP_USE_HIER_SCHED
2111	void *hier;
2112	#endif
2113	#if KMP_HWLOC_ENABLED
2114	// When linking with libhwloc, the ORDERED EPCC test slows down on big
2115	// machines (> 48 cores). Performance analysis showed that a cache thrash
2116	// was occurring and this padding helps alleviate the problem.
2117	char padding[64];
2118	#endif // KMP_HWLOC_ENABLED
2119	} dispatch_shared_info_t;
2120
2121	typedef struct kmp_disp {
2122	/* Vector for ORDERED SECTION */
2123	void (*th_deo_fcn)(int *gtid, int *cid, ident_t *);
2124	/* Vector for END ORDERED SECTION */
2125	void (*th_dxo_fcn)(int *gtid, int *cid, ident_t *);
2126
2127	dispatch_shared_info_t *th_dispatch_sh_current;
2128	dispatch_private_info_t *th_dispatch_pr_current;
2129
2130	dispatch_private_info_t *th_disp_buffer;
2131	kmp_uint32 th_disp_index;
2132	kmp_int32 th_doacross_buf_idx; // thread's doacross buffer index
2133	volatile kmp_uint32 *th_doacross_flags; // pointer to shared array of flags
2134	kmp_int64 *th_doacross_info; // info on loop bounds
2135	#if KMP_USE_INTERNODE_ALIGNMENT
2136	char more_padding[INTERNODE_CACHE_LINE];
2137	#endif
2138	} kmp_disp_t;
2139
2140	/* ------------------------------------------------------------------------ */
2141	/* Barrier stuff */
2142
2143	/* constants for barrier state update */
2144	#define KMP_INIT_BARRIER_STATE 0 /* should probably start from zero */
2145	#define KMP_BARRIER_SLEEP_BIT 0 /* bit used for suspend/sleep part of state */
2146	#define KMP_BARRIER_UNUSED_BIT 1 // bit that must never be set for valid state
2147	#define KMP_BARRIER_BUMP_BIT 2 /* lsb used for bump of go/arrived state */
2148
2149	#define KMP_BARRIER_SLEEP_STATE (1 << KMP_BARRIER_SLEEP_BIT)
2150	#define KMP_BARRIER_UNUSED_STATE (1 << KMP_BARRIER_UNUSED_BIT)
2151	#define KMP_BARRIER_STATE_BUMP (1 << KMP_BARRIER_BUMP_BIT)
2152
2153	#if (KMP_BARRIER_SLEEP_BIT >= KMP_BARRIER_BUMP_BIT)
2154	#error "Barrier sleep bit must be smaller than barrier bump bit"
2155	#endif
2156	#if (KMP_BARRIER_UNUSED_BIT >= KMP_BARRIER_BUMP_BIT)
2157	#error "Barrier unused bit must be smaller than barrier bump bit"
2158	#endif
2159
2160	// Constants for release barrier wait state: currently, hierarchical only
2161	#define KMP_BARRIER_NOT_WAITING 0 // Normal state; worker not in wait_sleep
2162	#define KMP_BARRIER_OWN_FLAG \
2163	1 // Normal state; worker waiting on own b_go flag in release
2164	#define KMP_BARRIER_PARENT_FLAG \
2165	2 // Special state; worker waiting on parent's b_go flag in release
2166	#define KMP_BARRIER_SWITCH_TO_OWN_FLAG \
2167	3 // Special state; tells worker to shift from parent to own b_go
2168	#define KMP_BARRIER_SWITCHING \
2169	4 // Special state; worker resets appropriate flag on wake-up
2170
2171	#define KMP_NOT_SAFE_TO_REAP \
2172	0 // Thread th_reap_state: not safe to reap (tasking)
2173	#define KMP_SAFE_TO_REAP 1 // Thread th_reap_state: safe to reap (not tasking)
2174
2175	// The flag_type describes the storage used for the flag.
2176	enum flag_type {
2177	flag32, /**< atomic 32 bit flags */
2178	flag64, /**< 64 bit flags */
2179	atomic_flag64, /**< atomic 64 bit flags */
2180	flag_oncore, /**< special 64-bit flag for on-core barrier (hierarchical) */
2181	flag_unset
2182	};
2183
2184	enum barrier_type {
2185	bs_plain_barrier = 0, /* 0, All non-fork/join barriers (except reduction
2186	barriers if enabled) */
2187	bs_forkjoin_barrier, /* 1, All fork/join (parallel region) barriers */
2188	#if KMP_FAST_REDUCTION_BARRIER
2189	bs_reduction_barrier, /* 2, All barriers that are used in reduction */
2190	#endif // KMP_FAST_REDUCTION_BARRIER
2191	bs_last_barrier /* Just a placeholder to mark the end */
2192	};
2193
2194	// to work with reduction barriers just like with plain barriers
2195	#if !KMP_FAST_REDUCTION_BARRIER
2196	#define bs_reduction_barrier bs_plain_barrier
2197	#endif // KMP_FAST_REDUCTION_BARRIER
2198
2199	typedef enum kmp_bar_pat { /* Barrier communication patterns */
2200	bp_linear_bar =
2201	0, /* Single level (degenerate) tree */
2202	bp_tree_bar =
2203	1, /* Balanced tree with branching factor 2^n */
2204	bp_hyper_bar = 2, /* Hypercube-embedded tree with min
2205	branching factor 2^n */
2206	bp_hierarchical_bar = 3, /* Machine hierarchy tree */
2207	bp_dist_bar = 4, /* Distributed barrier */
2208	bp_last_bar /* Placeholder to mark the end */
2209	} kmp_bar_pat_e;
2210
2211	#define KMP_BARRIER_ICV_PUSH 1
2212
2213	/* Record for holding the values of the internal controls stack records */
2214	typedef struct kmp_internal_control {
2215	int serial_nesting_level; /* corresponds to the value of the
2216	th_team_serialized field */
2217	kmp_int8 dynamic; /* internal control for dynamic adjustment of threads (per
2218	thread) */
2219	kmp_int8
2220	bt_set; /* internal control for whether blocktime is explicitly set */
2221	int blocktime; /* internal control for blocktime */
2222	#if KMP_USE_MONITOR
2223	int bt_intervals; /* internal control for blocktime intervals */
2224	#endif
2225	int nproc; /* internal control for #threads for next parallel region (per
2226	thread) */
2227	int thread_limit; /* internal control for thread-limit-var */
2228	int task_thread_limit; /* internal control for thread-limit-var of a task*/
2229	int max_active_levels; /* internal control for max_active_levels */
2230	kmp_r_sched_t
2231	sched; /* internal control for runtime schedule {sched,chunk} pair */
2232	kmp_proc_bind_t proc_bind; /* internal control for affinity */
2233	kmp_int32 default_device; /* internal control for default device */
2234	struct kmp_internal_control *next;
2235	} kmp_internal_control_t;
2236
2237	static inline void copy_icvs(kmp_internal_control_t *dst,
2238	kmp_internal_control_t *src) {
2239	*dst = *src;
2240	}
2241
2242	/* Thread barrier needs volatile barrier fields */
2243	typedef struct KMP_ALIGN_CACHE kmp_bstate {
2244	// th_fixed_icvs is aligned by virtue of kmp_bstate being aligned (and all
2245	// uses of it). It is not explicitly aligned below, because we *don't* want
2246	// it to be padded -- instead, we fit b_go into the same cache line with
2247	// th_fixed_icvs, enabling NGO cache lines stores in the hierarchical barrier.
2248	kmp_internal_control_t th_fixed_icvs; // Initial ICVs for the thread
2249	// Tuck b_go into end of th_fixed_icvs cache line, so it can be stored with
2250	// same NGO store
2251	volatile kmp_uint64 b_go; // STATE => task should proceed (hierarchical)
2252	KMP_ALIGN_CACHE volatile kmp_uint64
2253	b_arrived; // STATE => task reached synch point.
2254	kmp_uint32 *skip_per_level;
2255	kmp_uint32 my_level;
2256	kmp_int32 parent_tid;
2257	kmp_int32 old_tid;
2258	kmp_uint32 depth;
2259	struct kmp_bstate *parent_bar;
2260	kmp_team_t *team;
2261	kmp_uint64 leaf_state;
2262	kmp_uint32 nproc;
2263	kmp_uint8 base_leaf_kids;
2264	kmp_uint8 leaf_kids;
2265	kmp_uint8 offset;
2266	kmp_uint8 wait_flag;
2267	kmp_uint8 use_oncore_barrier;
2268	#if USE_DEBUGGER
2269	// The following field is intended for the debugger solely. Only the worker
2270	// thread itself accesses this field: the worker increases it by 1 when it
2271	// arrives to a barrier.
2272	KMP_ALIGN_CACHE kmp_uint b_worker_arrived;
2273	#endif /* USE_DEBUGGER */
2274	} kmp_bstate_t;
2275
2276	union KMP_ALIGN_CACHE kmp_barrier_union {
2277	double b_align; /* use worst case alignment */
2278	char b_pad[KMP_PAD(kmp_bstate_t, CACHE_LINE)];
2279	kmp_bstate_t bb;
2280	};
2281
2282	typedef union kmp_barrier_union kmp_balign_t;
2283
2284	/* Team barrier needs only non-volatile arrived counter */
2285	union KMP_ALIGN_CACHE kmp_barrier_team_union {
2286	double b_align; /* use worst case alignment */
2287	char b_pad[CACHE_LINE];
2288	struct {
2289	kmp_uint64 b_arrived; /* STATE => task reached synch point. */
2290	#if USE_DEBUGGER
2291	// The following two fields are indended for the debugger solely. Only
2292	// primary thread of the team accesses these fields: the first one is
2293	// increased by 1 when the primary thread arrives to a barrier, the second
2294	// one is increased by one when all the threads arrived.
2295	kmp_uint b_master_arrived;
2296	kmp_uint b_team_arrived;
2297	#endif
2298	};
2299	};
2300
2301	typedef union kmp_barrier_team_union kmp_balign_team_t;
2302
2303	/* Padding for Linux* OS pthreads condition variables and mutexes used to signal
2304	threads when a condition changes. This is to workaround an NPTL bug where
2305	padding was added to pthread_cond_t which caused the initialization routine
2306	to write outside of the structure if compiled on pre-NPTL threads. */
2307	#if KMP_OS_WINDOWS
2308	typedef struct kmp_win32_mutex {
2309	/* The Lock */
2310	CRITICAL_SECTION cs;
2311	} kmp_win32_mutex_t;
2312
2313	typedef struct kmp_win32_cond {
2314	/* Count of the number of waiters. */
2315	int waiters_count_;
2316
2317	/* Serialize access to <waiters_count_> */
2318	kmp_win32_mutex_t waiters_count_lock_;
2319
2320	/* Number of threads to release via a <cond_broadcast> or a <cond_signal> */
2321	int release_count_;
2322
2323	/* Keeps track of the current "generation" so that we don't allow */
2324	/* one thread to steal all the "releases" from the broadcast. */
2325	int wait_generation_count_;
2326
2327	/* A manual-reset event that's used to block and release waiting threads. */
2328	HANDLE event_;
2329	} kmp_win32_cond_t;
2330	#endif
2331
2332	#if KMP_OS_UNIX
2333
2334	union KMP_ALIGN_CACHE kmp_cond_union {
2335	double c_align;
2336	char c_pad[CACHE_LINE];
2337	pthread_cond_t c_cond;
2338	};
2339
2340	typedef union kmp_cond_union kmp_cond_align_t;
2341
2342	union KMP_ALIGN_CACHE kmp_mutex_union {
2343	double m_align;
2344	char m_pad[CACHE_LINE];
2345	pthread_mutex_t m_mutex;
2346	};
2347
2348	typedef union kmp_mutex_union kmp_mutex_align_t;
2349
2350	#endif /* KMP_OS_UNIX */
2351
2352	typedef struct kmp_desc_base {
2353	void *ds_stackbase;
2354	size_t ds_stacksize;
2355	int ds_stackgrow;
2356	kmp_thread_t ds_thread;
2357	volatile int ds_tid;
2358	int ds_gtid;
2359	#if KMP_OS_WINDOWS
2360	volatile int ds_alive;
2361	DWORD ds_thread_id;
2362	/* ds_thread keeps thread handle on Windows* OS. It is enough for RTL purposes.
2363	However, debugger support (libomp_db) cannot work with handles, because they
2364	uncomparable. For example, debugger requests info about thread with handle h.
2365	h is valid within debugger process, and meaningless within debugee process.
2366	Even if h is duped by call to DuplicateHandle(), so the result h' is valid
2367	within debugee process, but it is a *new* handle which does *not* equal to
2368	any other handle in debugee... The only way to compare handles is convert
2369	them to system-wide ids. GetThreadId() function is available only in
2370	Longhorn and Server 2003. :-( In contrast, GetCurrentThreadId() is available
2371	on all Windows* OS flavours (including Windows* 95). Thus, we have to get
2372	thread id by call to GetCurrentThreadId() from within the thread and save it
2373	to let libomp_db identify threads. */
2374	#endif /* KMP_OS_WINDOWS */
2375	} kmp_desc_base_t;
2376
2377	typedef union KMP_ALIGN_CACHE kmp_desc {
2378	double ds_align; /* use worst case alignment */
2379	char ds_pad[KMP_PAD(kmp_desc_base_t, CACHE_LINE)];
2380	kmp_desc_base_t ds;
2381	} kmp_desc_t;
2382
2383	typedef struct kmp_local {
2384	volatile int this_construct; /* count of single's encountered by thread */
2385	void *reduce_data;
2386	#if KMP_USE_BGET
2387	void *bget_data;
2388	void *bget_list;
2389	#if !USE_CMP_XCHG_FOR_BGET
2390	#ifdef USE_QUEUING_LOCK_FOR_BGET
2391	kmp_lock_t bget_lock; /* Lock for accessing bget free list */
2392	#else
2393	kmp_bootstrap_lock_t bget_lock; // Lock for accessing bget free list. Must be
2394	// bootstrap lock so we can use it at library
2395	// shutdown.
2396	#endif /* USE_LOCK_FOR_BGET */
2397	#endif /* ! USE_CMP_XCHG_FOR_BGET */
2398	#endif /* KMP_USE_BGET */
2399
2400	PACKED_REDUCTION_METHOD_T
2401	packed_reduction_method; /* stored by __kmpc_reduce*(), used by
2402	__kmpc_end_reduce*() */
2403
2404	} kmp_local_t;
2405
2406	#define KMP_CHECK_UPDATE(a, b) \
2407	if ((a) != (b)) \
2408	(a) = (b)
2409	#define KMP_CHECK_UPDATE_SYNC(a, b) \
2410	if ((a) != (b)) \
2411	TCW_SYNC_PTR((a), (b))
2412
2413	#define get__blocktime(xteam, xtid) \
2414	((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.blocktime)
2415	#define get__bt_set(xteam, xtid) \
2416	((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.bt_set)
2417	#if KMP_USE_MONITOR
2418	#define get__bt_intervals(xteam, xtid) \
2419	((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.bt_intervals)
2420	#endif
2421
2422	#define get__dynamic_2(xteam, xtid) \
2423	((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.dynamic)
2424	#define get__nproc_2(xteam, xtid) \
2425	((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.nproc)
2426	#define get__sched_2(xteam, xtid) \
2427	((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.sched)
2428
2429	#define set__blocktime_team(xteam, xtid, xval) \
2430	(((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.blocktime) = \
2431	(xval))
2432
2433	#if KMP_USE_MONITOR
2434	#define set__bt_intervals_team(xteam, xtid, xval) \
2435	(((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.bt_intervals) = \
2436	(xval))
2437	#endif
2438
2439	#define set__bt_set_team(xteam, xtid, xval) \
2440	(((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.bt_set) = (xval))
2441
2442	#define set__dynamic(xthread, xval) \
2443	(((xthread)->th.th_current_task->td_icvs.dynamic) = (xval))
2444	#define get__dynamic(xthread) \
2445	(((xthread)->th.th_current_task->td_icvs.dynamic) ? (FTN_TRUE) : (FTN_FALSE))
2446
2447	#define set__nproc(xthread, xval) \
2448	(((xthread)->th.th_current_task->td_icvs.nproc) = (xval))
2449
2450	#define set__thread_limit(xthread, xval) \
2451	(((xthread)->th.th_current_task->td_icvs.thread_limit) = (xval))
2452
2453	#define set__max_active_levels(xthread, xval) \
2454	(((xthread)->th.th_current_task->td_icvs.max_active_levels) = (xval))
2455
2456	#define get__max_active_levels(xthread) \
2457	((xthread)->th.th_current_task->td_icvs.max_active_levels)
2458
2459	#define set__sched(xthread, xval) \
2460	(((xthread)->th.th_current_task->td_icvs.sched) = (xval))
2461
2462	#define set__proc_bind(xthread, xval) \
2463	(((xthread)->th.th_current_task->td_icvs.proc_bind) = (xval))
2464	#define get__proc_bind(xthread) \
2465	((xthread)->th.th_current_task->td_icvs.proc_bind)
2466
2467	// OpenMP tasking data structures
2468
2469	typedef enum kmp_tasking_mode {
2470	tskm_immediate_exec = 0,
2471	tskm_extra_barrier = 1,
2472	tskm_task_teams = 2,
2473	tskm_max = 2
2474	} kmp_tasking_mode_t;
2475
2476	extern kmp_tasking_mode_t
2477	__kmp_tasking_mode; /* determines how/when to execute tasks */
2478	extern int __kmp_task_stealing_constraint;
2479	extern int __kmp_enable_task_throttling;
2480	extern kmp_int32 __kmp_default_device; // Set via OMP_DEFAULT_DEVICE if
2481	// specified, defaults to 0 otherwise
2482	// Set via OMP_MAX_TASK_PRIORITY if specified, defaults to 0 otherwise
2483	extern kmp_int32 __kmp_max_task_priority;
2484	// Set via KMP_TASKLOOP_MIN_TASKS if specified, defaults to 0 otherwise
2485	extern kmp_uint64 __kmp_taskloop_min_tasks;
2486
2487	/* NOTE: kmp_taskdata_t and kmp_task_t structures allocated in single block with
2488	taskdata first */
2489	#define KMP_TASK_TO_TASKDATA(task) (((kmp_taskdata_t *)task) - 1)
2490	#define KMP_TASKDATA_TO_TASK(taskdata) (kmp_task_t *)(taskdata + 1)
2491
2492	// The tt_found_tasks flag is a signal to all threads in the team that tasks
2493	// were spawned and queued since the previous barrier release.
2494	#define KMP_TASKING_ENABLED(task_team) \
2495	(TRUE == TCR_SYNC_4((task_team)->tt.tt_found_tasks))
2496	/*!
2497	@ingroup BASIC_TYPES
2498	@{
2499	*/
2500
2501	/*!
2502	*/
2503	typedef kmp_int32 (*kmp_routine_entry_t)(kmp_int32, void *);
2504
2505	typedef union kmp_cmplrdata {
2506	kmp_int32 priority; /**< priority specified by user for the task */
2507	kmp_routine_entry_t
2508	destructors; /* pointer to function to invoke deconstructors of
2509	firstprivate C++ objects */
2510	/* future data */
2511	} kmp_cmplrdata_t;
2512
2513	/* sizeof_kmp_task_t passed as arg to kmpc_omp_task call */
2514	/*!
2515	*/
2516	typedef struct kmp_task { /* GEH: Shouldn't this be aligned somehow? */
2517	void *shareds; /**< pointer to block of pointers to shared vars */
2518	kmp_routine_entry_t
2519	routine; /**< pointer to routine to call for executing task */
2520	kmp_int32 part_id; /**< part id for the task */
2521	kmp_cmplrdata_t
2522	data1; /* Two known optional additions: destructors and priority */
2523	kmp_cmplrdata_t data2; /* Process destructors first, priority second */
2524	/* future data */
2525	/* private vars */
2526	} kmp_task_t;
2527
2528	/*!
2529	@}
2530	*/
2531
2532	typedef struct kmp_taskgroup {
2533	std::atomic<kmp_int32> count; // number of allocated and incomplete tasks
2534	std::atomic<kmp_int32>
2535	cancel_request; // request for cancellation of this taskgroup
2536	struct kmp_taskgroup *parent; // parent taskgroup
2537	// Block of data to perform task reduction
2538	void *reduce_data; // reduction related info
2539	kmp_int32 reduce_num_data; // number of data items to reduce
2540	uintptr_t *gomp_data; // gomp reduction data
2541	} kmp_taskgroup_t;
2542
2543	// forward declarations
2544	typedef union kmp_depnode kmp_depnode_t;
2545	typedef struct kmp_depnode_list kmp_depnode_list_t;
2546	typedef struct kmp_dephash_entry kmp_dephash_entry_t;
2547
2548	// macros for checking dep flag as an integer
2549	#define KMP_DEP_IN 0x1
2550	#define KMP_DEP_OUT 0x2
2551	#define KMP_DEP_INOUT 0x3
2552	#define KMP_DEP_MTX 0x4
2553	#define KMP_DEP_SET 0x8
2554	#define KMP_DEP_ALL 0x80
2555	// Compiler sends us this info. Note: some test cases contain an explicit copy
2556	// of this struct and should be in sync with any changes here.
2557	typedef struct kmp_depend_info {
2558	kmp_intptr_t base_addr;
2559	size_t len;
2560	union {
2561	kmp_uint8 flag; // flag as an unsigned char
2562	struct { // flag as a set of 8 bits
2563	#if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
2564	/* Same fields as in the #else branch, but in reverse order */
2565	unsigned all : 1;
2566	unsigned unused : 3;
2567	unsigned set : 1;
2568	unsigned mtx : 1;
2569	unsigned out : 1;
2570	unsigned in : 1;
2571	#else
2572	unsigned in : 1;
2573	unsigned out : 1;
2574	unsigned mtx : 1;
2575	unsigned set : 1;
2576	unsigned unused : 3;
2577	unsigned all : 1;
2578	#endif
2579	} flags;
2580	};
2581	} kmp_depend_info_t;
2582
2583	// Internal structures to work with task dependencies:
2584	struct kmp_depnode_list {
2585	kmp_depnode_t *node;
2586	kmp_depnode_list_t *next;
2587	};
2588
2589	// Max number of mutexinoutset dependencies per node
2590	#define MAX_MTX_DEPS 4
2591
2592	typedef struct kmp_base_depnode {
2593	kmp_depnode_list_t *successors; /* used under lock */
2594	kmp_task_t *task; /* non-NULL if depnode is active, used under lock */
2595	kmp_lock_t *mtx_locks[MAX_MTX_DEPS]; /* lock mutexinoutset dependent tasks */
2596	kmp_int32 mtx_num_locks; /* number of locks in mtx_locks array */
2597	kmp_lock_t lock; /* guards shared fields: task, successors */
2598	#if KMP_SUPPORT_GRAPH_OUTPUT
2599	kmp_uint32 id;
2600	#endif
2601	std::atomic<kmp_int32> npredecessors;
2602	std::atomic<kmp_int32> nrefs;
2603	} kmp_base_depnode_t;
2604
2605	union KMP_ALIGN_CACHE kmp_depnode {
2606	double dn_align; /* use worst case alignment */
2607	char dn_pad[KMP_PAD(kmp_base_depnode_t, CACHE_LINE)];
2608	kmp_base_depnode_t dn;
2609	};
2610
2611	struct kmp_dephash_entry {
2612	kmp_intptr_t addr;
2613	kmp_depnode_t *last_out;
2614	kmp_depnode_list_t *last_set;
2615	kmp_depnode_list_t *prev_set;
2616	kmp_uint8 last_flag;
2617	kmp_lock_t *mtx_lock; /* is referenced by depnodes w/mutexinoutset dep */
2618	kmp_dephash_entry_t *next_in_bucket;
2619	};
2620
2621	typedef struct kmp_dephash {
2622	kmp_dephash_entry_t **buckets;
2623	size_t size;
2624	kmp_depnode_t *last_all;
2625	size_t generation;
2626	kmp_uint32 nelements;
2627	kmp_uint32 nconflicts;
2628	} kmp_dephash_t;
2629
2630	typedef struct kmp_task_affinity_info {
2631	kmp_intptr_t base_addr;
2632	size_t len;
2633	struct {
2634	bool flag1 : 1;
2635	bool flag2 : 1;
2636	kmp_int32 reserved : 30;
2637	} flags;
2638	} kmp_task_affinity_info_t;
2639
2640	typedef enum kmp_event_type_t {
2641	KMP_EVENT_UNINITIALIZED = 0,
2642	KMP_EVENT_ALLOW_COMPLETION = 1
2643	} kmp_event_type_t;
2644
2645	typedef struct {
2646	kmp_event_type_t type;
2647	kmp_tas_lock_t lock;
2648	union {
2649	kmp_task_t *task;
2650	} ed;
2651	} kmp_event_t;
2652
2653	#if OMPX_TASKGRAPH
2654	// Initial number of allocated nodes while recording
2655	#define INIT_MAPSIZE 50
2656
2657	typedef struct kmp_taskgraph_flags { /*This needs to be exactly 32 bits */
2658	unsigned nowait : 1;
2659	unsigned re_record : 1;
2660	unsigned reserved : 30;
2661	} kmp_taskgraph_flags_t;
2662
2663	/// Represents a TDG node
2664	typedef struct kmp_node_info {
2665	kmp_task_t *task; // Pointer to the actual task
2666	kmp_int32 *successors; // Array of the succesors ids
2667	kmp_int32 nsuccessors; // Number of succesors of the node
2668	std::atomic<kmp_int32>
2669	npredecessors_counter; // Number of predessors on the fly
2670	kmp_int32 npredecessors; // Total number of predecessors
2671	kmp_int32 successors_size; // Number of allocated succesors ids
2672	kmp_taskdata_t *parent_task; // Parent implicit task
2673	} kmp_node_info_t;
2674
2675	/// Represent a TDG's current status
2676	typedef enum kmp_tdg_status {
2677	KMP_TDG_NONE = 0,
2678	KMP_TDG_RECORDING = 1,
2679	KMP_TDG_READY = 2
2680	} kmp_tdg_status_t;
2681
2682	/// Structure that contains a TDG
2683	typedef struct kmp_tdg_info {
2684	kmp_int32 tdg_id; // Unique idenfifier of the TDG
2685	kmp_taskgraph_flags_t tdg_flags; // Flags related to a TDG
2686	kmp_int32 map_size; // Number of allocated TDG nodes
2687	kmp_int32 num_roots; // Number of roots tasks int the TDG
2688	kmp_int32 *root_tasks; // Array of tasks identifiers that are roots
2689	kmp_node_info_t *record_map; // Array of TDG nodes
2690	kmp_tdg_status_t tdg_status =
2691	KMP_TDG_NONE; // Status of the TDG (recording, ready...)
2692	std::atomic<kmp_int32> num_tasks; // Number of TDG nodes
2693	kmp_bootstrap_lock_t
2694	graph_lock; // Protect graph attributes when updated via taskloop_recur
2695	// Taskloop reduction related
2696	void *rec_taskred_data; // Data to pass to __kmpc_task_reduction_init or
2697	// __kmpc_taskred_init
2698	kmp_int32 rec_num_taskred;
2699	} kmp_tdg_info_t;
2700
2701	extern int __kmp_tdg_dot;
2702	extern kmp_int32 __kmp_max_tdgs;
2703	extern kmp_tdg_info_t **__kmp_global_tdgs;
2704	extern kmp_int32 __kmp_curr_tdg_idx;
2705	extern kmp_int32 __kmp_successors_size;
2706	extern std::atomic<kmp_int32> __kmp_tdg_task_id;
2707	extern kmp_int32 __kmp_num_tdg;
2708	#endif
2709
2710	#ifdef BUILD_TIED_TASK_STACK
2711
2712	/* Tied Task stack definitions */
2713	typedef struct kmp_stack_block {
2714	kmp_taskdata_t *sb_block[TASK_STACK_BLOCK_SIZE];
2715	struct kmp_stack_block *sb_next;
2716	struct kmp_stack_block *sb_prev;
2717	} kmp_stack_block_t;
2718
2719	typedef struct kmp_task_stack {
2720	kmp_stack_block_t ts_first_block; // first block of stack entries
2721	kmp_taskdata_t **ts_top; // pointer to the top of stack
2722	kmp_int32 ts_entries; // number of entries on the stack
2723	} kmp_task_stack_t;
2724
2725	#endif // BUILD_TIED_TASK_STACK
2726
2727	typedef struct kmp_tasking_flags { /* Total struct must be exactly 32 bits */
2728	#if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
2729	/* Same fields as in the #else branch, but in reverse order */
2730	#if OMPX_TASKGRAPH
2731	unsigned reserved31 : 4;
2732	unsigned onced : 1;
2733	#else
2734	unsigned reserved31 : 5;
2735	#endif
2736	unsigned hidden_helper : 1;
2737	unsigned target : 1;
2738	unsigned native : 1;
2739	unsigned freed : 1;
2740	unsigned complete : 1;
2741	unsigned executing : 1;
2742	unsigned started : 1;
2743	unsigned team_serial : 1;
2744	unsigned tasking_ser : 1;
2745	unsigned task_serial : 1;
2746	unsigned tasktype : 1;
2747	unsigned reserved : 8;
2748	unsigned free_agent_eligible : 1;
2749	unsigned detachable : 1;
2750	unsigned priority_specified : 1;
2751	unsigned proxy : 1;
2752	unsigned destructors_thunk : 1;
2753	unsigned merged_if0 : 1;
2754	unsigned final : 1;
2755	unsigned tiedness : 1;
2756	#else
2757	/* Compiler flags */ /* Total compiler flags must be 16 bits */
2758	unsigned tiedness : 1; /* task is either tied (1) or untied (0) */
2759	unsigned final : 1; /* task is final(1) so execute immediately */
2760	unsigned merged_if0 : 1; /* no __kmpc_task_{begin/complete}_if0 calls in if0
2761	code path */
2762	unsigned destructors_thunk : 1; /* set if the compiler creates a thunk to
2763	invoke destructors from the runtime */
2764	unsigned proxy : 1; /* task is a proxy task (it will be executed outside the
2765	context of the RTL) */
2766	unsigned priority_specified : 1; /* set if the compiler provides priority
2767	setting for the task */
2768	unsigned detachable : 1; /* 1 == can detach */
2769	unsigned free_agent_eligible : 1; /* set if task can be executed by a
2770	free-agent thread */
2771	unsigned reserved : 8; /* reserved for compiler use */
2772
2773	/* Library flags */ /* Total library flags must be 16 bits */
2774	unsigned tasktype : 1; /* task is either explicit(1) or implicit (0) */
2775	unsigned task_serial : 1; // task is executed immediately (1) or deferred (0)
2776	unsigned tasking_ser : 1; // all tasks in team are either executed immediately
2777	// (1) or may be deferred (0)
2778	unsigned team_serial : 1; // entire team is serial (1) [1 thread] or parallel
2779	// (0) [>= 2 threads]
2780	/* If either team_serial or tasking_ser is set, task team may be NULL */
2781	/* Task State Flags: */
2782	unsigned started : 1; /* 1==started, 0==not started */
2783	unsigned executing : 1; /* 1==executing, 0==not executing */
2784	unsigned complete : 1; /* 1==complete, 0==not complete */
2785	unsigned freed : 1; /* 1==freed, 0==allocated */
2786	unsigned native : 1; /* 1==gcc-compiled task, 0==intel */
2787	unsigned target : 1;
2788	unsigned hidden_helper : 1; /* 1 == hidden helper task */
2789	#if OMPX_TASKGRAPH
2790	unsigned onced : 1; /* 1==ran once already, 0==never ran, record & replay purposes */
2791	unsigned reserved31 : 4; /* reserved for library use */
2792	#else
2793	unsigned reserved31 : 5; /* reserved for library use */
2794	#endif
2795	#endif
2796	} kmp_tasking_flags_t;
2797
2798	typedef struct kmp_target_data {
2799	void *async_handle; // libomptarget async handle for task completion query
2800	} kmp_target_data_t;
2801
2802	struct kmp_taskdata { /* aligned during dynamic allocation */
2803	kmp_int32 td_task_id; /* id, assigned by debugger */
2804	kmp_tasking_flags_t td_flags; /* task flags */
2805	kmp_team_t *td_team; /* team for this task */
2806	kmp_info_p *td_alloc_thread; /* thread that allocated data structures */
2807	/* Currently not used except for perhaps IDB */
2808	kmp_taskdata_t *td_parent; /* parent task */
2809	kmp_int32 td_level; /* task nesting level */
2810	std::atomic<kmp_int32> td_untied_count; // untied task active parts counter
2811	ident_t *td_ident; /* task identifier */
2812	// Taskwait data.
2813	ident_t *td_taskwait_ident;
2814	kmp_uint32 td_taskwait_counter;
2815	kmp_int32 td_taskwait_thread; /* gtid + 1 of thread encountered taskwait */
2816	KMP_ALIGN_CACHE kmp_internal_control_t
2817	td_icvs; /* Internal control variables for the task */
2818	KMP_ALIGN_CACHE std::atomic<kmp_int32>
2819	td_allocated_child_tasks; /* Child tasks (+ current task) not yet
2820	deallocated */
2821	std::atomic<kmp_int32>
2822	td_incomplete_child_tasks; /* Child tasks not yet complete */
2823	kmp_taskgroup_t
2824	*td_taskgroup; // Each task keeps pointer to its current taskgroup
2825	kmp_dephash_t
2826	*td_dephash; // Dependencies for children tasks are tracked from here
2827	kmp_depnode_t
2828	*td_depnode; // Pointer to graph node if this task has dependencies
2829	kmp_task_team_t *td_task_team;
2830	size_t td_size_alloc; // Size of task structure, including shareds etc.
2831	#if defined(KMP_GOMP_COMPAT)
2832	// 4 or 8 byte integers for the loop bounds in GOMP_taskloop
2833	kmp_int32 td_size_loop_bounds;
2834	#endif
2835	kmp_taskdata_t *td_last_tied; // keep tied task for task scheduling constraint
2836	#if defined(KMP_GOMP_COMPAT)
2837	// GOMP sends in a copy function for copy constructors
2838	void (*td_copy_func)(void *, void *);
2839	#endif
2840	kmp_event_t td_allow_completion_event;
2841	#if OMPT_SUPPORT
2842	ompt_task_info_t ompt_task_info;
2843	#endif
2844	#if OMPX_TASKGRAPH
2845	bool is_taskgraph = 0; // whether the task is within a TDG
2846	kmp_tdg_info_t *tdg; // used to associate task with a TDG
2847	kmp_int32 td_tdg_task_id; // local task id in its TDG
2848	#endif
2849	kmp_target_data_t td_target_data;
2850	}; // struct kmp_taskdata
2851
2852	// Make sure padding above worked
2853	KMP_BUILD_ASSERT(sizeof(kmp_taskdata_t) % sizeof(void *) == 0);
2854
2855	// Data for task team but per thread
2856	typedef struct kmp_base_thread_data {
2857	kmp_info_p *td_thr; // Pointer back to thread info
2858	// Used only in __kmp_execute_tasks_template, maybe not avail until task is
2859	// queued?
2860	kmp_bootstrap_lock_t td_deque_lock; // Lock for accessing deque
2861	kmp_taskdata_t *
2862	*td_deque; // Deque of tasks encountered by td_thr, dynamically allocated
2863	kmp_int32 td_deque_size; // Size of deck
2864	kmp_uint32 td_deque_head; // Head of deque (will wrap)
2865	kmp_uint32 td_deque_tail; // Tail of deque (will wrap)
2866	kmp_int32 td_deque_ntasks; // Number of tasks in deque
2867	// GEH: shouldn't this be volatile since used in while-spin?
2868	kmp_int32 td_deque_last_stolen; // Thread number of last successful steal
2869	#ifdef BUILD_TIED_TASK_STACK
2870	kmp_task_stack_t td_susp_tied_tasks; // Stack of suspended tied tasks for task
2871	// scheduling constraint
2872	#endif // BUILD_TIED_TASK_STACK
2873	} kmp_base_thread_data_t;
2874
2875	#define TASK_DEQUE_BITS 8 // Used solely to define INITIAL_TASK_DEQUE_SIZE
2876	#define INITIAL_TASK_DEQUE_SIZE (1 << TASK_DEQUE_BITS)
2877
2878	#define TASK_DEQUE_SIZE(td) ((td).td_deque_size)
2879	#define TASK_DEQUE_MASK(td) ((td).td_deque_size - 1)
2880
2881	typedef union KMP_ALIGN_CACHE kmp_thread_data {
2882	kmp_base_thread_data_t td;
2883	double td_align; /* use worst case alignment */
2884	char td_pad[KMP_PAD(kmp_base_thread_data_t, CACHE_LINE)];
2885	} kmp_thread_data_t;
2886
2887	typedef struct kmp_task_pri {
2888	kmp_thread_data_t td;
2889	kmp_int32 priority;
2890	kmp_task_pri *next;
2891	} kmp_task_pri_t;
2892
2893	// Data for task teams which are used when tasking is enabled for the team
2894	typedef struct kmp_base_task_team {
2895	kmp_bootstrap_lock_t
2896	tt_threads_lock; /* Lock used to allocate per-thread part of task team */
2897	/* must be bootstrap lock since used at library shutdown*/
2898
2899	// TODO: check performance vs kmp_tas_lock_t
2900	kmp_bootstrap_lock_t tt_task_pri_lock; /* Lock to access priority tasks */
2901	kmp_task_pri_t *tt_task_pri_list;
2902
2903	kmp_task_team_t *tt_next; /* For linking the task team free list */
2904	kmp_thread_data_t
2905	*tt_threads_data; /* Array of per-thread structures for task team */
2906	/* Data survives task team deallocation */
2907	kmp_int32 tt_found_tasks; /* Have we found tasks and queued them while
2908	executing this team? */
2909	/* TRUE means tt_threads_data is set up and initialized */
2910	kmp_int32 tt_nproc; /* #threads in team */
2911	kmp_int32 tt_max_threads; // # entries allocated for threads_data array
2912	kmp_int32 tt_found_proxy_tasks; // found proxy tasks since last barrier
2913	kmp_int32 tt_untied_task_encountered;
2914	std::atomic<kmp_int32> tt_num_task_pri; // number of priority tasks enqueued
2915	// There is hidden helper thread encountered in this task team so that we must
2916	// wait when waiting on task team
2917	kmp_int32 tt_hidden_helper_task_encountered;
2918
2919	KMP_ALIGN_CACHE
2920	std::atomic<kmp_int32> tt_unfinished_threads; /* #threads still active */
2921
2922	KMP_ALIGN_CACHE
2923	volatile kmp_uint32
2924	tt_active; /* is the team still actively executing tasks */
2925	} kmp_base_task_team_t;
2926
2927	union KMP_ALIGN_CACHE kmp_task_team {
2928	kmp_base_task_team_t tt;
2929	double tt_align; /* use worst case alignment */
2930	char tt_pad[KMP_PAD(kmp_base_task_team_t, CACHE_LINE)];
2931	};
2932
2933	typedef struct kmp_task_team_list_t {
2934	kmp_task_team_t *task_team;
2935	kmp_task_team_list_t *next;
2936	} kmp_task_team_list_t;
2937
2938	#if (USE_FAST_MEMORY == 3) || (USE_FAST_MEMORY == 5)
2939	// Free lists keep same-size free memory slots for fast memory allocation
2940	// routines
2941	typedef struct kmp_free_list {
2942	void *th_free_list_self; // Self-allocated tasks free list
2943	void *th_free_list_sync; // Self-allocated tasks stolen/returned by other
2944	// threads
2945	void *th_free_list_other; // Non-self free list (to be returned to owner's
2946	// sync list)
2947	} kmp_free_list_t;
2948	#endif
2949	#if KMP_NESTED_HOT_TEAMS
2950	// Hot teams array keeps hot teams and their sizes for given thread. Hot teams
2951	// are not put in teams pool, and they don't put threads in threads pool.
2952	typedef struct kmp_hot_team_ptr {
2953	kmp_team_p *hot_team; // pointer to hot_team of given nesting level
2954	kmp_int32 hot_team_nth; // number of threads allocated for the hot_team
2955	} kmp_hot_team_ptr_t;
2956	#endif
2957	typedef struct kmp_teams_size {
2958	kmp_int32 nteams; // number of teams in a league
2959	kmp_int32 nth; // number of threads in each team of the league
2960	} kmp_teams_size_t;
2961
2962	// This struct stores a thread that acts as a "root" for a contention
2963	// group. Contention groups are rooted at kmp_root threads, but also at
2964	// each primary thread of each team created in the teams construct.
2965	// This struct therefore also stores a thread_limit associated with
2966	// that contention group, and a counter to track the number of threads
2967	// active in that contention group. Each thread has a list of these: CG
2968	// root threads have an entry in their list in which cg_root refers to
2969	// the thread itself, whereas other workers in the CG will have a
2970	// single entry where cg_root is same as the entry containing their CG
2971	// root. When a thread encounters a teams construct, it will add a new
2972	// entry to the front of its list, because it now roots a new CG.
2973	typedef struct kmp_cg_root {
2974	kmp_info_p *cg_root; // "root" thread for a contention group
2975	// The CG root's limit comes from OMP_THREAD_LIMIT for root threads, or
2976	// thread_limit clause for teams primary threads
2977	kmp_int32 cg_thread_limit;
2978	kmp_int32 cg_nthreads; // Count of active threads in CG rooted at cg_root
2979	struct kmp_cg_root *up; // pointer to higher level CG root in list
2980	} kmp_cg_root_t;
2981
2982	// OpenMP thread data structures
2983
2984	typedef struct KMP_ALIGN_CACHE kmp_base_info {
2985	/* Start with the readonly data which is cache aligned and padded. This is
2986	written before the thread starts working by the primary thread. Uber
2987	masters may update themselves later. Usage does not consider serialized
2988	regions. */
2989	kmp_desc_t th_info;
2990	kmp_team_p *th_team; /* team we belong to */
2991	kmp_root_p *th_root; /* pointer to root of task hierarchy */
2992	kmp_info_p *th_next_pool; /* next available thread in the pool */
2993	kmp_disp_t *th_dispatch; /* thread's dispatch data */
2994	int th_in_pool; /* in thread pool (32 bits for TCR/TCW) */
2995
2996	/* The following are cached from the team info structure */
2997	/* TODO use these in more places as determined to be needed via profiling */
2998	int th_team_nproc; /* number of threads in a team */
2999	kmp_info_p *th_team_master; /* the team's primary thread */
3000	int th_team_serialized; /* team is serialized */
3001	microtask_t th_teams_microtask; /* save entry address for teams construct */
3002	int th_teams_level; /* save initial level of teams construct */
3003	/* it is 0 on device but may be any on host */
3004
3005	/* The blocktime info is copied from the team struct to the thread struct */
3006	/* at the start of a barrier, and the values stored in the team are used */
3007	/* at points in the code where the team struct is no longer guaranteed */
3008	/* to exist (from the POV of worker threads). */
3009	#if KMP_USE_MONITOR
3010	int th_team_bt_intervals;
3011	int th_team_bt_set;
3012	#else
3013	kmp_uint64 th_team_bt_intervals;
3014	#endif
3015
3016	#if KMP_AFFINITY_SUPPORTED
3017	kmp_affin_mask_t *th_affin_mask; /* thread's current affinity mask */
3018	kmp_affinity_ids_t th_topology_ids; /* thread's current topology ids */
3019	kmp_affinity_attrs_t th_topology_attrs; /* thread's current topology attrs */
3020	#endif
3021	omp_allocator_handle_t th_def_allocator; /* default allocator */
3022	/* The data set by the primary thread at reinit, then R/W by the worker */
3023	KMP_ALIGN_CACHE int
3024	th_set_nproc; /* if > 0, then only use this request for the next fork */
3025	int *th_set_nested_nth;
3026	bool th_nt_strict; // num_threads clause has strict modifier
3027	ident_t *th_nt_loc; // loc for strict modifier
3028	int th_nt_sev; // error severity for strict modifier
3029	const char *th_nt_msg; // error message for strict modifier
3030	int th_set_nested_nth_sz;
3031	#if KMP_NESTED_HOT_TEAMS
3032	kmp_hot_team_ptr_t *th_hot_teams; /* array of hot teams */
3033	#endif
3034	kmp_proc_bind_t
3035	th_set_proc_bind; /* if != proc_bind_default, use request for next fork */
3036	kmp_teams_size_t
3037	th_teams_size; /* number of teams/threads in teams construct */
3038	#if KMP_AFFINITY_SUPPORTED
3039	int th_current_place; /* place currently bound to */
3040	int th_new_place; /* place to bind to in par reg */
3041	int th_first_place; /* first place in partition */
3042	int th_last_place; /* last place in partition */
3043	#endif
3044	int th_prev_level; /* previous level for affinity format */
3045	int th_prev_num_threads; /* previous num_threads for affinity format */
3046	#if USE_ITT_BUILD
3047	kmp_uint64 th_bar_arrive_time; /* arrival to barrier timestamp */
3048	kmp_uint64 th_bar_min_time; /* minimum arrival time at the barrier */
3049	kmp_uint64 th_frame_time; /* frame timestamp */
3050	#endif /* USE_ITT_BUILD */
3051	kmp_local_t th_local;
3052	struct private_common *th_pri_head;
3053
3054	/* Now the data only used by the worker (after initial allocation) */
3055	/* TODO the first serial team should actually be stored in the info_t
3056	structure. this will help reduce initial allocation overhead */
3057	KMP_ALIGN_CACHE kmp_team_p
3058	*th_serial_team; /*serialized team held in reserve*/
3059
3060	#if OMPT_SUPPORT
3061	ompt_thread_info_t ompt_thread_info;
3062	#endif
3063
3064	/* The following are also read by the primary thread during reinit */
3065	struct common_table *th_pri_common;
3066
3067	volatile kmp_uint32 th_spin_here; /* thread-local location for spinning */
3068	/* while awaiting queuing lock acquire */
3069
3070	volatile void *th_sleep_loc; // this points at a kmp_flag<T>
3071	flag_type th_sleep_loc_type; // enum type of flag stored in th_sleep_loc
3072
3073	ident_t *th_ident;
3074	unsigned th_x; // Random number generator data
3075	unsigned th_a; // Random number generator data
3076
3077	/* Tasking-related data for the thread */
3078	kmp_task_team_t *th_task_team; // Task team struct
3079	kmp_taskdata_t *th_current_task; // Innermost Task being executed
3080	kmp_uint8 th_task_state; // alternating 0/1 for task team identification
3081	kmp_uint32 th_reap_state; // Non-zero indicates thread is not
3082	// tasking, thus safe to reap
3083
3084	/* More stuff for keeping track of active/sleeping threads (this part is
3085	written by the worker thread) */
3086	kmp_uint8 th_active_in_pool; // included in count of #active threads in pool
3087	int th_active; // ! sleeping; 32 bits for TCR/TCW
3088	std::atomic<kmp_uint32> th_used_in_team; // Flag indicating use in team
3089	// 0 = not used in team; 1 = used in team;
3090	// 2 = transitioning to not used in team; 3 = transitioning to used in team
3091	struct cons_header *th_cons; // used for consistency check
3092	#if KMP_USE_HIER_SCHED
3093	// used for hierarchical scheduling
3094	kmp_hier_private_bdata_t *th_hier_bar_data;
3095	#endif
3096
3097	/* Add the syncronizing data which is cache aligned and padded. */
3098	KMP_ALIGN_CACHE kmp_balign_t th_bar[bs_last_barrier];
3099
3100	KMP_ALIGN_CACHE volatile kmp_int32
3101	th_next_waiting; /* gtid+1 of next thread on lock wait queue, 0 if none */
3102
3103	#if (USE_FAST_MEMORY == 3) || (USE_FAST_MEMORY == 5)
3104	#define NUM_LISTS 4
3105	kmp_free_list_t th_free_lists[NUM_LISTS]; // Free lists for fast memory
3106	// allocation routines
3107	#endif
3108
3109	#if KMP_OS_WINDOWS
3110	kmp_win32_cond_t th_suspend_cv;
3111	kmp_win32_mutex_t th_suspend_mx;
3112	std::atomic<int> th_suspend_init;
3113	#endif
3114	#if KMP_OS_UNIX
3115	kmp_cond_align_t th_suspend_cv;
3116	kmp_mutex_align_t th_suspend_mx;
3117	std::atomic<int> th_suspend_init_count;
3118	#endif
3119
3120	#if USE_ITT_BUILD
3121	kmp_itt_mark_t th_itt_mark_single;
3122	// alignment ???
3123	#endif /* USE_ITT_BUILD */
3124	#if KMP_STATS_ENABLED
3125	kmp_stats_list *th_stats;
3126	#endif
3127	#if KMP_OS_UNIX
3128	std::atomic<bool> th_blocking;
3129	#endif
3130	kmp_cg_root_t *th_cg_roots; // list of cg_roots associated with this thread
3131	} kmp_base_info_t;
3132
3133	typedef union KMP_ALIGN_CACHE kmp_info {
3134	double th_align; /* use worst case alignment */
3135	char th_pad[KMP_PAD(kmp_base_info_t, CACHE_LINE)];
3136	kmp_base_info_t th;
3137	} kmp_info_t;
3138
3139	// OpenMP thread team data structures
3140
3141	typedef struct kmp_base_data {
3142	volatile kmp_uint32 t_value;
3143	} kmp_base_data_t;
3144
3145	typedef union KMP_ALIGN_CACHE kmp_sleep_team {
3146	double dt_align; /* use worst case alignment */
3147	char dt_pad[KMP_PAD(kmp_base_data_t, CACHE_LINE)];
3148	kmp_base_data_t dt;
3149	} kmp_sleep_team_t;
3150
3151	typedef union KMP_ALIGN_CACHE kmp_ordered_team {
3152	double dt_align; /* use worst case alignment */
3153	char dt_pad[KMP_PAD(kmp_base_data_t, CACHE_LINE)];
3154	kmp_base_data_t dt;
3155	} kmp_ordered_team_t;
3156
3157	typedef int (*launch_t)(int gtid);
3158
3159	/* Minimum number of ARGV entries to malloc if necessary */
3160	#define KMP_MIN_MALLOC_ARGV_ENTRIES 100
3161
3162	// Set up how many argv pointers will fit in cache lines containing
3163	// t_inline_argv. Historically, we have supported at least 96 bytes. Using a
3164	// larger value for more space between the primary write/worker read section and
3165	// read/write by all section seems to buy more performance on EPCC PARALLEL.
3166	#if KMP_ARCH_X86 || KMP_ARCH_X86_64
3167	#define KMP_INLINE_ARGV_BYTES \
3168	(4 * CACHE_LINE - \
3169	((3 * KMP_PTR_SKIP + 2 * sizeof(int) + 2 * sizeof(kmp_int8) + \
3170	sizeof(kmp_int16) + sizeof(kmp_uint32)) % \
3171	CACHE_LINE))
3172	#else
3173	#define KMP_INLINE_ARGV_BYTES \
3174	(2 * CACHE_LINE - ((3 * KMP_PTR_SKIP + 2 * sizeof(int)) % CACHE_LINE))
3175	#endif
3176	#define KMP_INLINE_ARGV_ENTRIES (int)(KMP_INLINE_ARGV_BYTES / KMP_PTR_SKIP)
3177
3178	typedef struct KMP_ALIGN_CACHE kmp_base_team {
3179	// Synchronization Data
3180	// ---------------------------------------------------------------------------
3181	KMP_ALIGN_CACHE kmp_ordered_team_t t_ordered;
3182	kmp_balign_team_t t_bar[bs_last_barrier];
3183	std::atomic<int> t_construct; // count of single directive encountered by team
3184	char pad[sizeof(kmp_lock_t)]; // padding to maintain performance on big iron
3185
3186	// [0] - parallel / [1] - worksharing task reduction data shared by taskgroups
3187	std::atomic<void *> t_tg_reduce_data[2]; // to support task modifier
3188	std::atomic<int> t_tg_fini_counter[2]; // sync end of task reductions
3189
3190	// Primary thread only
3191	// ---------------------------------------------------------------------------
3192	KMP_ALIGN_CACHE int t_master_tid; // tid of primary thread in parent team
3193	int t_master_this_cons; // "this_construct" single counter of primary thread
3194	// in parent team
3195	ident_t *t_ident; // if volatile, have to change too much other crud to
3196	// volatile too
3197	kmp_team_p *t_parent; // parent team
3198	kmp_team_p *t_next_pool; // next free team in the team pool
3199	kmp_disp_t *t_dispatch; // thread's dispatch data
3200	kmp_task_team_t *t_task_team[2]; // Task team struct; switch between 2
3201	kmp_proc_bind_t t_proc_bind; // bind type for par region
3202	int t_primary_task_state; // primary thread's task state saved
3203	#if USE_ITT_BUILD
3204	kmp_uint64 t_region_time; // region begin timestamp
3205	#endif /* USE_ITT_BUILD */
3206
3207	// Primary thread write, workers read
3208	// --------------------------------------------------------------------------
3209	KMP_ALIGN_CACHE void **t_argv;
3210	int t_argc;
3211	int t_nproc; // number of threads in team
3212	microtask_t t_pkfn;
3213	launch_t t_invoke; // procedure to launch the microtask
3214
3215	#if OMPT_SUPPORT
3216	ompt_team_info_t ompt_team_info;
3217	ompt_lw_taskteam_t *ompt_serialized_team_info;
3218	#endif
3219
3220	#if KMP_ARCH_X86 || KMP_ARCH_X86_64
3221	kmp_int8 t_fp_control_saved;
3222	kmp_int8 t_pad2b;
3223	kmp_int16 t_x87_fpu_control_word; // FP control regs
3224	kmp_uint32 t_mxcsr;
3225	#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
3226
3227	void *t_inline_argv[KMP_INLINE_ARGV_ENTRIES];
3228
3229	KMP_ALIGN_CACHE kmp_info_t **t_threads;
3230	kmp_taskdata_t
3231	*t_implicit_task_taskdata; // Taskdata for the thread's implicit task
3232	int t_level; // nested parallel level
3233
3234	KMP_ALIGN_CACHE int t_max_argc;
3235	int t_max_nproc; // max threads this team can handle (dynamically expandable)
3236	int t_serialized; // levels deep of serialized teams
3237	dispatch_shared_info_t *t_disp_buffer; // buffers for dispatch system
3238	int t_id; // team's id, assigned by debugger.
3239	int t_active_level; // nested active parallel level
3240	kmp_r_sched_t t_sched; // run-time schedule for the team
3241	#if KMP_AFFINITY_SUPPORTED
3242	int t_first_place; // first & last place in parent thread's partition.
3243	int t_last_place; // Restore these values to primary thread after par region.
3244	#endif // KMP_AFFINITY_SUPPORTED
3245	int t_display_affinity;
3246	int t_size_changed; // team size was changed?: 0: no, 1: yes, -1: changed via
3247	// omp_set_num_threads() call
3248	omp_allocator_handle_t t_def_allocator; /* default allocator */
3249
3250	// Read/write by workers as well
3251	#if (KMP_ARCH_X86 || KMP_ARCH_X86_64)
3252	// Using CACHE_LINE=64 reduces memory footprint, but causes a big perf
3253	// regression of epcc 'parallel' and 'barrier' on fxe256lin01. This extra
3254	// padding serves to fix the performance of epcc 'parallel' and 'barrier' when
3255	// CACHE_LINE=64. TODO: investigate more and get rid if this padding.
3256	char dummy_padding[1024];
3257	#endif
3258	// Internal control stack for additional nested teams.
3259	KMP_ALIGN_CACHE kmp_internal_control_t *t_control_stack_top;
3260	// for SERIALIZED teams nested 2 or more levels deep
3261	// typed flag to store request state of cancellation
3262	std::atomic<kmp_int32> t_cancel_request;
3263	int t_master_active; // save on fork, restore on join
3264	void *t_copypriv_data; // team specific pointer to copyprivate data array
3265	#if KMP_OS_WINDOWS
3266	std::atomic<kmp_uint32> t_copyin_counter;
3267	#endif
3268	#if USE_ITT_BUILD
3269	void *t_stack_id; // team specific stack stitching id (for ittnotify)
3270	#endif /* USE_ITT_BUILD */
3271	distributedBarrier *b; // Distributed barrier data associated with team
3272	kmp_nested_nthreads_t *t_nested_nth;
3273	} kmp_base_team_t;
3274
3275	// Assert that the list structure fits and aligns within
3276	// the double task team pointer
3277	KMP_BUILD_ASSERT(sizeof(kmp_task_team_t *[2]) == sizeof(kmp_task_team_list_t));
3278	KMP_BUILD_ASSERT(alignof(kmp_task_team_t *[2]) ==
3279	alignof(kmp_task_team_list_t));
3280
3281	union KMP_ALIGN_CACHE kmp_team {
3282	kmp_base_team_t t;
3283	double t_align; /* use worst case alignment */
3284	char t_pad[KMP_PAD(kmp_base_team_t, CACHE_LINE)];
3285	};
3286
3287	typedef union KMP_ALIGN_CACHE kmp_time_global {
3288	double dt_align; /* use worst case alignment */
3289	char dt_pad[KMP_PAD(kmp_base_data_t, CACHE_LINE)];
3290	kmp_base_data_t dt;
3291	} kmp_time_global_t;
3292
3293	typedef struct kmp_base_global {
3294	/* cache-aligned */
3295	kmp_time_global_t g_time;
3296
3297	/* non cache-aligned */
3298	volatile int g_abort;
3299	volatile int g_done;
3300
3301	int g_dynamic;
3302	enum dynamic_mode g_dynamic_mode;
3303	} kmp_base_global_t;
3304
3305	typedef union KMP_ALIGN_CACHE kmp_global {
3306	kmp_base_global_t g;
3307	double g_align; /* use worst case alignment */
3308	char g_pad[KMP_PAD(kmp_base_global_t, CACHE_LINE)];
3309	} kmp_global_t;
3310
3311	typedef struct kmp_base_root {
3312	// TODO: GEH - combine r_active with r_in_parallel then r_active ==
3313	// (r_in_parallel>= 0)
3314	// TODO: GEH - then replace r_active with t_active_levels if we can to reduce
3315	// the synch overhead or keeping r_active
3316	volatile int r_active; /* TRUE if some region in a nest has > 1 thread */
3317	// keeps a count of active parallel regions per root
3318	std::atomic<int> r_in_parallel;
3319	// GEH: This is misnamed, should be r_active_levels
3320	kmp_team_t *r_root_team;
3321	kmp_team_t *r_hot_team;
3322	kmp_info_t *r_uber_thread;
3323	kmp_lock_t r_begin_lock;
3324	volatile int r_begin;
3325	int r_blocktime; /* blocktime for this root and descendants */
3326	#if KMP_AFFINITY_SUPPORTED
3327	int r_affinity_assigned;
3328	#endif // KMP_AFFINITY_SUPPORTED
3329	} kmp_base_root_t;
3330
3331	typedef union KMP_ALIGN_CACHE kmp_root {
3332	kmp_base_root_t r;
3333	double r_align; /* use worst case alignment */
3334	char r_pad[KMP_PAD(kmp_base_root_t, CACHE_LINE)];
3335	} kmp_root_t;
3336
3337	struct fortran_inx_info {
3338	kmp_int32 data;
3339	};
3340
3341	// This list type exists to hold old __kmp_threads arrays so that
3342	// old references to them may complete while reallocation takes place when
3343	// expanding the array. The items in this list are kept alive until library
3344	// shutdown.
3345	typedef struct kmp_old_threads_list_t {
3346	kmp_info_t **threads;
3347	struct kmp_old_threads_list_t *next;
3348	} kmp_old_threads_list_t;
3349
3350	/* ------------------------------------------------------------------------ */
3351
3352	extern int __kmp_settings;
3353	extern int __kmp_duplicate_library_ok;
3354	#if USE_ITT_BUILD
3355	extern int __kmp_forkjoin_frames;
3356	extern int __kmp_forkjoin_frames_mode;
3357	#endif
3358	extern PACKED_REDUCTION_METHOD_T __kmp_force_reduction_method;
3359	extern int __kmp_determ_red;
3360
3361	#ifdef KMP_DEBUG
3362	extern int kmp_a_debug;
3363	extern int kmp_b_debug;
3364	extern int kmp_c_debug;
3365	extern int kmp_d_debug;
3366	extern int kmp_e_debug;
3367	extern int kmp_f_debug;
3368	#endif /* KMP_DEBUG */
3369
3370	/* For debug information logging using rotating buffer */
3371	#define KMP_DEBUG_BUF_LINES_INIT 512
3372	#define KMP_DEBUG_BUF_LINES_MIN 1
3373
3374	#define KMP_DEBUG_BUF_CHARS_INIT 128
3375	#define KMP_DEBUG_BUF_CHARS_MIN 2
3376
3377	extern int
3378	__kmp_debug_buf; /* TRUE means use buffer, FALSE means print to stderr */
3379	extern int __kmp_debug_buf_lines; /* How many lines of debug stored in buffer */
3380	extern int
3381	__kmp_debug_buf_chars; /* How many characters allowed per line in buffer */
3382	extern int __kmp_debug_buf_atomic; /* TRUE means use atomic update of buffer
3383	entry pointer */
3384
3385	extern char *__kmp_debug_buffer; /* Debug buffer itself */
3386	extern std::atomic<int> __kmp_debug_count; /* Counter for number of lines
3387	printed in buffer so far */
3388	extern int __kmp_debug_buf_warn_chars; /* Keep track of char increase
3389	recommended in warnings */
3390	/* end rotating debug buffer */
3391
3392	#ifdef KMP_DEBUG
3393	extern int __kmp_par_range; /* +1 => only go par for constructs in range */
3394
3395	#define KMP_PAR_RANGE_ROUTINE_LEN 1024
3396	extern char __kmp_par_range_routine[KMP_PAR_RANGE_ROUTINE_LEN];
3397	#define KMP_PAR_RANGE_FILENAME_LEN 1024
3398	extern char __kmp_par_range_filename[KMP_PAR_RANGE_FILENAME_LEN];
3399	extern int __kmp_par_range_lb;
3400	extern int __kmp_par_range_ub;
3401	#endif
3402
3403	/* For printing out dynamic storage map for threads and teams */
3404	extern int
3405	__kmp_storage_map; /* True means print storage map for threads and teams */
3406	extern int __kmp_storage_map_verbose; /* True means storage map includes
3407	placement info */
3408	extern int __kmp_storage_map_verbose_specified;
3409
3410	#if KMP_ARCH_X86 || KMP_ARCH_X86_64
3411	extern kmp_cpuinfo_t __kmp_cpuinfo;
3412	static inline bool __kmp_is_hybrid_cpu() { return __kmp_cpuinfo.flags.hybrid; }
3413	#elif KMP_OS_DARWIN && KMP_ARCH_AARCH64
3414	static inline bool __kmp_is_hybrid_cpu() { return true; }
3415	#else
3416	static inline bool __kmp_is_hybrid_cpu() { return false; }
3417	#endif
3418
3419	extern volatile int __kmp_init_serial;
3420	extern volatile int __kmp_init_gtid;
3421	extern volatile int __kmp_init_common;
3422	extern volatile int __kmp_need_register_serial;
3423	extern volatile int __kmp_init_middle;
3424	extern volatile int __kmp_init_parallel;
3425	#if KMP_USE_MONITOR
3426	extern volatile int __kmp_init_monitor;
3427	#endif
3428	extern volatile int __kmp_init_user_locks;
3429	extern volatile int __kmp_init_hidden_helper_threads;
3430	extern int __kmp_init_counter;
3431	extern int __kmp_root_counter;
3432	extern int __kmp_version;
3433
3434	/* list of address of allocated caches for commons */
3435	extern kmp_cached_addr_t *__kmp_threadpriv_cache_list;
3436
3437	/* Barrier algorithm types and options */
3438	extern kmp_uint32 __kmp_barrier_gather_bb_dflt;
3439	extern kmp_uint32 __kmp_barrier_release_bb_dflt;
3440	extern kmp_bar_pat_e __kmp_barrier_gather_pat_dflt;
3441	extern kmp_bar_pat_e __kmp_barrier_release_pat_dflt;
3442	extern kmp_uint32 __kmp_barrier_gather_branch_bits[bs_last_barrier];
3443	extern kmp_uint32 __kmp_barrier_release_branch_bits[bs_last_barrier];
3444	extern kmp_bar_pat_e __kmp_barrier_gather_pattern[bs_last_barrier];
3445	extern kmp_bar_pat_e __kmp_barrier_release_pattern[bs_last_barrier];
3446	extern char const *__kmp_barrier_branch_bit_env_name[bs_last_barrier];
3447	extern char const *__kmp_barrier_pattern_env_name[bs_last_barrier];
3448	extern char const *__kmp_barrier_type_name[bs_last_barrier];
3449	extern char const *__kmp_barrier_pattern_name[bp_last_bar];
3450
3451	/* Global Locks */
3452	extern kmp_bootstrap_lock_t __kmp_initz_lock; /* control initialization */
3453	extern kmp_bootstrap_lock_t __kmp_forkjoin_lock; /* control fork/join access */
3454	extern kmp_bootstrap_lock_t __kmp_task_team_lock;
3455	extern kmp_bootstrap_lock_t
3456	__kmp_exit_lock; /* exit() is not always thread-safe */
3457	#if KMP_USE_MONITOR
3458	extern kmp_bootstrap_lock_t
3459	__kmp_monitor_lock; /* control monitor thread creation */
3460	#endif
3461	extern kmp_bootstrap_lock_t
3462	__kmp_tp_cached_lock; /* used for the hack to allow threadprivate cache and
3463	__kmp_threads expansion to co-exist */
3464
3465	extern kmp_lock_t __kmp_global_lock; /* control OS/global access */
3466
3467	extern enum library_type __kmp_library;
3468
3469	extern enum sched_type __kmp_sched; /* default runtime scheduling */
3470	extern enum sched_type __kmp_static; /* default static scheduling method */
3471	extern enum sched_type __kmp_guided; /* default guided scheduling method */
3472	extern enum sched_type __kmp_auto; /* default auto scheduling method */
3473	extern int __kmp_chunk; /* default runtime chunk size */
3474	extern int __kmp_force_monotonic; /* whether monotonic scheduling forced */
3475
3476	extern size_t __kmp_stksize; /* stack size per thread */
3477	#if KMP_USE_MONITOR
3478	extern size_t __kmp_monitor_stksize; /* stack size for monitor thread */
3479	#endif
3480	extern size_t __kmp_stkoffset; /* stack offset per thread */
3481	extern int __kmp_stkpadding; /* Should we pad root thread(s) stack */
3482
3483	extern size_t
3484	__kmp_malloc_pool_incr; /* incremental size of pool for kmp_malloc() */
3485	extern int __kmp_env_stksize; /* was KMP_STACKSIZE specified? */
3486	extern int __kmp_env_blocktime; /* was KMP_BLOCKTIME specified? */
3487	extern int __kmp_env_checks; /* was KMP_CHECKS specified? */
3488	extern int __kmp_env_consistency_check; // was KMP_CONSISTENCY_CHECK specified?
3489	extern int __kmp_generate_warnings; /* should we issue warnings? */
3490	extern int __kmp_reserve_warn; /* have we issued reserve_threads warning? */
3491
3492	#ifdef DEBUG_SUSPEND
3493	extern int __kmp_suspend_count; /* count inside __kmp_suspend_template() */
3494	#endif
3495
3496	extern kmp_int32 __kmp_use_yield;
3497	extern kmp_int32 __kmp_use_yield_exp_set;
3498	extern kmp_uint32 __kmp_yield_init;
3499	extern kmp_uint32 __kmp_yield_next;
3500	extern kmp_uint64 __kmp_pause_init;
3501
3502	/* ------------------------------------------------------------------------- */
3503	extern int __kmp_allThreadsSpecified;
3504
3505	extern size_t __kmp_align_alloc;
3506	/* following data protected by initialization routines */
3507	extern int __kmp_xproc; /* number of processors in the system */
3508	extern int __kmp_avail_proc; /* number of processors available to the process */
3509	extern size_t __kmp_sys_min_stksize; /* system-defined minimum stack size */
3510	extern int __kmp_sys_max_nth; /* system-imposed maximum number of threads */
3511	// maximum total number of concurrently-existing threads on device
3512	extern int __kmp_max_nth;
3513	// maximum total number of concurrently-existing threads in a contention group
3514	extern int __kmp_cg_max_nth;
3515	extern int __kmp_task_max_nth; // max threads used in a task
3516	extern int __kmp_teams_max_nth; // max threads used in a teams construct
3517	extern int __kmp_threads_capacity; /* capacity of the arrays __kmp_threads and
3518	__kmp_root */
3519	extern int __kmp_dflt_team_nth; /* default number of threads in a parallel
3520	region a la OMP_NUM_THREADS */
3521	extern int __kmp_dflt_team_nth_ub; /* upper bound on "" determined at serial
3522	initialization */
3523	extern int __kmp_tp_capacity; /* capacity of __kmp_threads if threadprivate is
3524	used (fixed) */
3525	extern int __kmp_tp_cached; /* whether threadprivate cache has been created
3526	(__kmpc_threadprivate_cached()) */
3527	extern int __kmp_dflt_blocktime; /* number of microseconds to wait before
3528	blocking (env setting) */
3529	extern char __kmp_blocktime_units; /* 'm' or 'u' to note units specified */
3530	extern bool __kmp_wpolicy_passive; /* explicitly set passive wait policy */
3531
3532	// Convert raw blocktime from ms to us if needed.
3533	static inline void __kmp_aux_convert_blocktime(int *bt) {
3534	if (__kmp_blocktime_units == 'm') {
3535	if (*bt > INT_MAX / 1000) {
3536	*bt = INT_MAX / 1000;
3537	KMP_INFORM(MaxValueUsing, "kmp_set_blocktime(ms)", bt);
3538	}
3539	*bt = *bt * 1000;
3540	}
3541	}
3542
3543	#if KMP_USE_MONITOR
3544	extern int
3545	__kmp_monitor_wakeups; /* number of times monitor wakes up per second */
3546	extern int __kmp_bt_intervals; /* number of monitor timestamp intervals before
3547	blocking */
3548	#endif
3549	#ifdef KMP_ADJUST_BLOCKTIME
3550	extern int __kmp_zero_bt; /* whether blocktime has been forced to zero */
3551	#endif /* KMP_ADJUST_BLOCKTIME */
3552	#ifdef KMP_DFLT_NTH_CORES
3553	extern int __kmp_ncores; /* Total number of cores for threads placement */
3554	#endif
3555	/* Number of millisecs to delay on abort for Intel(R) VTune(TM) tools */
3556	extern int __kmp_abort_delay;
3557
3558	extern int __kmp_need_register_atfork_specified;
3559	extern int __kmp_need_register_atfork; /* At initialization, call pthread_atfork
3560	to install fork handler */
3561	extern int __kmp_gtid_mode; /* Method of getting gtid, values:
3562	0 - not set, will be set at runtime
3563	1 - using stack search
3564	2 - dynamic TLS (pthread_getspecific(Linux* OS/OS
3565	X*) or TlsGetValue(Windows* OS))
3566	3 - static TLS (__declspec(thread) __kmp_gtid),
3567	Linux* OS .so only. */
3568	extern int
3569	__kmp_adjust_gtid_mode; /* If true, adjust method based on #threads */
3570	#ifdef KMP_TDATA_GTID
3571	extern KMP_THREAD_LOCAL int __kmp_gtid;
3572	#endif
3573	extern int __kmp_tls_gtid_min; /* #threads below which use sp search for gtid */
3574	extern int __kmp_foreign_tp; // If true, separate TP var for each foreign thread
3575	#if KMP_ARCH_X86 || KMP_ARCH_X86_64
3576	extern int __kmp_inherit_fp_control; // copy fp creg(s) parent->workers at fork
3577	extern kmp_int16 __kmp_init_x87_fpu_control_word; // init thread's FP ctrl reg
3578	extern kmp_uint32 __kmp_init_mxcsr; /* init thread's mxscr */
3579	#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
3580
3581	// max_active_levels for nested parallelism enabled by default via
3582	// OMP_MAX_ACTIVE_LEVELS, OMP_NESTED, OMP_NUM_THREADS, and OMP_PROC_BIND
3583	extern int __kmp_dflt_max_active_levels;
3584	// Indicates whether value of __kmp_dflt_max_active_levels was already
3585	// explicitly set by OMP_MAX_ACTIVE_LEVELS or OMP_NESTED=false
3586	extern bool __kmp_dflt_max_active_levels_set;
3587	extern int __kmp_dispatch_num_buffers; /* max possible dynamic loops in
3588	concurrent execution per team */
3589	#if KMP_NESTED_HOT_TEAMS
3590	extern int __kmp_hot_teams_mode;
3591	extern int __kmp_hot_teams_max_level;
3592	#endif
3593
3594	#if KMP_MIC_SUPPORTED
3595	extern enum mic_type __kmp_mic_type;
3596	#endif
3597
3598	#ifdef USE_LOAD_BALANCE
3599	extern double __kmp_load_balance_interval; // load balance algorithm interval
3600	#endif /* USE_LOAD_BALANCE */
3601
3602	#if KMP_USE_ADAPTIVE_LOCKS
3603
3604	// Parameters for the speculative lock backoff system.
3605	struct kmp_adaptive_backoff_params_t {
3606	// Number of soft retries before it counts as a hard retry.
3607	kmp_uint32 max_soft_retries;
3608	// Badness is a bit mask : 0,1,3,7,15,... on each hard failure we move one to
3609	// the right
3610	kmp_uint32 max_badness;
3611	};
3612
3613	extern kmp_adaptive_backoff_params_t __kmp_adaptive_backoff_params;
3614
3615	#if KMP_DEBUG_ADAPTIVE_LOCKS
3616	extern const char *__kmp_speculative_statsfile;
3617	#endif
3618
3619	#endif // KMP_USE_ADAPTIVE_LOCKS
3620
3621	extern int __kmp_display_env; /* TRUE or FALSE */
3622	extern int __kmp_display_env_verbose; /* TRUE if OMP_DISPLAY_ENV=VERBOSE */
3623	extern int __kmp_omp_cancellation; /* TRUE or FALSE */
3624	extern int __kmp_nteams;
3625	extern int __kmp_teams_thread_limit;
3626
3627	/* ------------------------------------------------------------------------- */
3628
3629	/* the following are protected by the fork/join lock */
3630	/* write: lock read: anytime */
3631	extern kmp_info_t **__kmp_threads; /* Descriptors for the threads */
3632	/* Holds old arrays of __kmp_threads until library shutdown */
3633	extern kmp_old_threads_list_t *__kmp_old_threads_list;
3634	/* read/write: lock */
3635	extern volatile kmp_team_t *__kmp_team_pool;
3636	extern volatile kmp_info_t *__kmp_thread_pool;
3637	extern kmp_info_t *__kmp_thread_pool_insert_pt;
3638
3639	// total num threads reachable from some root thread including all root threads
3640	extern volatile int __kmp_nth;
3641	/* total number of threads reachable from some root thread including all root
3642	threads, and those in the thread pool */
3643	extern volatile int __kmp_all_nth;
3644	extern std::atomic<int> __kmp_thread_pool_active_nth;
3645
3646	extern kmp_root_t **__kmp_root; /* root of thread hierarchy */
3647	/* end data protected by fork/join lock */
3648	/* ------------------------------------------------------------------------- */
3649
3650	#define __kmp_get_gtid() __kmp_get_global_thread_id()
3651	#define __kmp_entry_gtid() __kmp_get_global_thread_id_reg()
3652	#define __kmp_get_tid() (__kmp_tid_from_gtid(__kmp_get_gtid()))
3653	#define __kmp_get_team() (__kmp_threads[(__kmp_get_gtid())]->th.th_team)
3654	#define __kmp_get_thread() (__kmp_thread_from_gtid(__kmp_get_gtid()))
3655
3656	// AT: Which way is correct?
3657	// AT: 1. nproc = __kmp_threads[ ( gtid ) ] -> th.th_team -> t.t_nproc;
3658	// AT: 2. nproc = __kmp_threads[ ( gtid ) ] -> th.th_team_nproc;
3659	#define __kmp_get_team_num_threads(gtid) \
3660	(__kmp_threads[(gtid)]->th.th_team->t.t_nproc)
3661
3662	static inline bool KMP_UBER_GTID(int gtid) {
3663	KMP_DEBUG_ASSERT(gtid >= KMP_GTID_MIN);
3664	KMP_DEBUG_ASSERT(gtid < __kmp_threads_capacity);
3665	return (gtid >= 0 && __kmp_root[gtid] && __kmp_threads[gtid] &&
3666	__kmp_threads[gtid] == __kmp_root[gtid]->r.r_uber_thread);
3667	}
3668
3669	static inline int __kmp_tid_from_gtid(int gtid) {
3670	KMP_DEBUG_ASSERT(gtid >= 0);
3671	return __kmp_threads[gtid]->th.th_info.ds.ds_tid;
3672	}
3673
3674	static inline int __kmp_gtid_from_tid(int tid, const kmp_team_t *team) {
3675	KMP_DEBUG_ASSERT(tid >= 0 && team);
3676	return team->t.t_threads[tid]->th.th_info.ds.ds_gtid;
3677	}
3678
3679	static inline int __kmp_gtid_from_thread(const kmp_info_t *thr) {
3680	KMP_DEBUG_ASSERT(thr);
3681	return thr->th.th_info.ds.ds_gtid;
3682	}
3683
3684	static inline kmp_info_t *__kmp_thread_from_gtid(int gtid) {
3685	KMP_DEBUG_ASSERT(gtid >= 0);
3686	return __kmp_threads[gtid];
3687	}
3688
3689	static inline kmp_team_t *__kmp_team_from_gtid(int gtid) {
3690	KMP_DEBUG_ASSERT(gtid >= 0);
3691	return __kmp_threads[gtid]->th.th_team;
3692	}
3693
3694	static inline void __kmp_assert_valid_gtid(kmp_int32 gtid) {
3695	if (UNLIKELY(gtid < 0 || gtid >= __kmp_threads_capacity))
3696	KMP_FATAL(ThreadIdentInvalid);
3697	}
3698
3699	#if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT
3700	extern int __kmp_user_level_mwait; // TRUE or FALSE; from KMP_USER_LEVEL_MWAIT
3701	extern int __kmp_umwait_enabled; // Runtime check if user-level mwait enabled
3702	extern int __kmp_mwait_enabled; // Runtime check if ring3 mwait is enabled
3703	extern int __kmp_mwait_hints; // Hints to pass in to mwait
3704	#endif
3705
3706	#if KMP_HAVE_UMWAIT
3707	extern int __kmp_waitpkg_enabled; // Runtime check if waitpkg exists
3708	extern int __kmp_tpause_state; // 0 (default), 1=C0.1, 2=C0.2; from KMP_TPAUSE
3709	extern int __kmp_tpause_hint; // 1=C0.1 (default), 0=C0.2; from KMP_TPAUSE
3710	extern int __kmp_tpause_enabled; // 0 (default), 1 (KMP_TPAUSE is non-zero)
3711	#endif
3712
3713	/* ------------------------------------------------------------------------- */
3714
3715	extern kmp_global_t __kmp_global; /* global status */
3716
3717	extern kmp_info_t __kmp_monitor;
3718	// For Debugging Support Library
3719	extern std::atomic<kmp_int32> __kmp_team_counter;
3720	// For Debugging Support Library
3721	extern std::atomic<kmp_int32> __kmp_task_counter;
3722
3723	#if USE_DEBUGGER
3724	#define _KMP_GEN_ID(counter) \
3725	(__kmp_debugging ? KMP_ATOMIC_INC(&counter) + 1 : ~0)
3726	#else
3727	#define _KMP_GEN_ID(counter) (~0)
3728	#endif /* USE_DEBUGGER */
3729
3730	#define KMP_GEN_TASK_ID() _KMP_GEN_ID(__kmp_task_counter)
3731	#define KMP_GEN_TEAM_ID() _KMP_GEN_ID(__kmp_team_counter)
3732
3733	/* ------------------------------------------------------------------------ */
3734
3735	extern void __kmp_print_storage_map_gtid(int gtid, void *p1, void *p2,
3736	size_t size, char const *format, ...);
3737
3738	extern void __kmp_serial_initialize(void);
3739	extern void __kmp_middle_initialize(void);
3740	extern void __kmp_parallel_initialize(void);
3741
3742	extern void __kmp_internal_begin(void);
3743	extern void __kmp_internal_end_library(int gtid);
3744	extern void __kmp_internal_end_thread(int gtid);
3745	extern void __kmp_internal_end_atexit(void);
3746	extern void __kmp_internal_end_dtor(void);
3747	extern void __kmp_internal_end_dest(void *);
3748
3749	extern int __kmp_register_root(int initial_thread);
3750	extern void __kmp_unregister_root(int gtid);
3751	extern void __kmp_unregister_library(void); // called by __kmp_internal_end()
3752
3753	extern int __kmp_ignore_mppbeg(void);
3754	extern int __kmp_ignore_mppend(void);
3755
3756	extern int __kmp_enter_single(int gtid, ident_t *id_ref, int push_ws);
3757	extern void __kmp_exit_single(int gtid);
3758
3759	extern void __kmp_parallel_deo(int *gtid_ref, int *cid_ref, ident_t *loc_ref);
3760	extern void __kmp_parallel_dxo(int *gtid_ref, int *cid_ref, ident_t *loc_ref);
3761
3762	#ifdef USE_LOAD_BALANCE
3763	extern int __kmp_get_load_balance(int);
3764	#endif
3765
3766	extern int __kmp_get_global_thread_id(void);
3767	extern int __kmp_get_global_thread_id_reg(void);
3768	extern void __kmp_exit_thread(int exit_status);
3769	extern void __kmp_abort(char const *format, ...);
3770	extern void __kmp_abort_thread(void);
3771	KMP_NORETURN extern void __kmp_abort_process(void);
3772	extern void __kmp_warn(char const *format, ...);
3773
3774	extern void __kmp_set_num_threads(int new_nth, int gtid);
3775
3776	extern bool __kmp_detect_shm();
3777	extern bool __kmp_detect_tmp();
3778
3779	// Returns current thread (pointer to kmp_info_t). Current thread *must* be
3780	// registered.
3781	static inline kmp_info_t *__kmp_entry_thread() {
3782	int gtid = __kmp_entry_gtid();
3783
3784	return __kmp_threads[gtid];
3785	}
3786
3787	extern void __kmp_set_max_active_levels(int gtid, int new_max_active_levels);
3788	extern int __kmp_get_max_active_levels(int gtid);
3789	extern int __kmp_get_ancestor_thread_num(int gtid, int level);
3790	extern int __kmp_get_team_size(int gtid, int level);
3791	extern void __kmp_set_schedule(int gtid, kmp_sched_t new_sched, int chunk);
3792	extern void __kmp_get_schedule(int gtid, kmp_sched_t *sched, int *chunk);
3793
3794	extern unsigned short __kmp_get_random(kmp_info_t *thread);
3795	extern void __kmp_init_random(kmp_info_t *thread);
3796
3797	extern kmp_r_sched_t __kmp_get_schedule_global(void);
3798	extern void __kmp_adjust_num_threads(int new_nproc);
3799	extern void __kmp_check_stksize(size_t *val);
3800
3801	extern void *___kmp_allocate(size_t size KMP_SRC_LOC_DECL);
3802	extern void *___kmp_page_allocate(size_t size KMP_SRC_LOC_DECL);
3803	extern void ___kmp_free(void *ptr KMP_SRC_LOC_DECL);
3804	#define __kmp_allocate(size) ___kmp_allocate((size)KMP_SRC_LOC_CURR)
3805	#define __kmp_page_allocate(size) ___kmp_page_allocate((size)KMP_SRC_LOC_CURR)
3806	#define __kmp_free(ptr) ___kmp_free((ptr)KMP_SRC_LOC_CURR)
3807
3808	#if USE_FAST_MEMORY
3809	extern void *___kmp_fast_allocate(kmp_info_t *this_thr,
3810	size_t size KMP_SRC_LOC_DECL);
3811	extern void ___kmp_fast_free(kmp_info_t *this_thr, void *ptr KMP_SRC_LOC_DECL);
3812	extern void __kmp_free_fast_memory(kmp_info_t *this_thr);
3813	extern void __kmp_initialize_fast_memory(kmp_info_t *this_thr);
3814	#define __kmp_fast_allocate(this_thr, size) \
3815	___kmp_fast_allocate((this_thr), (size)KMP_SRC_LOC_CURR)
3816	#define __kmp_fast_free(this_thr, ptr) \
3817	___kmp_fast_free((this_thr), (ptr)KMP_SRC_LOC_CURR)
3818	#endif
3819
3820	extern void *___kmp_thread_malloc(kmp_info_t *th, size_t size KMP_SRC_LOC_DECL);
3821	extern void *___kmp_thread_calloc(kmp_info_t *th, size_t nelem,
3822	size_t elsize KMP_SRC_LOC_DECL);
3823	extern void *___kmp_thread_realloc(kmp_info_t *th, void *ptr,
3824	size_t size KMP_SRC_LOC_DECL);
3825	extern void ___kmp_thread_free(kmp_info_t *th, void *ptr KMP_SRC_LOC_DECL);
3826	#define __kmp_thread_malloc(th, size) \
3827	___kmp_thread_malloc((th), (size)KMP_SRC_LOC_CURR)
3828	#define __kmp_thread_calloc(th, nelem, elsize) \
3829	___kmp_thread_calloc((th), (nelem), (elsize)KMP_SRC_LOC_CURR)
3830	#define __kmp_thread_realloc(th, ptr, size) \
3831	___kmp_thread_realloc((th), (ptr), (size)KMP_SRC_LOC_CURR)
3832	#define __kmp_thread_free(th, ptr) \
3833	___kmp_thread_free((th), (ptr)KMP_SRC_LOC_CURR)
3834
3835	extern void __kmp_push_num_threads(ident_t *loc, int gtid, int num_threads);
3836	extern void __kmp_push_num_threads_list(ident_t *loc, int gtid,
3837	kmp_uint32 list_length,
3838	int *num_threads_list);
3839	extern void __kmp_set_strict_num_threads(ident_t *loc, int gtid, int sev,
3840	const char *msg);
3841
3842	extern void __kmp_push_proc_bind(ident_t *loc, int gtid,
3843	kmp_proc_bind_t proc_bind);
3844	extern void __kmp_push_num_teams(ident_t *loc, int gtid, int num_teams,
3845	int num_threads);
3846	extern void __kmp_push_num_teams_51(ident_t *loc, int gtid, int num_teams_lb,
3847	int num_teams_ub, int num_threads);
3848
3849	extern void __kmp_yield();
3850
3851	extern void __kmpc_dispatch_init_4(ident_t *loc, kmp_int32 gtid,
3852	enum sched_type schedule, kmp_int32 lb,
3853	kmp_int32 ub, kmp_int32 st, kmp_int32 chunk);
3854	extern void __kmpc_dispatch_init_4u(ident_t *loc, kmp_int32 gtid,
3855	enum sched_type schedule, kmp_uint32 lb,
3856	kmp_uint32 ub, kmp_int32 st,
3857	kmp_int32 chunk);
3858	extern void __kmpc_dispatch_init_8(ident_t *loc, kmp_int32 gtid,
3859	enum sched_type schedule, kmp_int64 lb,
3860	kmp_int64 ub, kmp_int64 st, kmp_int64 chunk);
3861	extern void __kmpc_dispatch_init_8u(ident_t *loc, kmp_int32 gtid,
3862	enum sched_type schedule, kmp_uint64 lb,
3863	kmp_uint64 ub, kmp_int64 st,
3864	kmp_int64 chunk);
3865
3866	extern int __kmpc_dispatch_next_4(ident_t *loc, kmp_int32 gtid,
3867	kmp_int32 *p_last, kmp_int32 *p_lb,
3868	kmp_int32 *p_ub, kmp_int32 *p_st);
3869	extern int __kmpc_dispatch_next_4u(ident_t *loc, kmp_int32 gtid,
3870	kmp_int32 *p_last, kmp_uint32 *p_lb,
3871	kmp_uint32 *p_ub, kmp_int32 *p_st);
3872	extern int __kmpc_dispatch_next_8(ident_t *loc, kmp_int32 gtid,
3873	kmp_int32 *p_last, kmp_int64 *p_lb,
3874	kmp_int64 *p_ub, kmp_int64 *p_st);
3875	extern int __kmpc_dispatch_next_8u(ident_t *loc, kmp_int32 gtid,
3876	kmp_int32 *p_last, kmp_uint64 *p_lb,
3877	kmp_uint64 *p_ub, kmp_int64 *p_st);
3878
3879	extern void __kmpc_dispatch_fini_4(ident_t *loc, kmp_int32 gtid);
3880	extern void __kmpc_dispatch_fini_8(ident_t *loc, kmp_int32 gtid);
3881	extern void __kmpc_dispatch_fini_4u(ident_t *loc, kmp_int32 gtid);
3882	extern void __kmpc_dispatch_fini_8u(ident_t *loc, kmp_int32 gtid);
3883
3884	extern void __kmpc_dispatch_deinit(ident_t *loc, kmp_int32 gtid);
3885
3886	#ifdef KMP_GOMP_COMPAT
3887
3888	extern void __kmp_aux_dispatch_init_4(ident_t *loc, kmp_int32 gtid,
3889	enum sched_type schedule, kmp_int32 lb,
3890	kmp_int32 ub, kmp_int32 st,
3891	kmp_int32 chunk, int push_ws);
3892	extern void __kmp_aux_dispatch_init_4u(ident_t *loc, kmp_int32 gtid,
3893	enum sched_type schedule, kmp_uint32 lb,
3894	kmp_uint32 ub, kmp_int32 st,
3895	kmp_int32 chunk, int push_ws);
3896	extern void __kmp_aux_dispatch_init_8(ident_t *loc, kmp_int32 gtid,
3897	enum sched_type schedule, kmp_int64 lb,
3898	kmp_int64 ub, kmp_int64 st,
3899	kmp_int64 chunk, int push_ws);
3900	extern void __kmp_aux_dispatch_init_8u(ident_t *loc, kmp_int32 gtid,
3901	enum sched_type schedule, kmp_uint64 lb,
3902	kmp_uint64 ub, kmp_int64 st,
3903	kmp_int64 chunk, int push_ws);
3904	extern void __kmp_aux_dispatch_fini_chunk_4(ident_t *loc, kmp_int32 gtid);
3905	extern void __kmp_aux_dispatch_fini_chunk_8(ident_t *loc, kmp_int32 gtid);
3906	extern void __kmp_aux_dispatch_fini_chunk_4u(ident_t *loc, kmp_int32 gtid);
3907	extern void __kmp_aux_dispatch_fini_chunk_8u(ident_t *loc, kmp_int32 gtid);
3908
3909	#endif /* KMP_GOMP_COMPAT */
3910
3911	extern kmp_uint32 __kmp_eq_4(kmp_uint32 value, kmp_uint32 checker);
3912	extern kmp_uint32 __kmp_neq_4(kmp_uint32 value, kmp_uint32 checker);
3913	extern kmp_uint32 __kmp_lt_4(kmp_uint32 value, kmp_uint32 checker);
3914	extern kmp_uint32 __kmp_ge_4(kmp_uint32 value, kmp_uint32 checker);
3915	extern kmp_uint32 __kmp_le_4(kmp_uint32 value, kmp_uint32 checker);
3916	extern kmp_uint32 __kmp_wait_4(kmp_uint32 volatile *spinner, kmp_uint32 checker,
3917	kmp_uint32 (*pred)(kmp_uint32, kmp_uint32),
3918	void *obj);
3919	extern void __kmp_wait_4_ptr(void *spinner, kmp_uint32 checker,
3920	kmp_uint32 (*pred)(void *, kmp_uint32), void *obj);
3921
3922	extern void __kmp_wait_64(kmp_info_t *this_thr, kmp_flag_64<> *flag,
3923	int final_spin
3924	#if USE_ITT_BUILD
3925	,
3926	void *itt_sync_obj
3927	#endif
3928	);
3929	extern void __kmp_release_64(kmp_flag_64<> *flag);
3930
3931	extern void __kmp_infinite_loop(void);
3932
3933	extern void __kmp_cleanup(void);
3934
3935	#if KMP_HANDLE_SIGNALS
3936	extern int __kmp_handle_signals;
3937	extern void __kmp_install_signals(int parallel_init);
3938	extern void __kmp_remove_signals(void);
3939	#endif
3940
3941	extern void __kmp_clear_system_time(void);
3942	extern void __kmp_read_system_time(double *delta);
3943
3944	extern void __kmp_check_stack_overlap(kmp_info_t *thr);
3945
3946	extern void __kmp_expand_host_name(char *buffer, size_t size);
3947	extern void __kmp_expand_file_name(char *result, size_t rlen, char *pattern);
3948
3949	#if KMP_ARCH_X86 || KMP_ARCH_X86_64 || (KMP_OS_WINDOWS && (KMP_ARCH_AARCH64 || KMP_ARCH_ARM))
3950	extern void
3951	__kmp_initialize_system_tick(void); /* Initialize timer tick value */
3952	#endif
3953
3954	extern void
3955	__kmp_runtime_initialize(void); /* machine specific initialization */
3956	extern void __kmp_runtime_destroy(void);
3957
3958	#if KMP_AFFINITY_SUPPORTED
3959	extern char *__kmp_affinity_print_mask(char *buf, int buf_len,
3960	kmp_affin_mask_t *mask);
3961	extern kmp_str_buf_t *__kmp_affinity_str_buf_mask(kmp_str_buf_t *buf,
3962	kmp_affin_mask_t *mask);
3963	extern void __kmp_affinity_initialize(kmp_affinity_t &affinity);
3964	extern void __kmp_affinity_uninitialize(void);
3965	extern void __kmp_affinity_set_init_mask(
3966	int gtid, int isa_root); /* set affinity according to KMP_AFFINITY */
3967	void __kmp_affinity_bind_init_mask(int gtid);
3968	extern void __kmp_affinity_bind_place(int gtid);
3969	extern void __kmp_affinity_determine_capable(const char *env_var);
3970	extern int __kmp_aux_set_affinity(void **mask);
3971	extern int __kmp_aux_get_affinity(void **mask);
3972	extern int __kmp_aux_get_affinity_max_proc();
3973	extern int __kmp_aux_set_affinity_mask_proc(int proc, void **mask);
3974	extern int __kmp_aux_unset_affinity_mask_proc(int proc, void **mask);
3975	extern int __kmp_aux_get_affinity_mask_proc(int proc, void **mask);
3976	extern void __kmp_balanced_affinity(kmp_info_t *th, int team_size);
3977	#if KMP_WEIGHTED_ITERATIONS_SUPPORTED
3978	extern int __kmp_get_first_osid_with_ecore(void);
3979	#endif
3980	#if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY || \
3981	KMP_OS_AIX
3982	extern int kmp_set_thread_affinity_mask_initial(void);
3983	#endif
3984	static inline void __kmp_assign_root_init_mask() {
3985	int gtid = __kmp_entry_gtid();
3986	kmp_root_t *r = __kmp_threads[gtid]->th.th_root;
3987	if (r->r.r_uber_thread == __kmp_threads[gtid] && !r->r.r_affinity_assigned) {
3988	__kmp_affinity_set_init_mask(gtid, /*isa_root=*/TRUE);
3989	__kmp_affinity_bind_init_mask(gtid);
3990	r->r.r_affinity_assigned = TRUE;
3991	}
3992	}
3993	static inline void __kmp_reset_root_init_mask(int gtid) {
3994	if (!KMP_AFFINITY_CAPABLE())
3995	return;
3996	kmp_info_t *th = __kmp_threads[gtid];
3997	kmp_root_t *r = th->th.th_root;
3998	if (r->r.r_uber_thread == th && r->r.r_affinity_assigned) {
3999	__kmp_set_system_affinity(__kmp_affin_origMask, FALSE);
4000	KMP_CPU_COPY(th->th.th_affin_mask, __kmp_affin_origMask);
4001	r->r.r_affinity_assigned = FALSE;
4002	}
4003	}
4004	#else /* KMP_AFFINITY_SUPPORTED */
4005	#define __kmp_assign_root_init_mask() /* Nothing */
4006	static inline void __kmp_reset_root_init_mask(int gtid) {}
4007	#endif /* KMP_AFFINITY_SUPPORTED */
4008	// No need for KMP_AFFINITY_SUPPORTED guard as only one field in the
4009	// format string is for affinity, so platforms that do not support
4010	// affinity can still use the other fields, e.g., %n for num_threads
4011	extern size_t __kmp_aux_capture_affinity(int gtid, const char *format,
4012	kmp_str_buf_t *buffer);
4013	extern void __kmp_aux_display_affinity(int gtid, const char *format);
4014
4015	extern void __kmp_cleanup_hierarchy();
4016	extern void __kmp_get_hierarchy(kmp_uint32 nproc, kmp_bstate_t *thr_bar);
4017
4018	#if KMP_USE_FUTEX
4019
4020	extern int __kmp_futex_determine_capable(void);
4021
4022	#endif // KMP_USE_FUTEX
4023
4024	extern void __kmp_gtid_set_specific(int gtid);
4025	extern int __kmp_gtid_get_specific(void);
4026
4027	extern double __kmp_read_cpu_time(void);
4028
4029	extern int __kmp_read_system_info(struct kmp_sys_info *info);
4030
4031	#if KMP_USE_MONITOR
4032	extern void __kmp_create_monitor(kmp_info_t *th);
4033	#endif
4034
4035	extern void *__kmp_launch_thread(kmp_info_t *thr);
4036
4037	extern void __kmp_create_worker(int gtid, kmp_info_t *th, size_t stack_size);
4038
4039	#if KMP_OS_WINDOWS
4040	extern int __kmp_still_running(kmp_info_t *th);
4041	extern int __kmp_is_thread_alive(kmp_info_t *th, DWORD *exit_val);
4042	extern void __kmp_free_handle(kmp_thread_t tHandle);
4043	#endif
4044
4045	#if KMP_USE_MONITOR
4046	extern void __kmp_reap_monitor(kmp_info_t *th);
4047	#endif
4048	extern void __kmp_reap_worker(kmp_info_t *th);
4049	extern void __kmp_terminate_thread(int gtid);
4050
4051	extern int __kmp_try_suspend_mx(kmp_info_t *th);
4052	extern void __kmp_lock_suspend_mx(kmp_info_t *th);
4053	extern void __kmp_unlock_suspend_mx(kmp_info_t *th);
4054
4055	extern void __kmp_elapsed(double *);
4056	extern void __kmp_elapsed_tick(double *);
4057
4058	extern void __kmp_enable(int old_state);
4059	extern void __kmp_disable(int *old_state);
4060
4061	extern void __kmp_thread_sleep(int millis);
4062
4063	extern void __kmp_common_initialize(void);
4064	extern void __kmp_common_destroy(void);
4065	extern void __kmp_common_destroy_gtid(int gtid);
4066
4067	#if KMP_OS_UNIX
4068	extern void __kmp_register_atfork(void);
4069	#endif
4070	extern void __kmp_suspend_initialize(void);
4071	extern void __kmp_suspend_initialize_thread(kmp_info_t *th);
4072	extern void __kmp_suspend_uninitialize_thread(kmp_info_t *th);
4073
4074	extern kmp_info_t *__kmp_allocate_thread(kmp_root_t *root, kmp_team_t *team,
4075	int tid);
4076	extern kmp_team_t *
4077	__kmp_allocate_team(kmp_root_t *root, int new_nproc, int max_nproc,
4078	#if OMPT_SUPPORT
4079	ompt_data_t ompt_parallel_data,
4080	#endif
4081	kmp_proc_bind_t proc_bind, kmp_internal_control_t *new_icvs,
4082	int argc USE_NESTED_HOT_ARG(kmp_info_t *thr));
4083	extern void __kmp_free_thread(kmp_info_t *);
4084	extern void __kmp_free_team(kmp_root_t *,
4085	kmp_team_t *USE_NESTED_HOT_ARG(kmp_info_t *));
4086	extern kmp_team_t *__kmp_reap_team(kmp_team_t *);
4087
4088	/* ------------------------------------------------------------------------ */
4089
4090	extern void __kmp_initialize_bget(kmp_info_t *th);
4091	extern void __kmp_finalize_bget(kmp_info_t *th);
4092
4093	KMP_EXPORT void *kmpc_malloc(size_t size);
4094	KMP_EXPORT void *kmpc_aligned_malloc(size_t size, size_t alignment);
4095	KMP_EXPORT void *kmpc_calloc(size_t nelem, size_t elsize);
4096	KMP_EXPORT void *kmpc_realloc(void *ptr, size_t size);
4097	KMP_EXPORT void kmpc_free(void *ptr);
4098
4099	/* declarations for internal use */
4100
4101	extern int __kmp_barrier(enum barrier_type bt, int gtid, int is_split,
4102	size_t reduce_size, void *reduce_data,
4103	void (*reduce)(void *, void *));
4104	extern void __kmp_end_split_barrier(enum barrier_type bt, int gtid);
4105	extern int __kmp_barrier_gomp_cancel(int gtid);
4106
4107	/*!
4108	* Tell the fork call which compiler generated the fork call, and therefore how
4109	* to deal with the call.
4110	*/
4111	enum fork_context_e {
4112	fork_context_gnu, /**< Called from GNU generated code, so must not invoke the
4113	microtask internally. */
4114	fork_context_intel, /**< Called from Intel generated code. */
4115	fork_context_last
4116	};
4117	extern int __kmp_fork_call(ident_t *loc, int gtid,
4118	enum fork_context_e fork_context, kmp_int32 argc,
4119	microtask_t microtask, launch_t invoker,
4120	kmp_va_list ap);
4121
4122	extern void __kmp_join_call(ident_t *loc, int gtid
4123	#if OMPT_SUPPORT
4124	,
4125	enum fork_context_e fork_context
4126	#endif
4127	,
4128	int exit_teams = 0);
4129
4130	extern void __kmp_serialized_parallel(ident_t *id, kmp_int32 gtid);
4131	extern void __kmp_internal_fork(ident_t *id, int gtid, kmp_team_t *team);
4132	extern void __kmp_internal_join(ident_t *id, int gtid, kmp_team_t *team);
4133	extern int __kmp_invoke_task_func(int gtid);
4134	extern void __kmp_run_before_invoked_task(int gtid, int tid,
4135	kmp_info_t *this_thr,
4136	kmp_team_t *team);
4137	extern void __kmp_run_after_invoked_task(int gtid, int tid,
4138	kmp_info_t *this_thr,
4139	kmp_team_t *team);
4140
4141	// should never have been exported
4142	KMP_EXPORT int __kmpc_invoke_task_func(int gtid);
4143	extern int __kmp_invoke_teams_master(int gtid);
4144	extern void __kmp_teams_master(int gtid);
4145	extern int __kmp_aux_get_team_num();
4146	extern int __kmp_aux_get_num_teams();
4147	extern void __kmp_save_internal_controls(kmp_info_t *thread);
4148	extern void __kmp_user_set_library(enum library_type arg);
4149	extern void __kmp_aux_set_library(enum library_type arg);
4150	extern void __kmp_aux_set_stacksize(size_t arg);
4151	extern void __kmp_aux_set_blocktime(int arg, kmp_info_t *thread, int tid);
4152	extern void __kmp_aux_set_defaults(char const *str, size_t len);
4153
4154	/* Functions called from __kmp_aux_env_initialize() in kmp_settings.cpp */
4155	void kmpc_set_blocktime(int arg);
4156	void ompc_set_nested(int flag);
4157	void ompc_set_dynamic(int flag);
4158	void ompc_set_num_threads(int arg);
4159
4160	extern void __kmp_push_current_task_to_thread(kmp_info_t *this_thr,
4161	kmp_team_t *team, int tid);
4162	extern void __kmp_pop_current_task_from_thread(kmp_info_t *this_thr);
4163	extern kmp_task_t *__kmp_task_alloc(ident_t *loc_ref, kmp_int32 gtid,
4164	kmp_tasking_flags_t *flags,
4165	size_t sizeof_kmp_task_t,
4166	size_t sizeof_shareds,
4167	kmp_routine_entry_t task_entry);
4168	extern void __kmp_init_implicit_task(ident_t *loc_ref, kmp_info_t *this_thr,
4169	kmp_team_t *team, int tid,
4170	int set_curr_task);
4171	extern void __kmp_finish_implicit_task(kmp_info_t *this_thr);
4172	extern void __kmp_free_implicit_task(kmp_info_t *this_thr);
4173
4174	extern kmp_event_t *__kmpc_task_allow_completion_event(ident_t *loc_ref,
4175	int gtid,
4176	kmp_task_t *task);
4177	extern void __kmp_fulfill_event(kmp_event_t *event);
4178
4179	extern void __kmp_free_task_team(kmp_info_t *thread,
4180	kmp_task_team_t *task_team);
4181	extern void __kmp_reap_task_teams(void);
4182	extern void __kmp_push_task_team_node(kmp_info_t *thread, kmp_team_t *team);
4183	extern void __kmp_pop_task_team_node(kmp_info_t *thread, kmp_team_t *team);
4184	extern void __kmp_wait_to_unref_task_teams(void);
4185	extern void __kmp_task_team_setup(kmp_info_t *this_thr, kmp_team_t *team);
4186	extern void __kmp_task_team_sync(kmp_info_t *this_thr, kmp_team_t *team);
4187	extern void __kmp_task_team_wait(kmp_info_t *this_thr, kmp_team_t *team
4188	#if USE_ITT_BUILD
4189	,
4190	void *itt_sync_obj
4191	#endif /* USE_ITT_BUILD */
4192	,
4193	int wait = 1);
4194	extern void __kmp_tasking_barrier(kmp_team_t *team, kmp_info_t *thread,
4195	int gtid);
4196	#if KMP_DEBUG
4197	#define KMP_DEBUG_ASSERT_TASKTEAM_INVARIANT(team, thr) \
4198	KMP_DEBUG_ASSERT( \
4199	__kmp_tasking_mode != tskm_task_teams || team->t.t_nproc == 1 || \
4200	thr->th.th_task_team == team->t.t_task_team[thr->th.th_task_state])
4201	#else
4202	#define KMP_DEBUG_ASSERT_TASKTEAM_INVARIANT(team, thr) /* Nothing */
4203	#endif
4204
4205	extern int __kmp_is_address_mapped(void *addr);
4206	extern kmp_uint64 __kmp_hardware_timestamp(void);
4207
4208	#if KMP_OS_UNIX
4209	extern int __kmp_read_from_file(char const *path, char const *format, ...);
4210	#endif
4211
4212	/* ------------------------------------------------------------------------ */
4213	//
4214	// Assembly routines that have no compiler intrinsic replacement
4215	//
4216
4217	extern int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int npr, int argc,
4218	void *argv[]
4219	#if OMPT_SUPPORT
4220	,
4221	void **exit_frame_ptr
4222	#endif
4223	);
4224
4225	/* ------------------------------------------------------------------------ */
4226
4227	KMP_EXPORT void __kmpc_begin(ident_t *, kmp_int32 flags);
4228	KMP_EXPORT void __kmpc_end(ident_t *);
4229
4230	KMP_EXPORT void __kmpc_threadprivate_register_vec(ident_t *, void *data,
4231	kmpc_ctor_vec ctor,
4232	kmpc_cctor_vec cctor,
4233	kmpc_dtor_vec dtor,
4234	size_t vector_length);
4235	KMP_EXPORT void __kmpc_threadprivate_register(ident_t *, void *data,
4236	kmpc_ctor ctor, kmpc_cctor cctor,
4237	kmpc_dtor dtor);
4238	KMP_EXPORT void *__kmpc_threadprivate(ident_t *, kmp_int32 global_tid,
4239	void *data, size_t size);
4240
4241	KMP_EXPORT kmp_int32 __kmpc_global_thread_num(ident_t *);
4242	KMP_EXPORT kmp_int32 __kmpc_global_num_threads(ident_t *);
4243	KMP_EXPORT kmp_int32 __kmpc_bound_thread_num(ident_t *);
4244	KMP_EXPORT kmp_int32 __kmpc_bound_num_threads(ident_t *);
4245
4246	KMP_EXPORT kmp_int32 __kmpc_ok_to_fork(ident_t *);
4247	KMP_EXPORT void __kmpc_fork_call(ident_t *, kmp_int32 nargs,
4248	kmpc_micro microtask, ...);
4249	KMP_EXPORT void __kmpc_fork_call_if(ident_t *loc, kmp_int32 nargs,
4250	kmpc_micro microtask, kmp_int32 cond,
4251	void *args);
4252
4253	KMP_EXPORT void __kmpc_serialized_parallel(ident_t *, kmp_int32 global_tid);
4254	KMP_EXPORT void __kmpc_end_serialized_parallel(ident_t *, kmp_int32 global_tid);
4255
4256	KMP_EXPORT void __kmpc_flush(ident_t *);
4257	KMP_EXPORT void __kmpc_barrier(ident_t *, kmp_int32 global_tid);
4258	KMP_EXPORT kmp_int32 __kmpc_master(ident_t *, kmp_int32 global_tid);
4259	KMP_EXPORT void __kmpc_end_master(ident_t *, kmp_int32 global_tid);
4260	KMP_EXPORT kmp_int32 __kmpc_masked(ident_t *, kmp_int32 global_tid,
4261	kmp_int32 filter);
4262	KMP_EXPORT void __kmpc_end_masked(ident_t *, kmp_int32 global_tid);
4263	KMP_EXPORT void __kmpc_ordered(ident_t *, kmp_int32 global_tid);
4264	KMP_EXPORT void __kmpc_end_ordered(ident_t *, kmp_int32 global_tid);
4265	KMP_EXPORT void __kmpc_critical(ident_t *, kmp_int32 global_tid,
4266	kmp_critical_name *);
4267	KMP_EXPORT void __kmpc_end_critical(ident_t *, kmp_int32 global_tid,
4268	kmp_critical_name *);
4269	KMP_EXPORT void __kmpc_critical_with_hint(ident_t *, kmp_int32 global_tid,
4270	kmp_critical_name *, uint32_t hint);
4271
4272	KMP_EXPORT kmp_int32 __kmpc_barrier_master(ident_t *, kmp_int32 global_tid);
4273	KMP_EXPORT void __kmpc_end_barrier_master(ident_t *, kmp_int32 global_tid);
4274
4275	KMP_EXPORT kmp_int32 __kmpc_barrier_master_nowait(ident_t *,
4276	kmp_int32 global_tid);
4277
4278	KMP_EXPORT kmp_int32 __kmpc_single(ident_t *, kmp_int32 global_tid);
4279	KMP_EXPORT void __kmpc_end_single(ident_t *, kmp_int32 global_tid);
4280
4281	KMP_EXPORT kmp_int32 __kmpc_sections_init(ident_t *loc, kmp_int32 global_tid);
4282	KMP_EXPORT kmp_int32 __kmpc_next_section(ident_t *loc, kmp_int32 global_tid,
4283	kmp_int32 numberOfSections);
4284	KMP_EXPORT void __kmpc_end_sections(ident_t *loc, kmp_int32 global_tid);
4285
4286	KMP_EXPORT void KMPC_FOR_STATIC_INIT(ident_t *loc, kmp_int32 global_tid,
4287	kmp_int32 schedtype, kmp_int32 *plastiter,
4288	kmp_int *plower, kmp_int *pupper,
4289	kmp_int *pstride, kmp_int incr,
4290	kmp_int chunk);
4291
4292	KMP_EXPORT void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid);
4293
4294	KMP_EXPORT void __kmpc_copyprivate(ident_t *loc, kmp_int32 global_tid,
4295	size_t cpy_size, void *cpy_data,
4296	void (*cpy_func)(void *, void *),
4297	kmp_int32 didit);
4298
4299	KMP_EXPORT void *__kmpc_copyprivate_light(ident_t *loc, kmp_int32 gtid,
4300	void *cpy_data);
4301
4302	extern void KMPC_SET_NUM_THREADS(int arg);
4303	extern void KMPC_SET_DYNAMIC(int flag);
4304	extern void KMPC_SET_NESTED(int flag);
4305
4306	/* OMP 3.0 tasking interface routines */
4307	KMP_EXPORT kmp_int32 __kmpc_omp_task(ident_t *loc_ref, kmp_int32 gtid,
4308	kmp_task_t *new_task);
4309	KMP_EXPORT kmp_task_t *__kmpc_omp_task_alloc(ident_t *loc_ref, kmp_int32 gtid,
4310	kmp_int32 flags,
4311	size_t sizeof_kmp_task_t,
4312	size_t sizeof_shareds,
4313	kmp_routine_entry_t task_entry);
4314	KMP_EXPORT kmp_task_t *__kmpc_omp_target_task_alloc(
4315	ident_t *loc_ref, kmp_int32 gtid, kmp_int32 flags, size_t sizeof_kmp_task_t,
4316	size_t sizeof_shareds, kmp_routine_entry_t task_entry, kmp_int64 device_id);
4317	KMP_EXPORT void __kmpc_omp_task_begin_if0(ident_t *loc_ref, kmp_int32 gtid,
4318	kmp_task_t *task);
4319	KMP_EXPORT void __kmpc_omp_task_complete_if0(ident_t *loc_ref, kmp_int32 gtid,
4320	kmp_task_t *task);
4321	KMP_EXPORT kmp_int32 __kmpc_omp_task_parts(ident_t *loc_ref, kmp_int32 gtid,
4322	kmp_task_t *new_task);
4323	KMP_EXPORT kmp_int32 __kmpc_omp_taskwait(ident_t *loc_ref, kmp_int32 gtid);
4324	KMP_EXPORT kmp_int32 __kmpc_omp_taskyield(ident_t *loc_ref, kmp_int32 gtid,
4325	int end_part);
4326
4327	#if TASK_UNUSED
4328	void __kmpc_omp_task_begin(ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *task);
4329	void __kmpc_omp_task_complete(ident_t *loc_ref, kmp_int32 gtid,
4330	kmp_task_t *task);
4331	#endif // TASK_UNUSED
4332
4333	/* ------------------------------------------------------------------------ */
4334
4335	KMP_EXPORT void __kmpc_taskgroup(ident_t *loc, int gtid);
4336	KMP_EXPORT void __kmpc_end_taskgroup(ident_t *loc, int gtid);
4337
4338	KMP_EXPORT kmp_int32 __kmpc_omp_task_with_deps(
4339	ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *new_task, kmp_int32 ndeps,
4340	kmp_depend_info_t *dep_list, kmp_int32 ndeps_noalias,
4341	kmp_depend_info_t *noalias_dep_list);
4342
4343	KMP_EXPORT kmp_base_depnode_t *__kmpc_task_get_depnode(kmp_task_t *task);
4344
4345	KMP_EXPORT kmp_depnode_list_t *__kmpc_task_get_successors(kmp_task_t *task);
4346
4347	KMP_EXPORT void __kmpc_omp_wait_deps(ident_t *loc_ref, kmp_int32 gtid,
4348	kmp_int32 ndeps,
4349	kmp_depend_info_t *dep_list,
4350	kmp_int32 ndeps_noalias,
4351	kmp_depend_info_t *noalias_dep_list);
4352	/* __kmpc_omp_taskwait_deps_51 : Function for OpenMP 5.1 nowait clause.
4353	* Placeholder for taskwait with nowait clause.*/
4354	KMP_EXPORT void __kmpc_omp_taskwait_deps_51(ident_t *loc_ref, kmp_int32 gtid,
4355	kmp_int32 ndeps,
4356	kmp_depend_info_t *dep_list,
4357	kmp_int32 ndeps_noalias,
4358	kmp_depend_info_t *noalias_dep_list,
4359	kmp_int32 has_no_wait);
4360
4361	extern kmp_int32 __kmp_omp_task(kmp_int32 gtid, kmp_task_t *new_task,
4362	bool serialize_immediate);
4363
4364	KMP_EXPORT kmp_int32 __kmpc_cancel(ident_t *loc_ref, kmp_int32 gtid,
4365	kmp_int32 cncl_kind);
4366	KMP_EXPORT kmp_int32 __kmpc_cancellationpoint(ident_t *loc_ref, kmp_int32 gtid,
4367	kmp_int32 cncl_kind);
4368	KMP_EXPORT kmp_int32 __kmpc_cancel_barrier(ident_t *loc_ref, kmp_int32 gtid);
4369	KMP_EXPORT int __kmp_get_cancellation_status(int cancel_kind);
4370
4371	KMP_EXPORT void __kmpc_proxy_task_completed(kmp_int32 gtid, kmp_task_t *ptask);
4372	KMP_EXPORT void __kmpc_proxy_task_completed_ooo(kmp_task_t *ptask);
4373	KMP_EXPORT void __kmpc_taskloop(ident_t *loc, kmp_int32 gtid, kmp_task_t *task,
4374	kmp_int32 if_val, kmp_uint64 *lb,
4375	kmp_uint64 *ub, kmp_int64 st, kmp_int32 nogroup,
4376	kmp_int32 sched, kmp_uint64 grainsize,
4377	void *task_dup);
4378	KMP_EXPORT void __kmpc_taskloop_5(ident_t *loc, kmp_int32 gtid,
4379	kmp_task_t *task, kmp_int32 if_val,
4380	kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
4381	kmp_int32 nogroup, kmp_int32 sched,
4382	kmp_uint64 grainsize, kmp_int32 modifier,
4383	void *task_dup);
4384	KMP_EXPORT void *__kmpc_task_reduction_init(int gtid, int num_data, void *data);
4385	KMP_EXPORT void *__kmpc_taskred_init(int gtid, int num_data, void *data);
4386	KMP_EXPORT void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void *d);
4387	KMP_EXPORT void *__kmpc_task_reduction_modifier_init(ident_t *loc, int gtid,
4388	int is_ws, int num,
4389	void *data);
4390	KMP_EXPORT void *__kmpc_taskred_modifier_init(ident_t *loc, int gtid, int is_ws,
4391	int num, void *data);
4392	KMP_EXPORT void __kmpc_task_reduction_modifier_fini(ident_t *loc, int gtid,
4393	int is_ws);
4394	KMP_EXPORT kmp_int32 __kmpc_omp_reg_task_with_affinity(
4395	ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *new_task, kmp_int32 naffins,
4396	kmp_task_affinity_info_t *affin_list);
4397	KMP_EXPORT void __kmp_set_num_teams(int num_teams);
4398	KMP_EXPORT int __kmp_get_max_teams(void);
4399	KMP_EXPORT void __kmp_set_teams_thread_limit(int limit);
4400	KMP_EXPORT int __kmp_get_teams_thread_limit(void);
4401
4402	/* Interface target task integration */
4403	KMP_EXPORT void **__kmpc_omp_get_target_async_handle_ptr(kmp_int32 gtid);
4404	KMP_EXPORT bool __kmpc_omp_has_task_team(kmp_int32 gtid);
4405
4406	/* Lock interface routines (fast versions with gtid passed in) */
4407	KMP_EXPORT void __kmpc_init_lock(ident_t *loc, kmp_int32 gtid,
4408	void **user_lock);
4409	KMP_EXPORT void __kmpc_init_nest_lock(ident_t *loc, kmp_int32 gtid,
4410	void **user_lock);
4411	KMP_EXPORT void __kmpc_destroy_lock(ident_t *loc, kmp_int32 gtid,
4412	void **user_lock);
4413	KMP_EXPORT void __kmpc_destroy_nest_lock(ident_t *loc, kmp_int32 gtid,
4414	void **user_lock);
4415	KMP_EXPORT void __kmpc_set_lock(ident_t *loc, kmp_int32 gtid, void **user_lock);
4416	KMP_EXPORT void __kmpc_set_nest_lock(ident_t *loc, kmp_int32 gtid,
4417	void **user_lock);
4418	KMP_EXPORT void __kmpc_unset_lock(ident_t *loc, kmp_int32 gtid,
4419	void **user_lock);
4420	KMP_EXPORT void __kmpc_unset_nest_lock(ident_t *loc, kmp_int32 gtid,
4421	void **user_lock);
4422	KMP_EXPORT int __kmpc_test_lock(ident_t *loc, kmp_int32 gtid, void **user_lock);
4423	KMP_EXPORT int __kmpc_test_nest_lock(ident_t *loc, kmp_int32 gtid,
4424	void **user_lock);
4425
4426	KMP_EXPORT void __kmpc_init_lock_with_hint(ident_t *loc, kmp_int32 gtid,
4427	void **user_lock, uintptr_t hint);
4428	KMP_EXPORT void __kmpc_init_nest_lock_with_hint(ident_t *loc, kmp_int32 gtid,
4429	void **user_lock,
4430	uintptr_t hint);
4431
4432	#if OMPX_TASKGRAPH
4433	// Taskgraph's Record & Replay mechanism
4434	// __kmp_tdg_is_recording: check whether a given TDG is recording
4435	// status: the tdg's current status
4436	static inline bool __kmp_tdg_is_recording(kmp_tdg_status_t status) {
4437	return status == KMP_TDG_RECORDING;
4438	}
4439
4440	KMP_EXPORT kmp_int32 __kmpc_start_record_task(ident_t *loc, kmp_int32 gtid,
4441	kmp_int32 input_flags,
4442	kmp_int32 tdg_id);
4443	KMP_EXPORT void __kmpc_end_record_task(ident_t *loc, kmp_int32 gtid,
4444	kmp_int32 input_flags, kmp_int32 tdg_id);
4445	#endif
4446	/* Interface to fast scalable reduce methods routines */
4447
4448	KMP_EXPORT kmp_int32 __kmpc_reduce_nowait(
4449	ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size,
4450	void *reduce_data, void (*reduce_func)(void *lhs_data, void *rhs_data),
4451	kmp_critical_name *lck);
4452	KMP_EXPORT void __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid,
4453	kmp_critical_name *lck);
4454	KMP_EXPORT kmp_int32 __kmpc_reduce(
4455	ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size,
4456	void *reduce_data, void (*reduce_func)(void *lhs_data, void *rhs_data),
4457	kmp_critical_name *lck);
4458	KMP_EXPORT void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid,
4459	kmp_critical_name *lck);
4460
4461	/* Internal fast reduction routines */
4462
4463	extern PACKED_REDUCTION_METHOD_T __kmp_determine_reduction_method(
4464	ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size,
4465	void *reduce_data, void (*reduce_func)(void *lhs_data, void *rhs_data),
4466	kmp_critical_name *lck);
4467
4468	// this function is for testing set/get/determine reduce method
4469	KMP_EXPORT kmp_int32 __kmp_get_reduce_method(void);
4470
4471	KMP_EXPORT kmp_uint64 __kmpc_get_taskid();
4472	KMP_EXPORT kmp_uint64 __kmpc_get_parent_taskid();
4473
4474	// C++ port
4475	// missing 'extern "C"' declarations
4476
4477	KMP_EXPORT kmp_int32 __kmpc_in_parallel(ident_t *loc);
4478	KMP_EXPORT void __kmpc_pop_num_threads(ident_t *loc, kmp_int32 global_tid);
4479	KMP_EXPORT void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid,
4480	kmp_int32 num_threads);
4481	KMP_EXPORT void __kmpc_push_num_threads_strict(ident_t *loc,
4482	kmp_int32 global_tid,
4483	kmp_int32 num_threads,
4484	int severity,
4485	const char *message);
4486
4487	KMP_EXPORT void __kmpc_push_num_threads_list(ident_t *loc, kmp_int32 global_tid,
4488	kmp_uint32 list_length,
4489	kmp_int32 *num_threads_list);
4490	KMP_EXPORT void __kmpc_push_num_threads_list_strict(
4491	ident_t *loc, kmp_int32 global_tid, kmp_uint32 list_length,
4492	kmp_int32 *num_threads_list, int severity, const char *message);
4493
4494	KMP_EXPORT void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid,
4495	int proc_bind);
4496	KMP_EXPORT void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid,
4497	kmp_int32 num_teams,
4498	kmp_int32 num_threads);
4499	KMP_EXPORT void __kmpc_set_thread_limit(ident_t *loc, kmp_int32 global_tid,
4500	kmp_int32 thread_limit);
4501	/* Function for OpenMP 5.1 num_teams clause */
4502	KMP_EXPORT void __kmpc_push_num_teams_51(ident_t *loc, kmp_int32 global_tid,
4503	kmp_int32 num_teams_lb,
4504	kmp_int32 num_teams_ub,
4505	kmp_int32 num_threads);
4506	KMP_EXPORT void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc,
4507	kmpc_micro microtask, ...);
4508	struct kmp_dim { // loop bounds info casted to kmp_int64
4509	kmp_int64 lo; // lower
4510	kmp_int64 up; // upper
4511	kmp_int64 st; // stride
4512	};
4513	KMP_EXPORT void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid,
4514	kmp_int32 num_dims,
4515	const struct kmp_dim *dims);
4516	KMP_EXPORT void __kmpc_doacross_wait(ident_t *loc, kmp_int32 gtid,
4517	const kmp_int64 *vec);
4518	KMP_EXPORT void __kmpc_doacross_post(ident_t *loc, kmp_int32 gtid,
4519	const kmp_int64 *vec);
4520	KMP_EXPORT void __kmpc_doacross_fini(ident_t *loc, kmp_int32 gtid);
4521
4522	KMP_EXPORT void *__kmpc_threadprivate_cached(ident_t *loc, kmp_int32 global_tid,
4523	void *data, size_t size,
4524	void ***cache);
4525
4526	// The routines below are not exported.
4527	// Consider making them 'static' in corresponding source files.
4528	void kmp_threadprivate_insert_private_data(int gtid, void *pc_addr,
4529	void *data_addr, size_t pc_size);
4530	struct private_common *kmp_threadprivate_insert(int gtid, void *pc_addr,
4531	void *data_addr,
4532	size_t pc_size);
4533	void __kmp_threadprivate_resize_cache(int newCapacity);
4534	void __kmp_cleanup_threadprivate_caches();
4535
4536	// ompc_, kmpc_ entries moved from omp.h.
4537	#if KMP_OS_WINDOWS
4538	#define KMPC_CONVENTION __cdecl
4539	#else
4540	#define KMPC_CONVENTION
4541	#endif
4542
4543	#ifndef __OMP_H
4544	typedef enum omp_sched_t {
4545	omp_sched_static = 1,
4546	omp_sched_dynamic = 2,
4547	omp_sched_guided = 3,
4548	omp_sched_auto = 4
4549	} omp_sched_t;
4550	typedef void *kmp_affinity_mask_t;
4551	#endif
4552
4553	KMP_EXPORT void KMPC_CONVENTION ompc_set_max_active_levels(int);
4554	KMP_EXPORT void KMPC_CONVENTION ompc_set_schedule(omp_sched_t, int);
4555	KMP_EXPORT int KMPC_CONVENTION ompc_get_ancestor_thread_num(int);
4556	KMP_EXPORT int KMPC_CONVENTION ompc_get_team_size(int);
4557	KMP_EXPORT int KMPC_CONVENTION
4558	kmpc_set_affinity_mask_proc(int, kmp_affinity_mask_t *);
4559	KMP_EXPORT int KMPC_CONVENTION
4560	kmpc_unset_affinity_mask_proc(int, kmp_affinity_mask_t *);
4561	KMP_EXPORT int KMPC_CONVENTION
4562	kmpc_get_affinity_mask_proc(int, kmp_affinity_mask_t *);
4563
4564	KMP_EXPORT void KMPC_CONVENTION kmpc_set_stacksize(int);
4565	KMP_EXPORT void KMPC_CONVENTION kmpc_set_stacksize_s(size_t);
4566	KMP_EXPORT void KMPC_CONVENTION kmpc_set_library(int);
4567	KMP_EXPORT void KMPC_CONVENTION kmpc_set_defaults(char const *);
4568	KMP_EXPORT void KMPC_CONVENTION kmpc_set_disp_num_buffers(int);
4569	void KMP_EXPAND_NAME(ompc_set_affinity_format)(char const *format);
4570	size_t KMP_EXPAND_NAME(ompc_get_affinity_format)(char *buffer, size_t size);
4571	void KMP_EXPAND_NAME(ompc_display_affinity)(char const *format);
4572	size_t KMP_EXPAND_NAME(ompc_capture_affinity)(char *buffer, size_t buf_size,
4573	char const *format);
4574
4575	enum kmp_target_offload_kind {
4576	tgt_disabled = 0,
4577	tgt_default = 1,
4578	tgt_mandatory = 2
4579	};
4580	typedef enum kmp_target_offload_kind kmp_target_offload_kind_t;
4581	// Set via OMP_TARGET_OFFLOAD if specified, defaults to tgt_default otherwise
4582	extern kmp_target_offload_kind_t __kmp_target_offload;
4583	extern int __kmpc_get_target_offload();
4584
4585	// Constants used in libomptarget
4586	#define KMP_DEVICE_DEFAULT -1 // This is libomptarget's default device.
4587	#define KMP_DEVICE_ALL -11 // This is libomptarget's "all devices".
4588
4589	// OMP Pause Resource
4590
4591	// The following enum is used both to set the status in __kmp_pause_status, and
4592	// as the internal equivalent of the externally-visible omp_pause_resource_t.
4593	typedef enum kmp_pause_status_t {
4594	kmp_not_paused = 0, // status is not paused, or, requesting resume
4595	kmp_soft_paused = 1, // status is soft-paused, or, requesting soft pause
4596	kmp_hard_paused = 2, // status is hard-paused, or, requesting hard pause
4597	kmp_stop_tool_paused = 3 // requesting stop_tool pause
4598	} kmp_pause_status_t;
4599
4600	// This stores the pause state of the runtime
4601	extern kmp_pause_status_t __kmp_pause_status;
4602	extern int __kmpc_pause_resource(kmp_pause_status_t level);
4603	extern int __kmp_pause_resource(kmp_pause_status_t level);
4604	// Soft resume sets __kmp_pause_status, and wakes up all threads.
4605	extern void __kmp_resume_if_soft_paused();
4606	// Hard resume simply resets the status to not paused. Library will appear to
4607	// be uninitialized after hard pause. Let OMP constructs trigger required
4608	// initializations.
4609	static inline void __kmp_resume_if_hard_paused() {
4610	if (__kmp_pause_status == kmp_hard_paused) {
4611	__kmp_pause_status = kmp_not_paused;
4612	}
4613	}
4614
4615	extern void __kmp_omp_display_env(int verbose);
4616
4617	// 1: it is initializing hidden helper team
4618	extern volatile int __kmp_init_hidden_helper;
4619	// 1: the hidden helper team is done
4620	extern volatile int __kmp_hidden_helper_team_done;
4621	// 1: enable hidden helper task
4622	extern kmp_int32 __kmp_enable_hidden_helper;
4623	// Main thread of hidden helper team
4624	extern kmp_info_t *__kmp_hidden_helper_main_thread;
4625	// Descriptors for the hidden helper threads
4626	extern kmp_info_t **__kmp_hidden_helper_threads;
4627	// Number of hidden helper threads
4628	extern kmp_int32 __kmp_hidden_helper_threads_num;
4629	// Number of hidden helper tasks that have not been executed yet
4630	extern std::atomic<kmp_int32> __kmp_unexecuted_hidden_helper_tasks;
4631
4632	extern void __kmp_hidden_helper_initialize();
4633	extern void __kmp_hidden_helper_threads_initz_routine();
4634	extern void __kmp_do_initialize_hidden_helper_threads();
4635	extern void __kmp_hidden_helper_threads_initz_wait();
4636	extern void __kmp_hidden_helper_initz_release();
4637	extern void __kmp_hidden_helper_threads_deinitz_wait();
4638	extern void __kmp_hidden_helper_threads_deinitz_release();
4639	extern void __kmp_hidden_helper_main_thread_wait();
4640	extern void __kmp_hidden_helper_worker_thread_wait();
4641	extern void __kmp_hidden_helper_worker_thread_signal();
4642	extern void __kmp_hidden_helper_main_thread_release();
4643
4644	// Check whether a given thread is a hidden helper thread
4645	#define KMP_HIDDEN_HELPER_THREAD(gtid) \
4646	((gtid) >= 1 && (gtid) <= __kmp_hidden_helper_threads_num)
4647
4648	#define KMP_HIDDEN_HELPER_WORKER_THREAD(gtid) \
4649	((gtid) > 1 && (gtid) <= __kmp_hidden_helper_threads_num)
4650
4651	#define KMP_HIDDEN_HELPER_MAIN_THREAD(gtid) \
4652	((gtid) == 1 && (gtid) <= __kmp_hidden_helper_threads_num)
4653
4654	#define KMP_HIDDEN_HELPER_TEAM(team) \
4655	(team->t.t_threads[0] == __kmp_hidden_helper_main_thread)
4656
4657	// Map a gtid to a hidden helper thread. The first hidden helper thread, a.k.a
4658	// main thread, is skipped.
4659	#define KMP_GTID_TO_SHADOW_GTID(gtid) \
4660	((gtid) % (__kmp_hidden_helper_threads_num - 1) + 2)
4661
4662	// Return the adjusted gtid value by subtracting from gtid the number
4663	// of hidden helper threads. This adjusted value is the gtid the thread would
4664	// have received if there were no hidden helper threads.
4665	static inline int __kmp_adjust_gtid_for_hidden_helpers(int gtid) {
4666	int adjusted_gtid = gtid;
4667	if (__kmp_hidden_helper_threads_num > 0 && gtid > 0 &&
4668	gtid - __kmp_hidden_helper_threads_num >= 0) {
4669	adjusted_gtid -= __kmp_hidden_helper_threads_num;
4670	}
4671	return adjusted_gtid;
4672	}
4673
4674	// Support for error directive
4675	typedef enum kmp_severity_t {
4676	severity_warning = 1,
4677	severity_fatal = 2
4678	} kmp_severity_t;
4679	extern void __kmpc_error(ident_t *loc, int severity, const char *message);
4680
4681	// Support for scope directive
4682	KMP_EXPORT void __kmpc_scope(ident_t *loc, kmp_int32 gtid, void *reserved);
4683	KMP_EXPORT void __kmpc_end_scope(ident_t *loc, kmp_int32 gtid, void *reserved);
4684
4685	#ifdef __cplusplus
4686	}
4687	#endif
4688
4689	template <bool C, bool S>
4690	extern void __kmp_suspend_32(int th_gtid, kmp_flag_32<C, S> *flag);
4691	template <bool C, bool S>
4692	extern void __kmp_suspend_64(int th_gtid, kmp_flag_64<C, S> *flag);
4693	template <bool C, bool S>
4694	extern void __kmp_atomic_suspend_64(int th_gtid,
4695	kmp_atomic_flag_64<C, S> *flag);
4696	extern void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag);
4697	#if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT
4698	template <bool C, bool S>
4699	extern void __kmp_mwait_32(int th_gtid, kmp_flag_32<C, S> *flag);
4700	template <bool C, bool S>
4701	extern void __kmp_mwait_64(int th_gtid, kmp_flag_64<C, S> *flag);
4702	template <bool C, bool S>
4703	extern void __kmp_atomic_mwait_64(int th_gtid, kmp_atomic_flag_64<C, S> *flag);
4704	extern void __kmp_mwait_oncore(int th_gtid, kmp_flag_oncore *flag);
4705	#endif
4706	template <bool C, bool S>
4707	extern void __kmp_resume_32(int target_gtid, kmp_flag_32<C, S> *flag);
4708	template <bool C, bool S>
4709	extern void __kmp_resume_64(int target_gtid, kmp_flag_64<C, S> *flag);
4710	template <bool C, bool S>
4711	extern void __kmp_atomic_resume_64(int target_gtid,
4712	kmp_atomic_flag_64<C, S> *flag);
4713	extern void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag);
4714
4715	template <bool C, bool S>
4716	int __kmp_execute_tasks_32(kmp_info_t *thread, kmp_int32 gtid,
4717	kmp_flag_32<C, S> *flag, int final_spin,
4718	int *thread_finished,
4719	#if USE_ITT_BUILD
4720	void *itt_sync_obj,
4721	#endif /* USE_ITT_BUILD */
4722	kmp_int32 is_constrained);
4723	template <bool C, bool S>
4724	int __kmp_execute_tasks_64(kmp_info_t *thread, kmp_int32 gtid,
4725	kmp_flag_64<C, S> *flag, int final_spin,
4726	int *thread_finished,
4727	#if USE_ITT_BUILD
4728	void *itt_sync_obj,
4729	#endif /* USE_ITT_BUILD */
4730	kmp_int32 is_constrained);
4731	template <bool C, bool S>
4732	int __kmp_atomic_execute_tasks_64(kmp_info_t *thread, kmp_int32 gtid,
4733	kmp_atomic_flag_64<C, S> *flag,
4734	int final_spin, int *thread_finished,
4735	#if USE_ITT_BUILD
4736	void *itt_sync_obj,
4737	#endif /* USE_ITT_BUILD */
4738	kmp_int32 is_constrained);
4739	int __kmp_execute_tasks_oncore(kmp_info_t *thread, kmp_int32 gtid,
4740	kmp_flag_oncore *flag, int final_spin,
4741	int *thread_finished,
4742	#if USE_ITT_BUILD
4743	void *itt_sync_obj,
4744	#endif /* USE_ITT_BUILD */
4745	kmp_int32 is_constrained);
4746
4747	extern int __kmp_nesting_mode;
4748	extern int __kmp_nesting_mode_nlevels;
4749	extern int *__kmp_nesting_nth_level;
4750	extern void __kmp_init_nesting_mode();
4751	extern void __kmp_set_nesting_mode_threads();
4752
4753	/// This class safely opens and closes a C-style FILE* object using RAII
4754	/// semantics. There are also methods which allow using stdout or stderr as
4755	/// the underlying FILE* object. With the implicit conversion operator to
4756	/// FILE*, an object with this type can be used in any function which takes
4757	/// a FILE* object e.g., fprintf().
4758	/// No close method is needed at use sites.
4759	class kmp_safe_raii_file_t {
4760	FILE *f;
4761
4762	void close() {
4763	if (f && f != stdout && f != stderr) {
4764	fclose(stream: f);
4765	f = nullptr;
4766	}
4767	}
4768
4769	public:
4770	kmp_safe_raii_file_t() : f(nullptr) {}
4771	kmp_safe_raii_file_t(const char *filename, const char *mode,
4772	const char *env_var = nullptr)
4773	: f(nullptr) {
4774	open(filename, mode, env_var);
4775	}
4776	kmp_safe_raii_file_t(const kmp_safe_raii_file_t &other) = delete;
4777	kmp_safe_raii_file_t &operator=(const kmp_safe_raii_file_t &other) = delete;
4778	~kmp_safe_raii_file_t() { close(); }
4779
4780	/// Open filename using mode. This is automatically closed in the destructor.
4781	/// The env_var parameter indicates the environment variable the filename
4782	/// came from if != nullptr.
4783	void open(const char *filename, const char *mode,
4784	const char *env_var = nullptr) {
4785	KMP_ASSERT(!f);
4786	f = fopen(filename: filename, modes: mode);
4787	if (!f) {
4788	int code = errno;
4789	if (env_var) {
4790	__kmp_fatal(KMP_MSG(CantOpenFileForReading, filename), KMP_ERR(code),
4791	KMP_HNT(CheckEnvVar, env_var, filename), __kmp_msg_null);
4792	} else {
4793	__kmp_fatal(KMP_MSG(CantOpenFileForReading, filename), KMP_ERR(code),
4794	__kmp_msg_null);
4795	}
4796	}
4797	}
4798	/// Instead of erroring out, return non-zero when
4799	/// unsuccessful fopen() for any reason
4800	int try_open(const char *filename, const char *mode) {
4801	KMP_ASSERT(!f);
4802	f = fopen(filename: filename, modes: mode);
4803	if (!f)
4804	return errno;
4805	return 0;
4806	}
4807	/// Set the FILE* object to stdout and output there
4808	/// No open call should happen before this call.
4809	void set_stdout() {
4810	KMP_ASSERT(!f);
4811	f = stdout;
4812	}
4813	/// Set the FILE* object to stderr and output there
4814	/// No open call should happen before this call.
4815	void set_stderr() {
4816	KMP_ASSERT(!f);
4817	f = stderr;
4818	}
4819	operator bool() { return bool(f); }
4820	operator FILE *() { return f; }
4821	};
4822
4823	template <typename SourceType, typename TargetType,
4824	bool isSourceSmaller = (sizeof(SourceType) < sizeof(TargetType)),
4825	bool isSourceEqual = (sizeof(SourceType) == sizeof(TargetType)),
4826	bool isSourceSigned = std::is_signed<SourceType>::value,
4827	bool isTargetSigned = std::is_signed<TargetType>::value>
4828	struct kmp_convert {};
4829
4830	// Both types are signed; Source smaller
4831	template <typename SourceType, typename TargetType>
4832	struct kmp_convert<SourceType, TargetType, true, false, true, true> {
4833	static TargetType to(SourceType src) { return (TargetType)src; }
4834	};
4835	// Source equal
4836	template <typename SourceType, typename TargetType>
4837	struct kmp_convert<SourceType, TargetType, false, true, true, true> {
4838	static TargetType to(SourceType src) { return src; }
4839	};
4840	// Source bigger
4841	template <typename SourceType, typename TargetType>
4842	struct kmp_convert<SourceType, TargetType, false, false, true, true> {
4843	static TargetType to(SourceType src) {
4844	KMP_ASSERT(src <= static_cast<SourceType>(
4845	(std::numeric_limits<TargetType>::max)()));
4846	KMP_ASSERT(src >= static_cast<SourceType>(
4847	(std::numeric_limits<TargetType>::min)()));
4848	return (TargetType)src;
4849	}
4850	};
4851
4852	// Source signed, Target unsigned
4853	// Source smaller
4854	template <typename SourceType, typename TargetType>
4855	struct kmp_convert<SourceType, TargetType, true, false, true, false> {
4856	static TargetType to(SourceType src) {
4857	KMP_ASSERT(src >= 0);
4858	return (TargetType)src;
4859	}
4860	};
4861	// Source equal
4862	template <typename SourceType, typename TargetType>
4863	struct kmp_convert<SourceType, TargetType, false, true, true, false> {
4864	static TargetType to(SourceType src) {
4865	KMP_ASSERT(src >= 0);
4866	return (TargetType)src;
4867	}
4868	};
4869	// Source bigger
4870	template <typename SourceType, typename TargetType>
4871	struct kmp_convert<SourceType, TargetType, false, false, true, false> {
4872	static TargetType to(SourceType src) {
4873	KMP_ASSERT(src >= 0);
4874	KMP_ASSERT(src <= static_cast<SourceType>(
4875	(std::numeric_limits<TargetType>::max)()));
4876	return (TargetType)src;
4877	}
4878	};
4879
4880	// Source unsigned, Target signed
4881	// Source smaller
4882	template <typename SourceType, typename TargetType>
4883	struct kmp_convert<SourceType, TargetType, true, false, false, true> {
4884	static TargetType to(SourceType src) { return (TargetType)src; }
4885	};
4886	// Source equal
4887	template <typename SourceType, typename TargetType>
4888	struct kmp_convert<SourceType, TargetType, false, true, false, true> {
4889	static TargetType to(SourceType src) {
4890	KMP_ASSERT(src <= static_cast<SourceType>(
4891	(std::numeric_limits<TargetType>::max)()));
4892	return (TargetType)src;
4893	}
4894	};
4895	// Source bigger
4896	template <typename SourceType, typename TargetType>
4897	struct kmp_convert<SourceType, TargetType, false, false, false, true> {
4898	static TargetType to(SourceType src) {
4899	KMP_ASSERT(src <= static_cast<SourceType>(
4900	(std::numeric_limits<TargetType>::max)()));
4901	return (TargetType)src;
4902	}
4903	};
4904
4905	// Source unsigned, Target unsigned
4906	// Source smaller
4907	template <typename SourceType, typename TargetType>
4908	struct kmp_convert<SourceType, TargetType, true, false, false, false> {
4909	static TargetType to(SourceType src) { return (TargetType)src; }
4910	};
4911	// Source equal
4912	template <typename SourceType, typename TargetType>
4913	struct kmp_convert<SourceType, TargetType, false, true, false, false> {
4914	static TargetType to(SourceType src) { return src; }
4915	};
4916	// Source bigger
4917	template <typename SourceType, typename TargetType>
4918	struct kmp_convert<SourceType, TargetType, false, false, false, false> {
4919	static TargetType to(SourceType src) {
4920	KMP_ASSERT(src <= static_cast<SourceType>(
4921	(std::numeric_limits<TargetType>::max)()));
4922	return (TargetType)src;
4923	}
4924	};
4925
4926	template <typename T1, typename T2>
4927	static inline void __kmp_type_convert(T1 src, T2 *dest) {
4928	*dest = kmp_convert<T1, T2>::to(src);
4929	}
4930
4931	#endif /* KMP_H */
4932