1	/*
2	* z_Linux_util.cpp -- platform specific routines.
3	*/
4
5	//===----------------------------------------------------------------------===//
6	//
7	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
8	// See https://llvm.org/LICENSE.txt for license information.
9	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
10	//
11	//===----------------------------------------------------------------------===//
12
13	#include "kmp.h"
14	#include "kmp_affinity.h"
15	#include "kmp_i18n.h"
16	#include "kmp_io.h"
17	#include "kmp_itt.h"
18	#include "kmp_lock.h"
19	#include "kmp_stats.h"
20	#include "kmp_str.h"
21	#include "kmp_wait_release.h"
22	#include "kmp_wrapper_getpid.h"
23
24	#if !KMP_OS_DRAGONFLY && !KMP_OS_FREEBSD && !KMP_OS_NETBSD && !KMP_OS_OPENBSD
25	#include <alloca.h>
26	#endif
27	#include <math.h> // HUGE_VAL.
28	#if KMP_OS_LINUX
29	#include <semaphore.h>
30	#endif // KMP_OS_LINUX
31	#include <sys/resource.h>
32	#if KMP_OS_AIX
33	#include <sys/ldr.h>
34	#include <libperfstat.h>
35	#elif !KMP_OS_HAIKU
36	#include <sys/syscall.h>
37	#endif
38	#include <sys/time.h>
39	#include <sys/times.h>
40	#include <unistd.h>
41
42	#if KMP_OS_LINUX
43	#include <sys/sysinfo.h>
44	#if KMP_USE_FUTEX
45	// We should really include <futex.h>, but that causes compatibility problems on
46	// different Linux* OS distributions that either require that you include (or
47	// break when you try to include) <pci/types.h>. Since all we need is the two
48	// macros below (which are part of the kernel ABI, so can't change) we just
49	// define the constants here and don't include <futex.h>
50	#ifndef FUTEX_WAIT
51	#define FUTEX_WAIT 0
52	#endif
53	#ifndef FUTEX_WAKE
54	#define FUTEX_WAKE 1
55	#endif
56	#endif
57	#elif KMP_OS_DARWIN
58	#include <mach/mach.h>
59	#include <sys/sysctl.h>
60	#elif KMP_OS_DRAGONFLY || KMP_OS_FREEBSD
61	#include <sys/types.h>
62	#include <sys/sysctl.h>
63	#include <sys/user.h>
64	#include <pthread_np.h>
65	#if KMP_OS_DRAGONFLY
66	#include <kvm.h>
67	#endif
68	#elif KMP_OS_NETBSD || KMP_OS_OPENBSD
69	#include <sys/types.h>
70	#include <sys/sysctl.h>
71	#if KMP_OS_NETBSD
72	#include <sched.h>
73	#endif
74	#if KMP_OS_OPENBSD
75	#include <pthread_np.h>
76	#endif
77	#elif KMP_OS_SOLARIS
78	#include <procfs.h>
79	#include <thread.h>
80	#include <sys/loadavg.h>
81	#endif
82
83	#include <ctype.h>
84	#include <dirent.h>
85	#include <fcntl.h>
86
87	struct kmp_sys_timer {
88	struct timespec start;
89	};
90
91	#ifndef TIMEVAL_TO_TIMESPEC
92	// Convert timeval to timespec.
93	#define TIMEVAL_TO_TIMESPEC(tv, ts) \
94	do { \
95	(ts)->tv_sec = (tv)->tv_sec; \
96	(ts)->tv_nsec = (tv)->tv_usec * 1000; \
97	} while (0)
98	#endif
99
100	// Convert timespec to nanoseconds.
101	#define TS2NS(timespec) \
102	(((timespec).tv_sec * (long int)1e9) + (timespec).tv_nsec)
103
104	static struct kmp_sys_timer __kmp_sys_timer_data;
105
106	#if KMP_HANDLE_SIGNALS
107	typedef void (*sig_func_t)(int);
108	STATIC_EFI2_WORKAROUND struct sigaction __kmp_sighldrs[NSIG];
109	static sigset_t __kmp_sigset;
110	#endif
111
112	static int __kmp_init_runtime = FALSE;
113
114	static int __kmp_fork_count = 0;
115
116	static pthread_condattr_t __kmp_suspend_cond_attr;
117	static pthread_mutexattr_t __kmp_suspend_mutex_attr;
118
119	static kmp_cond_align_t __kmp_wait_cv;
120	static kmp_mutex_align_t __kmp_wait_mx;
121
122	kmp_uint64 __kmp_ticks_per_msec = 1000000;
123	kmp_uint64 __kmp_ticks_per_usec = 1000;
124
125	#ifdef DEBUG_SUSPEND
126	static void __kmp_print_cond(char *buffer, kmp_cond_align_t *cond) {
127	KMP_SNPRINTF(buffer, 128, "(cond (lock (%ld, %d)), (descr (%p)))",
128	cond->c_cond.__c_lock.__status, cond->c_cond.__c_lock.__spinlock,
129	cond->c_cond.__c_waiting);
130	}
131	#endif
132
133	#if ((KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY || \
134	KMP_OS_AIX) && \
135	KMP_AFFINITY_SUPPORTED)
136
137	/* Affinity support */
138
139	void __kmp_affinity_bind_thread(int which) {
140	KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
141	"Illegal set affinity operation when not capable");
142
143	kmp_affin_mask_t *mask;
144	KMP_CPU_ALLOC_ON_STACK(mask);
145	KMP_CPU_ZERO(mask);
146	KMP_CPU_SET(which, mask);
147	__kmp_set_system_affinity(mask, TRUE);
148	KMP_CPU_FREE_FROM_STACK(mask);
149	}
150
151	#if KMP_OS_AIX
152	void __kmp_affinity_determine_capable(const char *env_var) {
153	// All versions of AIX support bindprocessor().
154
155	size_t mask_size = __kmp_xproc / CHAR_BIT;
156	// Round up to byte boundary.
157	if (__kmp_xproc % CHAR_BIT)
158	++mask_size;
159
160	// Round up to the mask_size_type boundary.
161	if (mask_size % sizeof(__kmp_affin_mask_size))
162	mask_size += sizeof(__kmp_affin_mask_size) -
163	mask_size % sizeof(__kmp_affin_mask_size);
164	KMP_AFFINITY_ENABLE(mask_size);
165	KA_TRACE(10,
166	("__kmp_affinity_determine_capable: "
167	"AIX OS affinity interface bindprocessor functional (mask size = "
168	"%" KMP_SIZE_T_SPEC ").\n",
169	__kmp_affin_mask_size));
170	}
171
172	#else // !KMP_OS_AIX
173
174	/* Determine if we can access affinity functionality on this version of
175	* Linux* OS by checking __NR_sched_{get,set}affinity system calls, and set
176	* __kmp_affin_mask_size to the appropriate value (0 means not capable). */
177	void __kmp_affinity_determine_capable(const char *env_var) {
178	// Check and see if the OS supports thread affinity.
179
180	#if KMP_OS_LINUX
181	#define KMP_CPU_SET_SIZE_LIMIT (1024 * 1024)
182	#define KMP_CPU_SET_TRY_SIZE CACHE_LINE
183	#elif KMP_OS_FREEBSD || KMP_OS_DRAGONFLY
184	#define KMP_CPU_SET_SIZE_LIMIT (sizeof(cpuset_t))
185	#elif KMP_OS_NETBSD
186	#define KMP_CPU_SET_SIZE_LIMIT (256)
187	#endif
188
189	int verbose = __kmp_affinity.flags.verbose;
190	int warnings = __kmp_affinity.flags.warnings;
191	enum affinity_type type = __kmp_affinity.type;
192
193	#if KMP_OS_LINUX
194	long gCode;
195	unsigned char *buf;
196	buf = (unsigned char *)KMP_INTERNAL_MALLOC(KMP_CPU_SET_SIZE_LIMIT);
197
198	// If the syscall returns a suggestion for the size,
199	// then we don't have to search for an appropriate size.
200	gCode = syscall(__NR_sched_getaffinity, 0, KMP_CPU_SET_TRY_SIZE, buf);
201	KA_TRACE(30, ("__kmp_affinity_determine_capable: "
202	"initial getaffinity call returned %ld errno = %d\n",
203	gCode, errno));
204
205	if (gCode < 0 && errno != EINVAL) {
206	// System call not supported
207	if (verbose ||
208	(warnings && (type != affinity_none) && (type != affinity_default) &&
209	(type != affinity_disabled))) {
210	int error = errno;
211	kmp_msg_t err_code = KMP_ERR(error);
212	__kmp_msg(kmp_ms_warning, KMP_MSG(GetAffSysCallNotSupported, env_var),
213	err_code, __kmp_msg_null);
214	if (__kmp_generate_warnings == kmp_warnings_off) {
215	__kmp_str_free(str: &err_code.str);
216	}
217	}
218	KMP_AFFINITY_DISABLE();
219	KMP_INTERNAL_FREE(buf);
220	return;
221	} else if (gCode > 0) {
222	// The optimal situation: the OS returns the size of the buffer it expects.
223	KMP_AFFINITY_ENABLE(gCode);
224	KA_TRACE(10, ("__kmp_affinity_determine_capable: "
225	"affinity supported (mask size %d)\n",
226	(int)__kmp_affin_mask_size));
227	KMP_INTERNAL_FREE(buf);
228	return;
229	}
230
231	// Call the getaffinity system call repeatedly with increasing set sizes
232	// until we succeed, or reach an upper bound on the search.
233	KA_TRACE(30, ("__kmp_affinity_determine_capable: "
234	"searching for proper set size\n"));
235	int size;
236	for (size = 1; size <= KMP_CPU_SET_SIZE_LIMIT; size *= 2) {
237	gCode = syscall(__NR_sched_getaffinity, 0, size, buf);
238	KA_TRACE(30, ("__kmp_affinity_determine_capable: "
239	"getaffinity for mask size %ld returned %ld errno = %d\n",
240	size, gCode, errno));
241
242	if (gCode < 0) {
243	if (errno == ENOSYS) {
244	// We shouldn't get here
245	KA_TRACE(30, ("__kmp_affinity_determine_capable: "
246	"inconsistent OS call behavior: errno == ENOSYS for mask "
247	"size %d\n",
248	size));
249	if (verbose ||
250	(warnings && (type != affinity_none) &&
251	(type != affinity_default) && (type != affinity_disabled))) {
252	int error = errno;
253	kmp_msg_t err_code = KMP_ERR(error);
254	__kmp_msg(kmp_ms_warning, KMP_MSG(GetAffSysCallNotSupported, env_var),
255	err_code, __kmp_msg_null);
256	if (__kmp_generate_warnings == kmp_warnings_off) {
257	__kmp_str_free(str: &err_code.str);
258	}
259	}
260	KMP_AFFINITY_DISABLE();
261	KMP_INTERNAL_FREE(buf);
262	return;
263	}
264	continue;
265	}
266
267	KMP_AFFINITY_ENABLE(gCode);
268	KA_TRACE(10, ("__kmp_affinity_determine_capable: "
269	"affinity supported (mask size %d)\n",
270	(int)__kmp_affin_mask_size));
271	KMP_INTERNAL_FREE(buf);
272	return;
273	}
274	#elif KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY
275	long gCode;
276	unsigned char *buf;
277	buf = (unsigned char *)KMP_INTERNAL_MALLOC(KMP_CPU_SET_SIZE_LIMIT);
278	gCode = pthread_getaffinity_np(pthread_self(), KMP_CPU_SET_SIZE_LIMIT,
279	reinterpret_cast<cpuset_t *>(buf));
280	KA_TRACE(30, ("__kmp_affinity_determine_capable: "
281	"initial getaffinity call returned %d errno = %d\n",
282	gCode, errno));
283	if (gCode == 0) {
284	KMP_AFFINITY_ENABLE(KMP_CPU_SET_SIZE_LIMIT);
285	KA_TRACE(10, ("__kmp_affinity_determine_capable: "
286	"affinity supported (mask size %d)\n",
287	(int)__kmp_affin_mask_size));
288	KMP_INTERNAL_FREE(buf);
289	return;
290	}
291	#endif
292	KMP_INTERNAL_FREE(buf);
293
294	// Affinity is not supported
295	KMP_AFFINITY_DISABLE();
296	KA_TRACE(10, ("__kmp_affinity_determine_capable: "
297	"cannot determine mask size - affinity not supported\n"));
298	if (verbose || (warnings && (type != affinity_none) &&
299	(type != affinity_default) && (type != affinity_disabled))) {
300	KMP_WARNING(AffCantGetMaskSize, env_var);
301	}
302	}
303	#endif // KMP_OS_AIX
304	#endif // (KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
305	KMP_OS_DRAGONFLY || KMP_OS_AIX) && KMP_AFFINITY_SUPPORTED
306
307	#if KMP_USE_FUTEX
308
309	int __kmp_futex_determine_capable() {
310	int loc = 0;
311	long rc = syscall(__NR_futex, &loc, FUTEX_WAKE, 1, NULL, NULL, 0);
312	int retval = (rc == 0) || (errno != ENOSYS);
313
314	KA_TRACE(10,
315	("__kmp_futex_determine_capable: rc = %d errno = %d\n", rc, errno));
316	KA_TRACE(10, ("__kmp_futex_determine_capable: futex syscall%s supported\n",
317	retval ? "" : " not"));
318
319	return retval;
320	}
321
322	#endif // KMP_USE_FUTEX
323
324	#if (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_WASM) && (!KMP_ASM_INTRINS)
325	/* Only 32-bit "add-exchange" instruction on IA-32 architecture causes us to
326	use compare_and_store for these routines */
327
328	kmp_int8 __kmp_test_then_or8(volatile kmp_int8 *p, kmp_int8 d) {
329	kmp_int8 old_value, new_value;
330
331	old_value = TCR_1(*p);
332	new_value = old_value | d;
333
334	while (!KMP_COMPARE_AND_STORE_REL8(p, old_value, new_value)) {
335	KMP_CPU_PAUSE();
336	old_value = TCR_1(*p);
337	new_value = old_value | d;
338	}
339	return old_value;
340	}
341
342	kmp_int8 __kmp_test_then_and8(volatile kmp_int8 *p, kmp_int8 d) {
343	kmp_int8 old_value, new_value;
344
345	old_value = TCR_1(*p);
346	new_value = old_value & d;
347
348	while (!KMP_COMPARE_AND_STORE_REL8(p, old_value, new_value)) {
349	KMP_CPU_PAUSE();
350	old_value = TCR_1(*p);
351	new_value = old_value & d;
352	}
353	return old_value;
354	}
355
356	kmp_uint32 __kmp_test_then_or32(volatile kmp_uint32 *p, kmp_uint32 d) {
357	kmp_uint32 old_value, new_value;
358
359	old_value = TCR_4(*p);
360	new_value = old_value | d;
361
362	while (!KMP_COMPARE_AND_STORE_REL32(p, old_value, new_value)) {
363	KMP_CPU_PAUSE();
364	old_value = TCR_4(*p);
365	new_value = old_value | d;
366	}
367	return old_value;
368	}
369
370	kmp_uint32 __kmp_test_then_and32(volatile kmp_uint32 *p, kmp_uint32 d) {
371	kmp_uint32 old_value, new_value;
372
373	old_value = TCR_4(*p);
374	new_value = old_value & d;
375
376	while (!KMP_COMPARE_AND_STORE_REL32(p, old_value, new_value)) {
377	KMP_CPU_PAUSE();
378	old_value = TCR_4(*p);
379	new_value = old_value & d;
380	}
381	return old_value;
382	}
383
384	#if KMP_ARCH_X86 || KMP_ARCH_WASM
385	kmp_int8 __kmp_test_then_add8(volatile kmp_int8 *p, kmp_int8 d) {
386	kmp_int8 old_value, new_value;
387
388	old_value = TCR_1(*p);
389	new_value = old_value + d;
390
391	while (!KMP_COMPARE_AND_STORE_REL8(p, old_value, new_value)) {
392	KMP_CPU_PAUSE();
393	old_value = TCR_1(*p);
394	new_value = old_value + d;
395	}
396	return old_value;
397	}
398
399	kmp_int64 __kmp_test_then_add64(volatile kmp_int64 *p, kmp_int64 d) {
400	kmp_int64 old_value, new_value;
401
402	old_value = TCR_8(*p);
403	new_value = old_value + d;
404
405	while (!KMP_COMPARE_AND_STORE_REL64(p, old_value, new_value)) {
406	KMP_CPU_PAUSE();
407	old_value = TCR_8(*p);
408	new_value = old_value + d;
409	}
410	return old_value;
411	}
412	#endif /* KMP_ARCH_X86 */
413
414	kmp_uint64 __kmp_test_then_or64(volatile kmp_uint64 *p, kmp_uint64 d) {
415	kmp_uint64 old_value, new_value;
416
417	old_value = TCR_8(*p);
418	new_value = old_value | d;
419	while (!KMP_COMPARE_AND_STORE_REL64(p, old_value, new_value)) {
420	KMP_CPU_PAUSE();
421	old_value = TCR_8(*p);
422	new_value = old_value | d;
423	}
424	return old_value;
425	}
426
427	kmp_uint64 __kmp_test_then_and64(volatile kmp_uint64 *p, kmp_uint64 d) {
428	kmp_uint64 old_value, new_value;
429
430	old_value = TCR_8(*p);
431	new_value = old_value & d;
432	while (!KMP_COMPARE_AND_STORE_REL64(p, old_value, new_value)) {
433	KMP_CPU_PAUSE();
434	old_value = TCR_8(*p);
435	new_value = old_value & d;
436	}
437	return old_value;
438	}
439
440	#endif /* (KMP_ARCH_X86 || KMP_ARCH_X86_64) && (! KMP_ASM_INTRINS) */
441
442	void __kmp_terminate_thread(int gtid) {
443	int status;
444	kmp_info_t *th = __kmp_threads[gtid];
445
446	if (!th)
447	return;
448
449	#ifdef KMP_CANCEL_THREADS
450	KA_TRACE(10, ("__kmp_terminate_thread: kill (%d)\n", gtid));
451	status = pthread_cancel(th: th->th.th_info.ds.ds_thread);
452	if (status != 0 && status != ESRCH) {
453	__kmp_fatal(KMP_MSG(CantTerminateWorkerThread), KMP_ERR(status),
454	__kmp_msg_null);
455	}
456	#endif
457	KMP_YIELD(TRUE);
458	} //
459
460	/* Set thread stack info.
461	If values are unreasonable, assume call failed and use incremental stack
462	refinement method instead. Returns TRUE if the stack parameters could be
463	determined exactly, FALSE if incremental refinement is necessary. */
464	static kmp_int32 __kmp_set_stack_info(int gtid, kmp_info_t *th) {
465	int stack_data;
466	#if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
467	KMP_OS_HAIKU || KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_AIX
468	int status;
469	size_t size = 0;
470	void *addr = 0;
471
472	/* Always do incremental stack refinement for ubermaster threads since the
473	initial thread stack range can be reduced by sibling thread creation so
474	pthread_attr_getstack may cause thread gtid aliasing */
475	if (!KMP_UBER_GTID(gtid)) {
476
477	#if KMP_OS_SOLARIS
478	stack_t s;
479	if ((status = thr_stksegment(&s)) < 0) {
480	KMP_CHECK_SYSFAIL("thr_stksegment", status);
481	}
482
483	addr = s.ss_sp;
484	size = s.ss_size;
485	KA_TRACE(60, ("__kmp_set_stack_info: T#%d thr_stksegment returned size:"
486	" %lu, low addr: %p\n",
487	gtid, size, addr));
488	#else
489	pthread_attr_t attr;
490	/* Fetch the real thread attributes */
491	status = pthread_attr_init(attr: &attr);
492	KMP_CHECK_SYSFAIL("pthread_attr_init", status);
493	#if KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD
494	status = pthread_attr_get_np(pthread_self(), &attr);
495	KMP_CHECK_SYSFAIL("pthread_attr_get_np", status);
496	#else
497	status = pthread_getattr_np(th: pthread_self(), attr: &attr);
498	KMP_CHECK_SYSFAIL("pthread_getattr_np", status);
499	#endif
500	status = pthread_attr_getstack(attr: &attr, stackaddr: &addr, stacksize: &size);
501	KMP_CHECK_SYSFAIL("pthread_attr_getstack", status);
502	KA_TRACE(60,
503	("__kmp_set_stack_info: T#%d pthread_attr_getstack returned size:"
504	" %lu, low addr: %p\n",
505	gtid, size, addr));
506	status = pthread_attr_destroy(attr: &attr);
507	KMP_CHECK_SYSFAIL("pthread_attr_destroy", status);
508	#endif
509	}
510
511	if (size != 0 && addr != 0) { // was stack parameter determination successful?
512	/* Store the correct base and size */
513	TCW_PTR(th->th.th_info.ds.ds_stackbase, (((char *)addr) + size));
514	TCW_PTR(th->th.th_info.ds.ds_stacksize, size);
515	TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE);
516	return TRUE;
517	}
518	#endif /* KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD \
519	|| KMP_OS_HAIKU || KMP_OS_HURD || KMP_OS_SOLARIS */
520	/* Use incremental refinement starting from initial conservative estimate */
521	TCW_PTR(th->th.th_info.ds.ds_stacksize, 0);
522	TCW_PTR(th->th.th_info.ds.ds_stackbase, &stack_data);
523	TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE);
524	return FALSE;
525	}
526
527	static void *__kmp_launch_worker(void *thr) {
528	int status, old_type, old_state;
529	#ifdef KMP_BLOCK_SIGNALS
530	sigset_t new_set, old_set;
531	#endif /* KMP_BLOCK_SIGNALS */
532	void *exit_val;
533	#if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
534	KMP_OS_OPENBSD || KMP_OS_HAIKU || KMP_OS_HURD || KMP_OS_SOLARIS || \
535	KMP_OS_AIX
536	void *volatile padding = 0;
537	#endif
538	int gtid;
539
540	gtid = ((kmp_info_t *)thr)->th.th_info.ds.ds_gtid;
541	__kmp_gtid_set_specific(gtid);
542	#ifdef KMP_TDATA_GTID
543	__kmp_gtid = gtid;
544	#endif
545	#if KMP_STATS_ENABLED
546	// set thread local index to point to thread-specific stats
547	__kmp_stats_thread_ptr = ((kmp_info_t *)thr)->th.th_stats;
548	__kmp_stats_thread_ptr->startLife();
549	KMP_SET_THREAD_STATE(IDLE);
550	KMP_INIT_PARTITIONED_TIMERS(OMP_idle);
551	#endif
552
553	#if USE_ITT_BUILD
554	__kmp_itt_thread_name(gtid);
555	#endif /* USE_ITT_BUILD */
556
557	#if KMP_AFFINITY_SUPPORTED
558	__kmp_affinity_bind_init_mask(gtid);
559	#endif
560
561	#ifdef KMP_CANCEL_THREADS
562	status = pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, oldtype: &old_type);
563	KMP_CHECK_SYSFAIL("pthread_setcanceltype", status);
564	// josh todo: isn't PTHREAD_CANCEL_ENABLE default for newly-created threads?
565	status = pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, oldstate: &old_state);
566	KMP_CHECK_SYSFAIL("pthread_setcancelstate", status);
567	#endif
568
569	#if KMP_ARCH_X86 || KMP_ARCH_X86_64
570	// Set FP control regs to be a copy of the parallel initialization thread's.
571	__kmp_clear_x87_fpu_status_word();
572	__kmp_load_x87_fpu_control_word(p: &__kmp_init_x87_fpu_control_word);
573	__kmp_load_mxcsr(p: &__kmp_init_mxcsr);
574	#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
575
576	#ifdef KMP_BLOCK_SIGNALS
577	status = sigfillset(&new_set);
578	KMP_CHECK_SYSFAIL_ERRNO("sigfillset", status);
579	status = pthread_sigmask(SIG_BLOCK, &new_set, &old_set);
580	KMP_CHECK_SYSFAIL("pthread_sigmask", status);
581	#endif /* KMP_BLOCK_SIGNALS */
582
583	#if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
584	KMP_OS_OPENBSD || KMP_OS_HAIKU || KMP_OS_HURD || KMP_OS_SOLARIS || \
585	KMP_OS_AIX
586	if (__kmp_stkoffset > 0 && gtid > 0) {
587	padding = KMP_ALLOCA(gtid * __kmp_stkoffset);
588	(void)padding;
589	}
590	#endif
591
592	KMP_MB();
593	__kmp_set_stack_info(gtid, th: (kmp_info_t *)thr);
594
595	__kmp_check_stack_overlap(thr: (kmp_info_t *)thr);
596
597	exit_val = __kmp_launch_thread(thr: (kmp_info_t *)thr);
598
599	#ifdef KMP_BLOCK_SIGNALS
600	status = pthread_sigmask(SIG_SETMASK, &old_set, NULL);
601	KMP_CHECK_SYSFAIL("pthread_sigmask", status);
602	#endif /* KMP_BLOCK_SIGNALS */
603
604	return exit_val;
605	}
606
607	#if KMP_USE_MONITOR
608	/* The monitor thread controls all of the threads in the complex */
609
610	static void *__kmp_launch_monitor(void *thr) {
611	int status, old_type, old_state;
612	#ifdef KMP_BLOCK_SIGNALS
613	sigset_t new_set;
614	#endif /* KMP_BLOCK_SIGNALS */
615	struct timespec interval;
616
617	KMP_MB(); /* Flush all pending memory write invalidates. */
618
619	KA_TRACE(10, ("__kmp_launch_monitor: #1 launched\n"));
620
621	/* register us as the monitor thread */
622	__kmp_gtid_set_specific(KMP_GTID_MONITOR);
623	#ifdef KMP_TDATA_GTID
624	__kmp_gtid = KMP_GTID_MONITOR;
625	#endif
626
627	KMP_MB();
628
629	#if USE_ITT_BUILD
630	// Instruct Intel(R) Threading Tools to ignore monitor thread.
631	__kmp_itt_thread_ignore();
632	#endif /* USE_ITT_BUILD */
633
634	__kmp_set_stack_info(((kmp_info_t *)thr)->th.th_info.ds.ds_gtid,
635	(kmp_info_t *)thr);
636
637	__kmp_check_stack_overlap((kmp_info_t *)thr);
638
639	#ifdef KMP_CANCEL_THREADS
640	status = pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &old_type);
641	KMP_CHECK_SYSFAIL("pthread_setcanceltype", status);
642	// josh todo: isn't PTHREAD_CANCEL_ENABLE default for newly-created threads?
643	status = pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &old_state);
644	KMP_CHECK_SYSFAIL("pthread_setcancelstate", status);
645	#endif
646
647	#if KMP_REAL_TIME_FIX
648	// This is a potential fix which allows application with real-time scheduling
649	// policy work. However, decision about the fix is not made yet, so it is
650	// disabled by default.
651	{ // Are program started with real-time scheduling policy?
652	int sched = sched_getscheduler(0);
653	if (sched == SCHED_FIFO || sched == SCHED_RR) {
654	// Yes, we are a part of real-time application. Try to increase the
655	// priority of the monitor.
656	struct sched_param param;
657	int max_priority = sched_get_priority_max(sched);
658	int rc;
659	KMP_WARNING(RealTimeSchedNotSupported);
660	sched_getparam(0, &param);
661	if (param.sched_priority < max_priority) {
662	param.sched_priority += 1;
663	rc = sched_setscheduler(0, sched, &param);
664	if (rc != 0) {
665	int error = errno;
666	kmp_msg_t err_code = KMP_ERR(error);
667	__kmp_msg(kmp_ms_warning, KMP_MSG(CantChangeMonitorPriority),
668	err_code, KMP_MSG(MonitorWillStarve), __kmp_msg_null);
669	if (__kmp_generate_warnings == kmp_warnings_off) {
670	__kmp_str_free(&err_code.str);
671	}
672	}
673	} else {
674	// We cannot abort here, because number of CPUs may be enough for all
675	// the threads, including the monitor thread, so application could
676	// potentially work...
677	__kmp_msg(kmp_ms_warning, KMP_MSG(RunningAtMaxPriority),
678	KMP_MSG(MonitorWillStarve), KMP_HNT(RunningAtMaxPriority),
679	__kmp_msg_null);
680	}
681	}
682	// AC: free thread that waits for monitor started
683	TCW_4(__kmp_global.g.g_time.dt.t_value, 0);
684	}
685	#endif // KMP_REAL_TIME_FIX
686
687	KMP_MB(); /* Flush all pending memory write invalidates. */
688
689	if (__kmp_monitor_wakeups == 1) {
690	interval.tv_sec = 1;
691	interval.tv_nsec = 0;
692	} else {
693	interval.tv_sec = 0;
694	interval.tv_nsec = (KMP_NSEC_PER_SEC / __kmp_monitor_wakeups);
695	}
696
697	KA_TRACE(10, ("__kmp_launch_monitor: #2 monitor\n"));
698
699	while (!TCR_4(__kmp_global.g.g_done)) {
700	struct timespec now;
701	struct timeval tval;
702
703	/* This thread monitors the state of the system */
704
705	KA_TRACE(15, ("__kmp_launch_monitor: update\n"));
706
707	status = gettimeofday(&tval, NULL);
708	KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status);
709	TIMEVAL_TO_TIMESPEC(&tval, &now);
710
711	now.tv_sec += interval.tv_sec;
712	now.tv_nsec += interval.tv_nsec;
713
714	if (now.tv_nsec >= KMP_NSEC_PER_SEC) {
715	now.tv_sec += 1;
716	now.tv_nsec -= KMP_NSEC_PER_SEC;
717	}
718
719	status = pthread_mutex_lock(&__kmp_wait_mx.m_mutex);
720	KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
721	// AC: the monitor should not fall asleep if g_done has been set
722	if (!TCR_4(__kmp_global.g.g_done)) { // check once more under mutex
723	status = pthread_cond_timedwait(&__kmp_wait_cv.c_cond,
724	&__kmp_wait_mx.m_mutex, &now);
725	if (status != 0) {
726	if (status != ETIMEDOUT && status != EINTR) {
727	KMP_SYSFAIL("pthread_cond_timedwait", status);
728	}
729	}
730	}
731	status = pthread_mutex_unlock(&__kmp_wait_mx.m_mutex);
732	KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
733
734	TCW_4(__kmp_global.g.g_time.dt.t_value,
735	TCR_4(__kmp_global.g.g_time.dt.t_value) + 1);
736
737	KMP_MB(); /* Flush all pending memory write invalidates. */
738	}
739
740	KA_TRACE(10, ("__kmp_launch_monitor: #3 cleanup\n"));
741
742	#ifdef KMP_BLOCK_SIGNALS
743	status = sigfillset(&new_set);
744	KMP_CHECK_SYSFAIL_ERRNO("sigfillset", status);
745	status = pthread_sigmask(SIG_UNBLOCK, &new_set, NULL);
746	KMP_CHECK_SYSFAIL("pthread_sigmask", status);
747	#endif /* KMP_BLOCK_SIGNALS */
748
749	KA_TRACE(10, ("__kmp_launch_monitor: #4 finished\n"));
750
751	if (__kmp_global.g.g_abort != 0) {
752	/* now we need to terminate the worker threads */
753	/* the value of t_abort is the signal we caught */
754
755	int gtid;
756
757	KA_TRACE(10, ("__kmp_launch_monitor: #5 terminate sig=%d\n",
758	__kmp_global.g.g_abort));
759
760	/* terminate the OpenMP worker threads */
761	/* TODO this is not valid for sibling threads!!
762	* the uber master might not be 0 anymore.. */
763	for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid)
764	__kmp_terminate_thread(gtid);
765
766	__kmp_cleanup();
767
768	KA_TRACE(10, ("__kmp_launch_monitor: #6 raise sig=%d\n",
769	__kmp_global.g.g_abort));
770
771	if (__kmp_global.g.g_abort > 0)
772	raise(__kmp_global.g.g_abort);
773	}
774
775	KA_TRACE(10, ("__kmp_launch_monitor: #7 exit\n"));
776
777	return thr;
778	}
779	#endif // KMP_USE_MONITOR
780
781	void __kmp_create_worker(int gtid, kmp_info_t *th, size_t stack_size) {
782	pthread_t handle;
783	pthread_attr_t thread_attr;
784	int status;
785
786	th->th.th_info.ds.ds_gtid = gtid;
787
788	#if KMP_STATS_ENABLED
789	// sets up worker thread stats
790	__kmp_acquire_tas_lock(&__kmp_stats_lock, gtid);
791
792	// th->th.th_stats is used to transfer thread-specific stats-pointer to
793	// __kmp_launch_worker. So when thread is created (goes into
794	// __kmp_launch_worker) it will set its thread local pointer to
795	// th->th.th_stats
796	if (!KMP_UBER_GTID(gtid)) {
797	th->th.th_stats = __kmp_stats_list->push_back(gtid);
798	} else {
799	// For root threads, __kmp_stats_thread_ptr is set in __kmp_register_root(),
800	// so set the th->th.th_stats field to it.
801	th->th.th_stats = __kmp_stats_thread_ptr;
802	}
803	__kmp_release_tas_lock(&__kmp_stats_lock, gtid);
804
805	#endif // KMP_STATS_ENABLED
806
807	if (KMP_UBER_GTID(gtid)) {
808	KA_TRACE(10, ("__kmp_create_worker: uber thread (%d)\n", gtid));
809	th->th.th_info.ds.ds_thread = pthread_self();
810	__kmp_set_stack_info(gtid, th);
811	__kmp_check_stack_overlap(thr: th);
812	return;
813	}
814
815	KA_TRACE(10, ("__kmp_create_worker: try to create thread (%d)\n", gtid));
816
817	KMP_MB(); /* Flush all pending memory write invalidates. */
818
819	#ifdef KMP_THREAD_ATTR
820	status = pthread_attr_init(attr: &thread_attr);
821	if (status != 0) {
822	__kmp_fatal(KMP_MSG(CantInitThreadAttrs), KMP_ERR(status), __kmp_msg_null);
823	}
824	status = pthread_attr_setdetachstate(attr: &thread_attr, PTHREAD_CREATE_JOINABLE);
825	if (status != 0) {
826	__kmp_fatal(KMP_MSG(CantSetWorkerState), KMP_ERR(status), __kmp_msg_null);
827	}
828
829	/* Set stack size for this thread now.
830	The multiple of 2 is there because on some machines, requesting an unusual
831	stacksize causes the thread to have an offset before the dummy alloca()
832	takes place to create the offset. Since we want the user to have a
833	sufficient stacksize AND support a stack offset, we alloca() twice the
834	offset so that the upcoming alloca() does not eliminate any premade offset,
835	and also gives the user the stack space they requested for all threads */
836	stack_size += gtid * __kmp_stkoffset * 2;
837
838	KA_TRACE(10, ("__kmp_create_worker: T#%d, default stacksize = %lu bytes, "
839	"__kmp_stksize = %lu bytes, final stacksize = %lu bytes\n",
840	gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size));
841
842	#ifdef _POSIX_THREAD_ATTR_STACKSIZE
843	status = pthread_attr_setstacksize(attr: &thread_attr, stacksize: stack_size);
844	#ifdef KMP_BACKUP_STKSIZE
845	if (status != 0) {
846	if (!__kmp_env_stksize) {
847	stack_size = KMP_BACKUP_STKSIZE + gtid * __kmp_stkoffset;
848	__kmp_stksize = KMP_BACKUP_STKSIZE;
849	KA_TRACE(10, ("__kmp_create_worker: T#%d, default stacksize = %lu bytes, "
850	"__kmp_stksize = %lu bytes, (backup) final stacksize = %lu "
851	"bytes\n",
852	gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size));
853	status = pthread_attr_setstacksize(attr: &thread_attr, stacksize: stack_size);
854	}
855	}
856	#endif /* KMP_BACKUP_STKSIZE */
857	if (status != 0) {
858	__kmp_fatal(KMP_MSG(CantSetWorkerStackSize, stack_size), KMP_ERR(status),
859	KMP_HNT(ChangeWorkerStackSize), __kmp_msg_null);
860	}
861	#endif /* _POSIX_THREAD_ATTR_STACKSIZE */
862
863	#endif /* KMP_THREAD_ATTR */
864
865	status =
866	pthread_create(newthread: &handle, attr: &thread_attr, start_routine: __kmp_launch_worker, arg: (void *)th);
867	if (status != 0 || !handle) { // ??? Why do we check handle??
868	#ifdef _POSIX_THREAD_ATTR_STACKSIZE
869	if (status == EINVAL) {
870	__kmp_fatal(KMP_MSG(CantSetWorkerStackSize, stack_size), KMP_ERR(status),
871	KMP_HNT(IncreaseWorkerStackSize), __kmp_msg_null);
872	}
873	if (status == ENOMEM) {
874	__kmp_fatal(KMP_MSG(CantSetWorkerStackSize, stack_size), KMP_ERR(status),
875	KMP_HNT(DecreaseWorkerStackSize), __kmp_msg_null);
876	}
877	#endif /* _POSIX_THREAD_ATTR_STACKSIZE */
878	if (status == EAGAIN) {
879	__kmp_fatal(KMP_MSG(NoResourcesForWorkerThread), KMP_ERR(status),
880	KMP_HNT(Decrease_NUM_THREADS), __kmp_msg_null);
881	}
882	KMP_SYSFAIL("pthread_create", status);
883	}
884
885	// Rename worker threads for improved debuggability
886	if (!KMP_UBER_GTID(gtid)) {
887	#if defined(LIBOMP_HAVE_PTHREAD_SET_NAME_NP)
888	pthread_set_name_np(handle, "openmp_worker");
889	#elif defined(LIBOMP_HAVE_PTHREAD_SETNAME_NP) && !KMP_OS_DARWIN
890	#if KMP_OS_NETBSD
891	pthread_setname_np(handle, "%s", const_cast<char *>("openmp_worker"));
892	#else
893	pthread_setname_np(target_thread: handle, name: "openmp_worker");
894	#endif
895	#endif
896	}
897
898	th->th.th_info.ds.ds_thread = handle;
899
900	#ifdef KMP_THREAD_ATTR
901	status = pthread_attr_destroy(attr: &thread_attr);
902	if (status) {
903	kmp_msg_t err_code = KMP_ERR(status);
904	__kmp_msg(kmp_ms_warning, KMP_MSG(CantDestroyThreadAttrs), err_code,
905	__kmp_msg_null);
906	if (__kmp_generate_warnings == kmp_warnings_off) {
907	__kmp_str_free(str: &err_code.str);
908	}
909	}
910	#endif /* KMP_THREAD_ATTR */
911
912	KMP_MB(); /* Flush all pending memory write invalidates. */
913
914	KA_TRACE(10, ("__kmp_create_worker: done creating thread (%d)\n", gtid));
915
916	} // __kmp_create_worker
917
918	#if KMP_USE_MONITOR
919	void __kmp_create_monitor(kmp_info_t *th) {
920	pthread_t handle;
921	pthread_attr_t thread_attr;
922	size_t size;
923	int status;
924	int auto_adj_size = FALSE;
925
926	if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) {
927	// We don't need monitor thread in case of MAX_BLOCKTIME
928	KA_TRACE(10, ("__kmp_create_monitor: skipping monitor thread because of "
929	"MAX blocktime\n"));
930	th->th.th_info.ds.ds_tid = 0; // this makes reap_monitor no-op
931	th->th.th_info.ds.ds_gtid = 0;
932	return;
933	}
934	KA_TRACE(10, ("__kmp_create_monitor: try to create monitor\n"));
935
936	KMP_MB(); /* Flush all pending memory write invalidates. */
937
938	th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR;
939	th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR;
940	#if KMP_REAL_TIME_FIX
941	TCW_4(__kmp_global.g.g_time.dt.t_value,
942	-1); // Will use it for synchronization a bit later.
943	#else
944	TCW_4(__kmp_global.g.g_time.dt.t_value, 0);
945	#endif // KMP_REAL_TIME_FIX
946
947	#ifdef KMP_THREAD_ATTR
948	if (__kmp_monitor_stksize == 0) {
949	__kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
950	auto_adj_size = TRUE;
951	}
952	status = pthread_attr_init(&thread_attr);
953	if (status != 0) {
954	__kmp_fatal(KMP_MSG(CantInitThreadAttrs), KMP_ERR(status), __kmp_msg_null);
955	}
956	status = pthread_attr_setdetachstate(&thread_attr, PTHREAD_CREATE_JOINABLE);
957	if (status != 0) {
958	__kmp_fatal(KMP_MSG(CantSetMonitorState), KMP_ERR(status), __kmp_msg_null);
959	}
960
961	#ifdef _POSIX_THREAD_ATTR_STACKSIZE
962	status = pthread_attr_getstacksize(&thread_attr, &size);
963	KMP_CHECK_SYSFAIL("pthread_attr_getstacksize", status);
964	#else
965	size = __kmp_sys_min_stksize;
966	#endif /* _POSIX_THREAD_ATTR_STACKSIZE */
967	#endif /* KMP_THREAD_ATTR */
968
969	if (__kmp_monitor_stksize == 0) {
970	__kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
971	}
972	if (__kmp_monitor_stksize < __kmp_sys_min_stksize) {
973	__kmp_monitor_stksize = __kmp_sys_min_stksize;
974	}
975
976	KA_TRACE(10, ("__kmp_create_monitor: default stacksize = %lu bytes,"
977	"requested stacksize = %lu bytes\n",
978	size, __kmp_monitor_stksize));
979
980	retry:
981
982	/* Set stack size for this thread now. */
983	#ifdef _POSIX_THREAD_ATTR_STACKSIZE
984	KA_TRACE(10, ("__kmp_create_monitor: setting stacksize = %lu bytes,",
985	__kmp_monitor_stksize));
986	status = pthread_attr_setstacksize(&thread_attr, __kmp_monitor_stksize);
987	if (status != 0) {
988	if (auto_adj_size) {
989	__kmp_monitor_stksize *= 2;
990	goto retry;
991	}
992	kmp_msg_t err_code = KMP_ERR(status);
993	__kmp_msg(kmp_ms_warning, // should this be fatal? BB
994	KMP_MSG(CantSetMonitorStackSize, (long int)__kmp_monitor_stksize),
995	err_code, KMP_HNT(ChangeMonitorStackSize), __kmp_msg_null);
996	if (__kmp_generate_warnings == kmp_warnings_off) {
997	__kmp_str_free(&err_code.str);
998	}
999	}
1000	#endif /* _POSIX_THREAD_ATTR_STACKSIZE */
1001
1002	status =
1003	pthread_create(&handle, &thread_attr, __kmp_launch_monitor, (void *)th);
1004
1005	if (status != 0) {
1006	#ifdef _POSIX_THREAD_ATTR_STACKSIZE
1007	if (status == EINVAL) {
1008	if (auto_adj_size && (__kmp_monitor_stksize < (size_t)0x40000000)) {
1009	__kmp_monitor_stksize *= 2;
1010	goto retry;
1011	}
1012	__kmp_fatal(KMP_MSG(CantSetMonitorStackSize, __kmp_monitor_stksize),
1013	KMP_ERR(status), KMP_HNT(IncreaseMonitorStackSize),
1014	__kmp_msg_null);
1015	}
1016	if (status == ENOMEM) {
1017	__kmp_fatal(KMP_MSG(CantSetMonitorStackSize, __kmp_monitor_stksize),
1018	KMP_ERR(status), KMP_HNT(DecreaseMonitorStackSize),
1019	__kmp_msg_null);
1020	}
1021	#endif /* _POSIX_THREAD_ATTR_STACKSIZE */
1022	if (status == EAGAIN) {
1023	__kmp_fatal(KMP_MSG(NoResourcesForMonitorThread), KMP_ERR(status),
1024	KMP_HNT(DecreaseNumberOfThreadsInUse), __kmp_msg_null);
1025	}
1026	KMP_SYSFAIL("pthread_create", status);
1027	}
1028
1029	th->th.th_info.ds.ds_thread = handle;
1030
1031	#if KMP_REAL_TIME_FIX
1032	// Wait for the monitor thread is really started and set its *priority*.
1033	KMP_DEBUG_ASSERT(sizeof(kmp_uint32) ==
1034	sizeof(__kmp_global.g.g_time.dt.t_value));
1035	__kmp_wait_4((kmp_uint32 volatile *)&__kmp_global.g.g_time.dt.t_value, -1,
1036	&__kmp_neq_4, NULL);
1037	#endif // KMP_REAL_TIME_FIX
1038
1039	#ifdef KMP_THREAD_ATTR
1040	status = pthread_attr_destroy(&thread_attr);
1041	if (status != 0) {
1042	kmp_msg_t err_code = KMP_ERR(status);
1043	__kmp_msg(kmp_ms_warning, KMP_MSG(CantDestroyThreadAttrs), err_code,
1044	__kmp_msg_null);
1045	if (__kmp_generate_warnings == kmp_warnings_off) {
1046	__kmp_str_free(&err_code.str);
1047	}
1048	}
1049	#endif
1050
1051	KMP_MB(); /* Flush all pending memory write invalidates. */
1052
1053	KA_TRACE(10, ("__kmp_create_monitor: monitor created %#.8lx\n",
1054	th->th.th_info.ds.ds_thread));
1055
1056	} // __kmp_create_monitor
1057	#endif // KMP_USE_MONITOR
1058
1059	void __kmp_exit_thread(int exit_status) {
1060	#if KMP_OS_WASI
1061	// TODO: the wasm32-wasi-threads target does not yet support pthread_exit.
1062	#else
1063	pthread_exit(retval: (void *)(intptr_t)exit_status);
1064	#endif
1065	} // __kmp_exit_thread
1066
1067	#if KMP_USE_MONITOR
1068	void __kmp_resume_monitor();
1069
1070	extern "C" void __kmp_reap_monitor(kmp_info_t *th) {
1071	int status;
1072	void *exit_val;
1073
1074	KA_TRACE(10, ("__kmp_reap_monitor: try to reap monitor thread with handle"
1075	" %#.8lx\n",
1076	th->th.th_info.ds.ds_thread));
1077
1078	// If monitor has been created, its tid and gtid should be KMP_GTID_MONITOR.
1079	// If both tid and gtid are 0, it means the monitor did not ever start.
1080	// If both tid and gtid are KMP_GTID_DNE, the monitor has been shut down.
1081	KMP_DEBUG_ASSERT(th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid);
1082	if (th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR) {
1083	KA_TRACE(10, ("__kmp_reap_monitor: monitor did not start, returning\n"));
1084	return;
1085	}
1086
1087	KMP_MB(); /* Flush all pending memory write invalidates. */
1088
1089	/* First, check to see whether the monitor thread exists to wake it up. This
1090	is to avoid performance problem when the monitor sleeps during
1091	blocktime-size interval */
1092
1093	status = pthread_kill(th->th.th_info.ds.ds_thread, 0);
1094	if (status != ESRCH) {
1095	__kmp_resume_monitor(); // Wake up the monitor thread
1096	}
1097	KA_TRACE(10, ("__kmp_reap_monitor: try to join with monitor\n"));
1098	status = pthread_join(th->th.th_info.ds.ds_thread, &exit_val);
1099	if (exit_val != th) {
1100	__kmp_fatal(KMP_MSG(ReapMonitorError), KMP_ERR(status), __kmp_msg_null);
1101	}
1102
1103	th->th.th_info.ds.ds_tid = KMP_GTID_DNE;
1104	th->th.th_info.ds.ds_gtid = KMP_GTID_DNE;
1105
1106	KA_TRACE(10, ("__kmp_reap_monitor: done reaping monitor thread with handle"
1107	" %#.8lx\n",
1108	th->th.th_info.ds.ds_thread));
1109
1110	KMP_MB(); /* Flush all pending memory write invalidates. */
1111	}
1112	#else
1113	// Empty symbol to export (see exports_so.txt) when
1114	// monitor thread feature is disabled
1115	extern "C" void __kmp_reap_monitor(kmp_info_t *th) { (void)th; }
1116	#endif // KMP_USE_MONITOR
1117
1118	void __kmp_reap_worker(kmp_info_t *th) {
1119	int status;
1120	void *exit_val;
1121
1122	KMP_MB(); /* Flush all pending memory write invalidates. */
1123
1124	KA_TRACE(
1125	10, ("__kmp_reap_worker: try to reap T#%d\n", th->th.th_info.ds.ds_gtid));
1126
1127	status = pthread_join(th: th->th.th_info.ds.ds_thread, thread_return: &exit_val);
1128	#ifdef KMP_DEBUG
1129	/* Don't expose these to the user until we understand when they trigger */
1130	if (status != 0) {
1131	__kmp_fatal(KMP_MSG(ReapWorkerError), KMP_ERR(status), __kmp_msg_null);
1132	}
1133	if (exit_val != th) {
1134	KA_TRACE(10, ("__kmp_reap_worker: worker T#%d did not reap properly, "
1135	"exit_val = %p\n",
1136	th->th.th_info.ds.ds_gtid, exit_val));
1137	}
1138	#else
1139	(void)status; // unused variable
1140	#endif /* KMP_DEBUG */
1141
1142	KA_TRACE(10, ("__kmp_reap_worker: done reaping T#%d\n",
1143	th->th.th_info.ds.ds_gtid));
1144
1145	KMP_MB(); /* Flush all pending memory write invalidates. */
1146	}
1147
1148	#if KMP_HANDLE_SIGNALS
1149
1150	static void __kmp_null_handler(int signo) {
1151	// Do nothing, for doing SIG_IGN-type actions.
1152	} // __kmp_null_handler
1153
1154	static void __kmp_team_handler(int signo) {
1155	if (__kmp_global.g.g_abort == 0) {
1156	/* Stage 1 signal handler, let's shut down all of the threads */
1157	#ifdef KMP_DEBUG
1158	__kmp_debug_printf(format: "__kmp_team_handler: caught signal = %d\n", signo);
1159	#endif
1160	switch (signo) {
1161	case SIGHUP:
1162	case SIGINT:
1163	case SIGQUIT:
1164	case SIGILL:
1165	case SIGABRT:
1166	case SIGFPE:
1167	case SIGBUS:
1168	case SIGSEGV:
1169	#ifdef SIGSYS
1170	case SIGSYS:
1171	#endif
1172	case SIGTERM:
1173	if (__kmp_debug_buf) {
1174	__kmp_dump_debug_buffer();
1175	}
1176	__kmp_unregister_library(); // cleanup shared memory
1177	KMP_MB(); // Flush all pending memory write invalidates.
1178	TCW_4(__kmp_global.g.g_abort, signo);
1179	KMP_MB(); // Flush all pending memory write invalidates.
1180	TCW_4(__kmp_global.g.g_done, TRUE);
1181	KMP_MB(); // Flush all pending memory write invalidates.
1182	break;
1183	default:
1184	#ifdef KMP_DEBUG
1185	__kmp_debug_printf(format: "__kmp_team_handler: unknown signal type");
1186	#endif
1187	break;
1188	}
1189	}
1190	} // __kmp_team_handler
1191
1192	static void __kmp_sigaction(int signum, const struct sigaction *act,
1193	struct sigaction *oldact) {
1194	int rc = sigaction(sig: signum, act: act, oact: oldact);
1195	KMP_CHECK_SYSFAIL_ERRNO("sigaction", rc);
1196	}
1197
1198	static void __kmp_install_one_handler(int sig, sig_func_t handler_func,
1199	int parallel_init) {
1200	KMP_MB(); // Flush all pending memory write invalidates.
1201	KB_TRACE(60,
1202	("__kmp_install_one_handler( %d, ..., %d )\n", sig, parallel_init));
1203	if (parallel_init) {
1204	struct sigaction new_action;
1205	struct sigaction old_action;
1206	new_action.sa_handler = handler_func;
1207	new_action.sa_flags = 0;
1208	sigfillset(set: &new_action.sa_mask);
1209	__kmp_sigaction(signum: sig, act: &new_action, oldact: &old_action);
1210	if (old_action.sa_handler == __kmp_sighldrs[sig].sa_handler) {
1211	sigaddset(set: &__kmp_sigset, signo: sig);
1212	} else {
1213	// Restore/keep user's handler if one previously installed.
1214	__kmp_sigaction(signum: sig, act: &old_action, NULL);
1215	}
1216	} else {
1217	// Save initial/system signal handlers to see if user handlers installed.
1218	__kmp_sigaction(signum: sig, NULL, oldact: &__kmp_sighldrs[sig]);
1219	}
1220	KMP_MB(); // Flush all pending memory write invalidates.
1221	} // __kmp_install_one_handler
1222
1223	static void __kmp_remove_one_handler(int sig) {
1224	KB_TRACE(60, ("__kmp_remove_one_handler( %d )\n", sig));
1225	if (sigismember(set: &__kmp_sigset, signo: sig)) {
1226	struct sigaction old;
1227	KMP_MB(); // Flush all pending memory write invalidates.
1228	__kmp_sigaction(signum: sig, act: &__kmp_sighldrs[sig], oldact: &old);
1229	if ((old.sa_handler != __kmp_team_handler) &&
1230	(old.sa_handler != __kmp_null_handler)) {
1231	// Restore the users signal handler.
1232	KB_TRACE(10, ("__kmp_remove_one_handler: oops, not our handler, "
1233	"restoring: sig=%d\n",
1234	sig));
1235	__kmp_sigaction(signum: sig, act: &old, NULL);
1236	}
1237	sigdelset(set: &__kmp_sigset, signo: sig);
1238	KMP_MB(); // Flush all pending memory write invalidates.
1239	}
1240	} // __kmp_remove_one_handler
1241
1242	void __kmp_install_signals(int parallel_init) {
1243	KB_TRACE(10, ("__kmp_install_signals( %d )\n", parallel_init));
1244	if (__kmp_handle_signals || !parallel_init) {
1245	// If ! parallel_init, we do not install handlers, just save original
1246	// handlers. Let us do it even __handle_signals is 0.
1247	sigemptyset(set: &__kmp_sigset);
1248	__kmp_install_one_handler(SIGHUP, handler_func: __kmp_team_handler, parallel_init);
1249	__kmp_install_one_handler(SIGINT, handler_func: __kmp_team_handler, parallel_init);
1250	__kmp_install_one_handler(SIGQUIT, handler_func: __kmp_team_handler, parallel_init);
1251	__kmp_install_one_handler(SIGILL, handler_func: __kmp_team_handler, parallel_init);
1252	__kmp_install_one_handler(SIGABRT, handler_func: __kmp_team_handler, parallel_init);
1253	__kmp_install_one_handler(SIGFPE, handler_func: __kmp_team_handler, parallel_init);
1254	__kmp_install_one_handler(SIGBUS, handler_func: __kmp_team_handler, parallel_init);
1255	__kmp_install_one_handler(SIGSEGV, handler_func: __kmp_team_handler, parallel_init);
1256	#ifdef SIGSYS
1257	__kmp_install_one_handler(SIGSYS, handler_func: __kmp_team_handler, parallel_init);
1258	#endif // SIGSYS
1259	__kmp_install_one_handler(SIGTERM, handler_func: __kmp_team_handler, parallel_init);
1260	#ifdef SIGPIPE
1261	__kmp_install_one_handler(SIGPIPE, handler_func: __kmp_team_handler, parallel_init);
1262	#endif // SIGPIPE
1263	}
1264	} // __kmp_install_signals
1265
1266	void __kmp_remove_signals(void) {
1267	int sig;
1268	KB_TRACE(10, ("__kmp_remove_signals()\n"));
1269	for (sig = 1; sig < NSIG; ++sig) {
1270	__kmp_remove_one_handler(sig);
1271	}
1272	} // __kmp_remove_signals
1273
1274	#endif // KMP_HANDLE_SIGNALS
1275
1276	void __kmp_enable(int new_state) {
1277	#ifdef KMP_CANCEL_THREADS
1278	int status, old_state;
1279	status = pthread_setcancelstate(state: new_state, oldstate: &old_state);
1280	KMP_CHECK_SYSFAIL("pthread_setcancelstate", status);
1281	KMP_DEBUG_ASSERT(old_state == PTHREAD_CANCEL_DISABLE);
1282	#endif
1283	}
1284
1285	void __kmp_disable(int *old_state) {
1286	#ifdef KMP_CANCEL_THREADS
1287	int status;
1288	status = pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, oldstate: old_state);
1289	KMP_CHECK_SYSFAIL("pthread_setcancelstate", status);
1290	#endif
1291	}
1292
1293	static void __kmp_atfork_prepare(void) {
1294	__kmp_acquire_bootstrap_lock(lck: &__kmp_initz_lock);
1295	__kmp_acquire_bootstrap_lock(lck: &__kmp_forkjoin_lock);
1296	}
1297
1298	static void __kmp_atfork_parent(void) {
1299	__kmp_release_bootstrap_lock(lck: &__kmp_forkjoin_lock);
1300	__kmp_release_bootstrap_lock(lck: &__kmp_initz_lock);
1301	}
1302
1303	/* Reset the library so execution in the child starts "all over again" with
1304	clean data structures in initial states. Don't worry about freeing memory
1305	allocated by parent, just abandon it to be safe. */
1306	static void __kmp_atfork_child(void) {
1307	__kmp_release_bootstrap_lock(lck: &__kmp_forkjoin_lock);
1308	__kmp_release_bootstrap_lock(lck: &__kmp_initz_lock);
1309	/* TODO make sure this is done right for nested/sibling */
1310	// ATT: Memory leaks are here? TODO: Check it and fix.
1311	/* KMP_ASSERT( 0 ); */
1312
1313	++__kmp_fork_count;
1314
1315	#if KMP_AFFINITY_SUPPORTED
1316	#if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY || \
1317	KMP_OS_AIX
1318	// reset the affinity in the child to the initial thread
1319	// affinity in the parent
1320	kmp_set_thread_affinity_mask_initial();
1321	#endif
1322	// Set default not to bind threads tightly in the child (we're expecting
1323	// over-subscription after the fork and this can improve things for
1324	// scripting languages that use OpenMP inside process-parallel code).
1325	if (__kmp_nested_proc_bind.bind_types != NULL) {
1326	__kmp_nested_proc_bind.bind_types[0] = proc_bind_false;
1327	}
1328	for (kmp_affinity_t *affinity : __kmp_affinities)
1329	*affinity = KMP_AFFINITY_INIT(affinity->env_var);
1330	__kmp_affin_fullMask = nullptr;
1331	__kmp_affin_origMask = nullptr;
1332	__kmp_topology = nullptr;
1333	#endif // KMP_AFFINITY_SUPPORTED
1334
1335	#if KMP_USE_MONITOR
1336	__kmp_init_monitor = 0;
1337	#endif
1338	__kmp_init_parallel = FALSE;
1339	__kmp_init_middle = FALSE;
1340	__kmp_init_serial = FALSE;
1341	TCW_4(__kmp_init_gtid, FALSE);
1342	__kmp_init_common = FALSE;
1343
1344	TCW_4(__kmp_init_user_locks, FALSE);
1345	#if !KMP_USE_DYNAMIC_LOCK
1346	__kmp_user_lock_table.used = 1;
1347	__kmp_user_lock_table.allocated = 0;
1348	__kmp_user_lock_table.table = NULL;
1349	__kmp_lock_blocks = NULL;
1350	#endif
1351
1352	__kmp_all_nth = 0;
1353	TCW_4(__kmp_nth, 0);
1354
1355	__kmp_thread_pool = NULL;
1356	__kmp_thread_pool_insert_pt = NULL;
1357	__kmp_team_pool = NULL;
1358
1359	/* Must actually zero all the *cache arguments passed to __kmpc_threadprivate
1360	here so threadprivate doesn't use stale data */
1361	KA_TRACE(10, ("__kmp_atfork_child: checking cache address list %p\n",
1362	__kmp_threadpriv_cache_list));
1363
1364	while (__kmp_threadpriv_cache_list != NULL) {
1365
1366	if (*__kmp_threadpriv_cache_list->addr != NULL) {
1367	KC_TRACE(50, ("__kmp_atfork_child: zeroing cache at address %p\n",
1368	&(*__kmp_threadpriv_cache_list->addr)));
1369
1370	*__kmp_threadpriv_cache_list->addr = NULL;
1371	}
1372	__kmp_threadpriv_cache_list = __kmp_threadpriv_cache_list->next;
1373	}
1374
1375	__kmp_init_runtime = FALSE;
1376
1377	/* reset statically initialized locks */
1378	__kmp_init_bootstrap_lock(lck: &__kmp_initz_lock);
1379	__kmp_init_bootstrap_lock(lck: &__kmp_stdio_lock);
1380	__kmp_init_bootstrap_lock(lck: &__kmp_console_lock);
1381	__kmp_init_bootstrap_lock(lck: &__kmp_task_team_lock);
1382
1383	#if USE_ITT_BUILD
1384	__kmp_itt_reset(); // reset ITT's global state
1385	#endif /* USE_ITT_BUILD */
1386
1387	{
1388	// Child process often get terminated without any use of OpenMP. That might
1389	// cause mapped shared memory file to be left unattended. Thus we postpone
1390	// library registration till middle initialization in the child process.
1391	__kmp_need_register_serial = FALSE;
1392	__kmp_serial_initialize();
1393	}
1394
1395	/* This is necessary to make sure no stale data is left around */
1396	/* AC: customers complain that we use unsafe routines in the atfork
1397	handler. Mathworks: dlsym() is unsafe. We call dlsym and dlopen
1398	in dynamic_link when check the presence of shared tbbmalloc library.
1399	Suggestion is to make the library initialization lazier, similar
1400	to what done for __kmpc_begin(). */
1401	// TODO: synchronize all static initializations with regular library
1402	// startup; look at kmp_global.cpp and etc.
1403	//__kmp_internal_begin ();
1404	}
1405
1406	void __kmp_register_atfork(void) {
1407	if (__kmp_need_register_atfork) {
1408	#if !KMP_OS_WASI
1409	int status = pthread_atfork(prepare: __kmp_atfork_prepare, parent: __kmp_atfork_parent,
1410	child: __kmp_atfork_child);
1411	KMP_CHECK_SYSFAIL("pthread_atfork", status);
1412	#endif
1413	__kmp_need_register_atfork = FALSE;
1414	}
1415	}
1416
1417	void __kmp_suspend_initialize(void) {
1418	int status;
1419	status = pthread_mutexattr_init(attr: &__kmp_suspend_mutex_attr);
1420	KMP_CHECK_SYSFAIL("pthread_mutexattr_init", status);
1421	status = pthread_condattr_init(attr: &__kmp_suspend_cond_attr);
1422	KMP_CHECK_SYSFAIL("pthread_condattr_init", status);
1423	}
1424
1425	void __kmp_suspend_initialize_thread(kmp_info_t *th) {
1426	int old_value = KMP_ATOMIC_LD_RLX(&th->th.th_suspend_init_count);
1427	int new_value = __kmp_fork_count + 1;
1428	// Return if already initialized
1429	if (old_value == new_value)
1430	return;
1431	// Wait, then return if being initialized
1432	if (old_value == -1 || !__kmp_atomic_compare_store(
1433	p: &th->th.th_suspend_init_count, expected: old_value, desired: -1)) {
1434	while (KMP_ATOMIC_LD_ACQ(&th->th.th_suspend_init_count) != new_value) {
1435	KMP_CPU_PAUSE();
1436	}
1437	} else {
1438	// Claim to be the initializer and do initializations
1439	int status;
1440	status = pthread_cond_init(cond: &th->th.th_suspend_cv.c_cond,
1441	cond_attr: &__kmp_suspend_cond_attr);
1442	KMP_CHECK_SYSFAIL("pthread_cond_init", status);
1443	status = pthread_mutex_init(mutex: &th->th.th_suspend_mx.m_mutex,
1444	mutexattr: &__kmp_suspend_mutex_attr);
1445	KMP_CHECK_SYSFAIL("pthread_mutex_init", status);
1446	KMP_ATOMIC_ST_REL(&th->th.th_suspend_init_count, new_value);
1447	}
1448	}
1449
1450	void __kmp_suspend_uninitialize_thread(kmp_info_t *th) {
1451	if (KMP_ATOMIC_LD_ACQ(&th->th.th_suspend_init_count) > __kmp_fork_count) {
1452	/* this means we have initialize the suspension pthread objects for this
1453	thread in this instance of the process */
1454	int status;
1455
1456	status = pthread_cond_destroy(cond: &th->th.th_suspend_cv.c_cond);
1457	if (status != 0 && status != EBUSY) {
1458	KMP_SYSFAIL("pthread_cond_destroy", status);
1459	}
1460	status = pthread_mutex_destroy(mutex: &th->th.th_suspend_mx.m_mutex);
1461	if (status != 0 && status != EBUSY) {
1462	KMP_SYSFAIL("pthread_mutex_destroy", status);
1463	}
1464	--th->th.th_suspend_init_count;
1465	KMP_DEBUG_ASSERT(KMP_ATOMIC_LD_RLX(&th->th.th_suspend_init_count) ==
1466	__kmp_fork_count);
1467	}
1468	}
1469
1470	// return true if lock obtained, false otherwise
1471	int __kmp_try_suspend_mx(kmp_info_t *th) {
1472	return (pthread_mutex_trylock(mutex: &th->th.th_suspend_mx.m_mutex) == 0);
1473	}
1474
1475	void __kmp_lock_suspend_mx(kmp_info_t *th) {
1476	int status = pthread_mutex_lock(mutex: &th->th.th_suspend_mx.m_mutex);
1477	KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
1478	}
1479
1480	void __kmp_unlock_suspend_mx(kmp_info_t *th) {
1481	int status = pthread_mutex_unlock(mutex: &th->th.th_suspend_mx.m_mutex);
1482	KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
1483	}
1484
1485	/* This routine puts the calling thread to sleep after setting the
1486	sleep bit for the indicated flag variable to true. */
1487	template <class C>
1488	static inline void __kmp_suspend_template(int th_gtid, C *flag) {
1489	KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_suspend);
1490	kmp_info_t *th = __kmp_threads[th_gtid];
1491	int status;
1492	typename C::flag_t old_spin;
1493
1494	KF_TRACE(30, ("__kmp_suspend_template: T#%d enter for flag = %p\n", th_gtid,
1495	flag->get()));
1496
1497	__kmp_suspend_initialize_thread(th);
1498
1499	__kmp_lock_suspend_mx(th);
1500
1501	KF_TRACE(10, ("__kmp_suspend_template: T#%d setting sleep bit for spin(%p)\n",
1502	th_gtid, flag->get()));
1503
1504	/* TODO: shouldn't this use release semantics to ensure that
1505	__kmp_suspend_initialize_thread gets called first? */
1506	old_spin = flag->set_sleeping();
1507	TCW_PTR(th->th.th_sleep_loc, (void *)flag);
1508	th->th.th_sleep_loc_type = flag->get_type();
1509	if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME &&
1510	__kmp_pause_status != kmp_soft_paused) {
1511	flag->unset_sleeping();
1512	TCW_PTR(th->th.th_sleep_loc, NULL);
1513	th->th.th_sleep_loc_type = flag_unset;
1514	__kmp_unlock_suspend_mx(th);
1515	return;
1516	}
1517	KF_TRACE(5, ("__kmp_suspend_template: T#%d set sleep bit for spin(%p)==%x,"
1518	" was %x\n",
1519	th_gtid, flag->get(), flag->load(), old_spin));
1520
1521	if (flag->done_check_val(old_spin) || flag->done_check()) {
1522	flag->unset_sleeping();
1523	TCW_PTR(th->th.th_sleep_loc, NULL);
1524	th->th.th_sleep_loc_type = flag_unset;
1525	KF_TRACE(5, ("__kmp_suspend_template: T#%d false alarm, reset sleep bit "
1526	"for spin(%p)\n",
1527	th_gtid, flag->get()));
1528	} else {
1529	/* Encapsulate in a loop as the documentation states that this may
1530	"with low probability" return when the condition variable has
1531	not been signaled or broadcast */
1532	int deactivated = FALSE;
1533
1534	while (flag->is_sleeping()) {
1535	#ifdef DEBUG_SUSPEND
1536	char buffer[128];
1537	__kmp_suspend_count++;
1538	__kmp_print_cond(buffer, &th->th.th_suspend_cv);
1539	__kmp_printf("__kmp_suspend_template: suspending T#%d: %s\n", th_gtid,
1540	buffer);
1541	#endif
1542	// Mark the thread as no longer active (only in the first iteration of the
1543	// loop).
1544	if (!deactivated) {
1545	th->th.th_active = FALSE;
1546	if (th->th.th_active_in_pool) {
1547	th->th.th_active_in_pool = FALSE;
1548	KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth);
1549	KMP_DEBUG_ASSERT(TCR_4(__kmp_thread_pool_active_nth) >= 0);
1550	}
1551	deactivated = TRUE;
1552	}
1553
1554	KMP_DEBUG_ASSERT(th->th.th_sleep_loc);
1555	KMP_DEBUG_ASSERT(flag->get_type() == th->th.th_sleep_loc_type);
1556
1557	#if USE_SUSPEND_TIMEOUT
1558	struct timespec now;
1559	struct timeval tval;
1560	int msecs;
1561
1562	status = gettimeofday(&tval, NULL);
1563	KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status);
1564	TIMEVAL_TO_TIMESPEC(&tval, &now);
1565
1566	msecs = (4 * __kmp_dflt_blocktime) + 200;
1567	now.tv_sec += msecs / 1000;
1568	now.tv_nsec += (msecs % 1000) * 1000;
1569
1570	KF_TRACE(15, ("__kmp_suspend_template: T#%d about to perform "
1571	"pthread_cond_timedwait\n",
1572	th_gtid));
1573	status = pthread_cond_timedwait(&th->th.th_suspend_cv.c_cond,
1574	&th->th.th_suspend_mx.m_mutex, &now);
1575	#else
1576	KF_TRACE(15, ("__kmp_suspend_template: T#%d about to perform"
1577	" pthread_cond_wait\n",
1578	th_gtid));
1579	status = pthread_cond_wait(cond: &th->th.th_suspend_cv.c_cond,
1580	mutex: &th->th.th_suspend_mx.m_mutex);
1581	#endif // USE_SUSPEND_TIMEOUT
1582
1583	if ((status != 0) && (status != EINTR) && (status != ETIMEDOUT)) {
1584	KMP_SYSFAIL("pthread_cond_wait", status);
1585	}
1586
1587	KMP_DEBUG_ASSERT(flag->get_type() == flag->get_ptr_type());
1588
1589	if (!flag->is_sleeping() &&
1590	((status == EINTR) || (status == ETIMEDOUT))) {
1591	// if interrupt or timeout, and thread is no longer sleeping, we need to
1592	// make sure sleep_loc gets reset; however, this shouldn't be needed if
1593	// we woke up with resume
1594	flag->unset_sleeping();
1595	TCW_PTR(th->th.th_sleep_loc, NULL);
1596	th->th.th_sleep_loc_type = flag_unset;
1597	}
1598	#ifdef KMP_DEBUG
1599	if (status == ETIMEDOUT) {
1600	if (flag->is_sleeping()) {
1601	KF_TRACE(100,
1602	("__kmp_suspend_template: T#%d timeout wakeup\n", th_gtid));
1603	} else {
1604	KF_TRACE(2, ("__kmp_suspend_template: T#%d timeout wakeup, sleep bit "
1605	"not set!\n",
1606	th_gtid));
1607	TCW_PTR(th->th.th_sleep_loc, NULL);
1608	th->th.th_sleep_loc_type = flag_unset;
1609	}
1610	} else if (flag->is_sleeping()) {
1611	KF_TRACE(100,
1612	("__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid));
1613	}
1614	#endif
1615	} // while
1616
1617	// Mark the thread as active again (if it was previous marked as inactive)
1618	if (deactivated) {
1619	th->th.th_active = TRUE;
1620	if (TCR_4(th->th.th_in_pool)) {
1621	KMP_ATOMIC_INC(&__kmp_thread_pool_active_nth);
1622	th->th.th_active_in_pool = TRUE;
1623	}
1624	}
1625	}
1626	// We may have had the loop variable set before entering the loop body;
1627	// so we need to reset sleep_loc.
1628	TCW_PTR(th->th.th_sleep_loc, NULL);
1629	th->th.th_sleep_loc_type = flag_unset;
1630
1631	KMP_DEBUG_ASSERT(!flag->is_sleeping());
1632	KMP_DEBUG_ASSERT(!th->th.th_sleep_loc);
1633	#ifdef DEBUG_SUSPEND
1634	{
1635	char buffer[128];
1636	__kmp_print_cond(buffer, &th->th.th_suspend_cv);
1637	__kmp_printf("__kmp_suspend_template: T#%d has awakened: %s\n", th_gtid,
1638	buffer);
1639	}
1640	#endif
1641
1642	__kmp_unlock_suspend_mx(th);
1643	KF_TRACE(30, ("__kmp_suspend_template: T#%d exit\n", th_gtid));
1644	}
1645
1646	template <bool C, bool S>
1647	void __kmp_suspend_32(int th_gtid, kmp_flag_32<C, S> *flag) {
1648	__kmp_suspend_template(th_gtid, flag);
1649	}
1650	template <bool C, bool S>
1651	void __kmp_suspend_64(int th_gtid, kmp_flag_64<C, S> *flag) {
1652	__kmp_suspend_template(th_gtid, flag);
1653	}
1654	template <bool C, bool S>
1655	void __kmp_atomic_suspend_64(int th_gtid, kmp_atomic_flag_64<C, S> *flag) {
1656	__kmp_suspend_template(th_gtid, flag);
1657	}
1658	void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag) {
1659	__kmp_suspend_template(th_gtid, flag);
1660	}
1661
1662	template void __kmp_suspend_32<false, false>(int, kmp_flag_32<false, false> *);
1663	template void __kmp_suspend_64<false, true>(int, kmp_flag_64<false, true> *);
1664	template void __kmp_suspend_64<true, false>(int, kmp_flag_64<true, false> *);
1665	template void
1666	__kmp_atomic_suspend_64<false, true>(int, kmp_atomic_flag_64<false, true> *);
1667	template void
1668	__kmp_atomic_suspend_64<true, false>(int, kmp_atomic_flag_64<true, false> *);
1669
1670	/* This routine signals the thread specified by target_gtid to wake up
1671	after setting the sleep bit indicated by the flag argument to FALSE.
1672	The target thread must already have called __kmp_suspend_template() */
1673	template <class C>
1674	static inline void __kmp_resume_template(int target_gtid, C *flag) {
1675	KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_resume);
1676	kmp_info_t *th = __kmp_threads[target_gtid];
1677	int status;
1678
1679	#ifdef KMP_DEBUG
1680	int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
1681	#endif
1682
1683	KF_TRACE(30, ("__kmp_resume_template: T#%d wants to wakeup T#%d enter\n",
1684	gtid, target_gtid));
1685	KMP_DEBUG_ASSERT(gtid != target_gtid);
1686
1687	__kmp_suspend_initialize_thread(th);
1688
1689	__kmp_lock_suspend_mx(th);
1690
1691	if (!flag || flag != th->th.th_sleep_loc) {
1692	// coming from __kmp_null_resume_wrapper, or thread is now sleeping on a
1693	// different location; wake up at new location
1694	flag = (C *)CCAST(void *, th->th.th_sleep_loc);
1695	}
1696
1697	// First, check if the flag is null or its type has changed. If so, someone
1698	// else woke it up.
1699	if (!flag) { // Thread doesn't appear to be sleeping on anything
1700	KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already "
1701	"awake: flag(%p)\n",
1702	gtid, target_gtid, (void *)NULL));
1703	__kmp_unlock_suspend_mx(th);
1704	return;
1705	} else if (flag->get_type() != th->th.th_sleep_loc_type) {
1706	// Flag type does not appear to match this function template; possibly the
1707	// thread is sleeping on something else. Try null resume again.
1708	KF_TRACE(
1709	5,
1710	("__kmp_resume_template: T#%d retrying, thread T#%d Mismatch flag(%p), "
1711	"spin(%p) type=%d ptr_type=%d\n",
1712	gtid, target_gtid, flag, flag->get(), flag->get_type(),
1713	th->th.th_sleep_loc_type));
1714	__kmp_unlock_suspend_mx(th);
1715	__kmp_null_resume_wrapper(thr: th);
1716	return;
1717	} else { // if multiple threads are sleeping, flag should be internally
1718	// referring to a specific thread here
1719	if (!flag->is_sleeping()) {
1720	KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already "
1721	"awake: flag(%p): %u\n",
1722	gtid, target_gtid, flag->get(), (unsigned int)flag->load()));
1723	__kmp_unlock_suspend_mx(th);
1724	return;
1725	}
1726	}
1727	KMP_DEBUG_ASSERT(flag);
1728	flag->unset_sleeping();
1729	TCW_PTR(th->th.th_sleep_loc, NULL);
1730	th->th.th_sleep_loc_type = flag_unset;
1731
1732	KF_TRACE(5, ("__kmp_resume_template: T#%d about to wakeup T#%d, reset "
1733	"sleep bit for flag's loc(%p): %u\n",
1734	gtid, target_gtid, flag->get(), (unsigned int)flag->load()));
1735
1736	#ifdef DEBUG_SUSPEND
1737	{
1738	char buffer[128];
1739	__kmp_print_cond(buffer, &th->th.th_suspend_cv);
1740	__kmp_printf("__kmp_resume_template: T#%d resuming T#%d: %s\n", gtid,
1741	target_gtid, buffer);
1742	}
1743	#endif
1744	status = pthread_cond_signal(cond: &th->th.th_suspend_cv.c_cond);
1745	KMP_CHECK_SYSFAIL("pthread_cond_signal", status);
1746	__kmp_unlock_suspend_mx(th);
1747	KF_TRACE(30, ("__kmp_resume_template: T#%d exiting after signaling wake up"
1748	" for T#%d\n",
1749	gtid, target_gtid));
1750	}
1751
1752	template <bool C, bool S>
1753	void __kmp_resume_32(int target_gtid, kmp_flag_32<C, S> *flag) {
1754	__kmp_resume_template(target_gtid, flag);
1755	}
1756	template <bool C, bool S>
1757	void __kmp_resume_64(int target_gtid, kmp_flag_64<C, S> *flag) {
1758	__kmp_resume_template(target_gtid, flag);
1759	}
1760	template <bool C, bool S>
1761	void __kmp_atomic_resume_64(int target_gtid, kmp_atomic_flag_64<C, S> *flag) {
1762	__kmp_resume_template(target_gtid, flag);
1763	}
1764	void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag) {
1765	__kmp_resume_template(target_gtid, flag);
1766	}
1767
1768	template void __kmp_resume_32<false, true>(int, kmp_flag_32<false, true> *);
1769	template void __kmp_resume_32<false, false>(int, kmp_flag_32<false, false> *);
1770	template void __kmp_resume_64<false, true>(int, kmp_flag_64<false, true> *);
1771	template void
1772	__kmp_atomic_resume_64<false, true>(int, kmp_atomic_flag_64<false, true> *);
1773
1774	#if KMP_USE_MONITOR
1775	void __kmp_resume_monitor() {
1776	KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_resume);
1777	int status;
1778	#ifdef KMP_DEBUG
1779	int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
1780	KF_TRACE(30, ("__kmp_resume_monitor: T#%d wants to wakeup T#%d enter\n", gtid,
1781	KMP_GTID_MONITOR));
1782	KMP_DEBUG_ASSERT(gtid != KMP_GTID_MONITOR);
1783	#endif
1784	status = pthread_mutex_lock(&__kmp_wait_mx.m_mutex);
1785	KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
1786	#ifdef DEBUG_SUSPEND
1787	{
1788	char buffer[128];
1789	__kmp_print_cond(buffer, &__kmp_wait_cv.c_cond);
1790	__kmp_printf("__kmp_resume_monitor: T#%d resuming T#%d: %s\n", gtid,
1791	KMP_GTID_MONITOR, buffer);
1792	}
1793	#endif
1794	status = pthread_cond_signal(&__kmp_wait_cv.c_cond);
1795	KMP_CHECK_SYSFAIL("pthread_cond_signal", status);
1796	status = pthread_mutex_unlock(&__kmp_wait_mx.m_mutex);
1797	KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
1798	KF_TRACE(30, ("__kmp_resume_monitor: T#%d exiting after signaling wake up"
1799	" for T#%d\n",
1800	gtid, KMP_GTID_MONITOR));
1801	}
1802	#endif // KMP_USE_MONITOR
1803
1804	void __kmp_yield() { sched_yield(); }
1805
1806	void __kmp_gtid_set_specific(int gtid) {
1807	if (__kmp_init_gtid) {
1808	int status;
1809	status = pthread_setspecific(key: __kmp_gtid_threadprivate_key,
1810	pointer: (void *)(intptr_t)(gtid + 1));
1811	KMP_CHECK_SYSFAIL("pthread_setspecific", status);
1812	} else {
1813	KA_TRACE(50, ("__kmp_gtid_set_specific: runtime shutdown, returning\n"));
1814	}
1815	}
1816
1817	int __kmp_gtid_get_specific() {
1818	int gtid;
1819	if (!__kmp_init_gtid) {
1820	KA_TRACE(50, ("__kmp_gtid_get_specific: runtime shutdown, returning "
1821	"KMP_GTID_SHUTDOWN\n"));
1822	return KMP_GTID_SHUTDOWN;
1823	}
1824	gtid = (int)(size_t)pthread_getspecific(key: __kmp_gtid_threadprivate_key);
1825	if (gtid == 0) {
1826	gtid = KMP_GTID_DNE;
1827	} else {
1828	gtid--;
1829	}
1830	KA_TRACE(50, ("__kmp_gtid_get_specific: key:%d gtid:%d\n",
1831	__kmp_gtid_threadprivate_key, gtid));
1832	return gtid;
1833	}
1834
1835	double __kmp_read_cpu_time(void) {
1836	/*clock_t t;*/
1837	struct tms buffer;
1838
1839	/*t =*/times(buffer: &buffer);
1840
1841	return (double)(buffer.tms_utime + buffer.tms_cutime) /
1842	(double)CLOCKS_PER_SEC;
1843	}
1844
1845	int __kmp_read_system_info(struct kmp_sys_info *info) {
1846	int status;
1847	struct rusage r_usage;
1848
1849	memset(s: info, c: 0, n: sizeof(*info));
1850
1851	status = getrusage(RUSAGE_SELF, usage: &r_usage);
1852	KMP_CHECK_SYSFAIL_ERRNO("getrusage", status);
1853
1854	#if !KMP_OS_WASI
1855	// The maximum resident set size utilized (in kilobytes)
1856	info->maxrss = r_usage.ru_maxrss;
1857	// The number of page faults serviced without any I/O
1858	info->minflt = r_usage.ru_minflt;
1859	// The number of page faults serviced that required I/O
1860	info->majflt = r_usage.ru_majflt;
1861	// The number of times a process was "swapped" out of memory
1862	info->nswap = r_usage.ru_nswap;
1863	// The number of times the file system had to perform input
1864	info->inblock = r_usage.ru_inblock;
1865	// The number of times the file system had to perform output
1866	info->oublock = r_usage.ru_oublock;
1867	// The number of times a context switch was voluntarily
1868	info->nvcsw = r_usage.ru_nvcsw;
1869	// The number of times a context switch was forced
1870	info->nivcsw = r_usage.ru_nivcsw;
1871	#endif
1872
1873	return (status != 0);
1874	}
1875
1876	void __kmp_read_system_time(double *delta) {
1877	double t_ns;
1878	struct timeval tval;
1879	struct timespec stop;
1880	int status;
1881
1882	status = gettimeofday(tv: &tval, NULL);
1883	KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status);
1884	TIMEVAL_TO_TIMESPEC(&tval, &stop);
1885	t_ns = (double)(TS2NS(stop) - TS2NS(__kmp_sys_timer_data.start));
1886	*delta = (t_ns * 1e-9);
1887	}
1888
1889	void __kmp_clear_system_time(void) {
1890	struct timeval tval;
1891	int status;
1892	status = gettimeofday(tv: &tval, NULL);
1893	KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status);
1894	TIMEVAL_TO_TIMESPEC(&tval, &__kmp_sys_timer_data.start);
1895	}
1896
1897	static int __kmp_get_xproc(void) {
1898
1899	int r = 0;
1900
1901	#if KMP_OS_LINUX
1902
1903	__kmp_type_convert(src: sysconf(_SC_NPROCESSORS_CONF), dest: &(r));
1904
1905	#elif KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_OPENBSD || \
1906	KMP_OS_HAIKU || KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_WASI || KMP_OS_AIX
1907
1908	__kmp_type_convert(sysconf(_SC_NPROCESSORS_ONLN), &(r));
1909
1910	#elif KMP_OS_DARWIN
1911
1912	size_t len = sizeof(r);
1913	sysctlbyname("hw.logicalcpu", &r, &len, NULL, 0);
1914
1915	#else
1916
1917	#error "Unknown or unsupported OS."
1918
1919	#endif
1920
1921	return r > 0 ? r : 2; /* guess value of 2 if OS told us 0 */
1922
1923	} // __kmp_get_xproc
1924
1925	int __kmp_read_from_file(char const *path, char const *format, ...) {
1926	int result;
1927	va_list args;
1928
1929	va_start(args, format);
1930	FILE *f = fopen(filename: path, modes: "rb");
1931	if (f == NULL) {
1932	va_end(args);
1933	return 0;
1934	}
1935	result = vfscanf(s: f, format: format, arg: args);
1936	fclose(stream: f);
1937	va_end(args);
1938
1939	return result;
1940	}
1941
1942	void __kmp_runtime_initialize(void) {
1943	int status;
1944	pthread_mutexattr_t mutex_attr;
1945	pthread_condattr_t cond_attr;
1946
1947	if (__kmp_init_runtime) {
1948	return;
1949	}
1950
1951	#if (KMP_ARCH_X86 || KMP_ARCH_X86_64)
1952	if (!__kmp_cpuinfo.initialized) {
1953	__kmp_query_cpuid(p: &__kmp_cpuinfo);
1954	}
1955	#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
1956
1957	__kmp_xproc = __kmp_get_xproc();
1958
1959	#if !KMP_32_BIT_ARCH
1960	struct rlimit rlim;
1961	// read stack size of calling thread, save it as default for worker threads;
1962	// this should be done before reading environment variables
1963	status = getrlimit(RLIMIT_STACK, rlimits: &rlim);
1964	if (status == 0) { // success?
1965	__kmp_stksize = rlim.rlim_cur;
1966	__kmp_check_stksize(val: &__kmp_stksize); // check value and adjust if needed
1967	}
1968	#endif /* KMP_32_BIT_ARCH */
1969
1970	if (sysconf(_SC_THREADS)) {
1971
1972	/* Query the maximum number of threads */
1973	__kmp_type_convert(src: sysconf(_SC_THREAD_THREADS_MAX), dest: &(__kmp_sys_max_nth));
1974	#ifdef __ve__
1975	if (__kmp_sys_max_nth == -1) {
1976	// VE's pthread supports only up to 64 threads per a VE process.
1977	// So we use that KMP_MAX_NTH (predefined as 64) here.
1978	__kmp_sys_max_nth = KMP_MAX_NTH;
1979	}
1980	#else
1981	if (__kmp_sys_max_nth == -1) {
1982	/* Unlimited threads for NPTL */
1983	__kmp_sys_max_nth = INT_MAX;
1984	} else if (__kmp_sys_max_nth <= 1) {
1985	/* Can't tell, just use PTHREAD_THREADS_MAX */
1986	__kmp_sys_max_nth = KMP_MAX_NTH;
1987	}
1988	#endif
1989
1990	/* Query the minimum stack size */
1991	__kmp_sys_min_stksize = sysconf(_SC_THREAD_STACK_MIN);
1992	if (__kmp_sys_min_stksize <= 1) {
1993	__kmp_sys_min_stksize = KMP_MIN_STKSIZE;
1994	}
1995	}
1996
1997	/* Set up minimum number of threads to switch to TLS gtid */
1998	__kmp_tls_gtid_min = KMP_TLS_GTID_MIN;
1999
2000	status = pthread_key_create(key: &__kmp_gtid_threadprivate_key,
2001	destr_function: __kmp_internal_end_dest);
2002	KMP_CHECK_SYSFAIL("pthread_key_create", status);
2003	status = pthread_mutexattr_init(attr: &mutex_attr);
2004	KMP_CHECK_SYSFAIL("pthread_mutexattr_init", status);
2005	status = pthread_mutex_init(mutex: &__kmp_wait_mx.m_mutex, mutexattr: &mutex_attr);
2006	KMP_CHECK_SYSFAIL("pthread_mutex_init", status);
2007	status = pthread_mutexattr_destroy(attr: &mutex_attr);
2008	KMP_CHECK_SYSFAIL("pthread_mutexattr_destroy", status);
2009	status = pthread_condattr_init(attr: &cond_attr);
2010	KMP_CHECK_SYSFAIL("pthread_condattr_init", status);
2011	status = pthread_cond_init(cond: &__kmp_wait_cv.c_cond, cond_attr: &cond_attr);
2012	KMP_CHECK_SYSFAIL("pthread_cond_init", status);
2013	status = pthread_condattr_destroy(attr: &cond_attr);
2014	KMP_CHECK_SYSFAIL("pthread_condattr_destroy", status);
2015	#if USE_ITT_BUILD
2016	__kmp_itt_initialize();
2017	#endif /* USE_ITT_BUILD */
2018
2019	__kmp_init_runtime = TRUE;
2020	}
2021
2022	void __kmp_runtime_destroy(void) {
2023	int status;
2024
2025	if (!__kmp_init_runtime) {
2026	return; // Nothing to do.
2027	}
2028
2029	#if USE_ITT_BUILD
2030	__kmp_itt_destroy();
2031	#endif /* USE_ITT_BUILD */
2032
2033	status = pthread_key_delete(key: __kmp_gtid_threadprivate_key);
2034	KMP_CHECK_SYSFAIL("pthread_key_delete", status);
2035
2036	status = pthread_mutex_destroy(mutex: &__kmp_wait_mx.m_mutex);
2037	if (status != 0 && status != EBUSY) {
2038	KMP_SYSFAIL("pthread_mutex_destroy", status);
2039	}
2040	status = pthread_cond_destroy(cond: &__kmp_wait_cv.c_cond);
2041	if (status != 0 && status != EBUSY) {
2042	KMP_SYSFAIL("pthread_cond_destroy", status);
2043	}
2044	#if KMP_AFFINITY_SUPPORTED
2045	__kmp_affinity_uninitialize();
2046	#endif
2047
2048	__kmp_init_runtime = FALSE;
2049	}
2050
2051	/* Put the thread to sleep for a time period */
2052	/* NOTE: not currently used anywhere */
2053	void __kmp_thread_sleep(int millis) { sleep(seconds: (millis + 500) / 1000); }
2054
2055	/* Calculate the elapsed wall clock time for the user */
2056	void __kmp_elapsed(double *t) {
2057	int status;
2058	#ifdef FIX_SGI_CLOCK
2059	struct timespec ts;
2060
2061	status = clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts);
2062	KMP_CHECK_SYSFAIL_ERRNO("clock_gettime", status);
2063	*t =
2064	(double)ts.tv_nsec * (1.0 / (double)KMP_NSEC_PER_SEC) + (double)ts.tv_sec;
2065	#else
2066	struct timeval tv;
2067
2068	status = gettimeofday(tv: &tv, NULL);
2069	KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status);
2070	*t =
2071	(double)tv.tv_usec * (1.0 / (double)KMP_USEC_PER_SEC) + (double)tv.tv_sec;
2072	#endif
2073	}
2074
2075	/* Calculate the elapsed wall clock tick for the user */
2076	void __kmp_elapsed_tick(double *t) { *t = 1 / (double)CLOCKS_PER_SEC; }
2077
2078	/* Return the current time stamp in nsec */
2079	kmp_uint64 __kmp_now_nsec() {
2080	struct timeval t;
2081	gettimeofday(tv: &t, NULL);
2082	kmp_uint64 nsec = (kmp_uint64)KMP_NSEC_PER_SEC * (kmp_uint64)t.tv_sec +
2083	(kmp_uint64)1000 * (kmp_uint64)t.tv_usec;
2084	return nsec;
2085	}
2086
2087	#if KMP_ARCH_X86 || KMP_ARCH_X86_64
2088	/* Measure clock ticks per millisecond */
2089	void __kmp_initialize_system_tick() {
2090	kmp_uint64 now, nsec2, diff;
2091	kmp_uint64 delay = 1000000; // ~450 usec on most machines.
2092	kmp_uint64 nsec = __kmp_now_nsec();
2093	kmp_uint64 goal = __kmp_hardware_timestamp() + delay;
2094	while ((now = __kmp_hardware_timestamp()) < goal)
2095	;
2096	nsec2 = __kmp_now_nsec();
2097	diff = nsec2 - nsec;
2098	if (diff > 0) {
2099	double tpus = 1000.0 * (double)(delay + (now - goal)) / (double)diff;
2100	if (tpus > 0.0) {
2101	__kmp_ticks_per_msec = (kmp_uint64)(tpus * 1000.0);
2102	__kmp_ticks_per_usec = (kmp_uint64)tpus;
2103	}
2104	}
2105	}
2106	#endif
2107
2108	/* Determine whether the given address is mapped into the current address
2109	space. */
2110
2111	int __kmp_is_address_mapped(void *addr) {
2112
2113	int found = 0;
2114	int rc;
2115
2116	#if KMP_OS_LINUX || KMP_OS_HURD
2117
2118	/* On GNUish OSes, read the /proc/<pid>/maps pseudo-file to get all the
2119	address ranges mapped into the address space. */
2120
2121	char *name = __kmp_str_format(format: "/proc/%d/maps", getpid());
2122	FILE *file = NULL;
2123
2124	file = fopen(filename: name, modes: "r");
2125	KMP_ASSERT(file != NULL);
2126
2127	for (;;) {
2128
2129	void *beginning = NULL;
2130	void *ending = NULL;
2131	char perms[5];
2132
2133	rc = fscanf(stream: file, format: "%p-%p %4s %*[^\n]\n", &beginning, &ending, perms);
2134	if (rc == EOF) {
2135	break;
2136	}
2137	KMP_ASSERT(rc == 3 &&
2138	KMP_STRLEN(perms) == 4); // Make sure all fields are read.
2139
2140	// Ending address is not included in the region, but beginning is.
2141	if ((addr >= beginning) && (addr < ending)) {
2142	perms[2] = 0; // 3th and 4th character does not matter.
2143	if (strcmp(s1: perms, s2: "rw") == 0) {
2144	// Memory we are looking for should be readable and writable.
2145	found = 1;
2146	}
2147	break;
2148	}
2149	}
2150
2151	// Free resources.
2152	fclose(stream: file);
2153	KMP_INTERNAL_FREE(name);
2154	#elif KMP_OS_FREEBSD
2155	char *buf;
2156	size_t lstsz;
2157	int mib[] = {CTL_KERN, KERN_PROC, KERN_PROC_VMMAP, getpid()};
2158	rc = sysctl(mib, 4, NULL, &lstsz, NULL, 0);
2159	if (rc < 0)
2160	return 0;
2161	// We pass from number of vm entry's semantic
2162	// to size of whole entry map list.
2163	lstsz = lstsz * 4 / 3;
2164	buf = reinterpret_cast<char *>(KMP_INTERNAL_MALLOC(lstsz));
2165	rc = sysctl(mib, 4, buf, &lstsz, NULL, 0);
2166	if (rc < 0) {
2167	KMP_INTERNAL_FREE(buf);
2168	return 0;
2169	}
2170
2171	char *lw = buf;
2172	char *up = buf + lstsz;
2173
2174	while (lw < up) {
2175	struct kinfo_vmentry *cur = reinterpret_cast<struct kinfo_vmentry *>(lw);
2176	size_t cursz = cur->kve_structsize;
2177	if (cursz == 0)
2178	break;
2179	void *start = reinterpret_cast<void *>(cur->kve_start);
2180	void *end = reinterpret_cast<void *>(cur->kve_end);
2181	// Readable/Writable addresses within current map entry
2182	if ((addr >= start) && (addr < end)) {
2183	if ((cur->kve_protection & KVME_PROT_READ) != 0 &&
2184	(cur->kve_protection & KVME_PROT_WRITE) != 0) {
2185	found = 1;
2186	break;
2187	}
2188	}
2189	lw += cursz;
2190	}
2191	KMP_INTERNAL_FREE(buf);
2192	#elif KMP_OS_DRAGONFLY
2193	char err[_POSIX2_LINE_MAX];
2194	kinfo_proc *proc;
2195	vmspace sp;
2196	vm_map *cur;
2197	vm_map_entry entry, *c;
2198	struct proc p;
2199	kvm_t *fd;
2200	uintptr_t uaddr;
2201	int num;
2202
2203	fd = kvm_openfiles(nullptr, nullptr, nullptr, O_RDONLY, err);
2204	if (!fd) {
2205	return 0;
2206	}
2207
2208	proc = kvm_getprocs(fd, KERN_PROC_PID, getpid(), &num);
2209
2210	if (kvm_read(fd, static_cast<uintptr_t>(proc->kp_paddr), &p, sizeof(p)) !=
2211	sizeof(p) ||
2212	kvm_read(fd, reinterpret_cast<uintptr_t>(p.p_vmspace), &sp, sizeof(sp)) !=
2213	sizeof(sp)) {
2214	kvm_close(fd);
2215	return 0;
2216	}
2217
2218	(void)rc;
2219	cur = &sp.vm_map;
2220	uaddr = reinterpret_cast<uintptr_t>(addr);
2221	for (c = kvm_vm_map_entry_first(fd, cur, &entry); c;
2222	c = kvm_vm_map_entry_next(fd, c, &entry)) {
2223	if ((uaddr >= entry.ba.start) && (uaddr <= entry.ba.end)) {
2224	if ((entry.protection & VM_PROT_READ) != 0 &&
2225	(entry.protection & VM_PROT_WRITE) != 0) {
2226	found = 1;
2227	break;
2228	}
2229	}
2230	}
2231
2232	kvm_close(fd);
2233	#elif KMP_OS_SOLARIS
2234	prxmap_t *cur, *map;
2235	void *buf;
2236	uintptr_t uaddr;
2237	ssize_t rd;
2238	int fd;
2239	pid_t pid = getpid();
2240	char *name = __kmp_str_format("/proc/%d/xmap", pid);
2241	fd = open(name, O_RDONLY);
2242	if (fd == -1) {
2243	KMP_INTERNAL_FREE(name);
2244	return 0;
2245	}
2246
2247	size_t sz = (1 << 20);
2248	buf = KMP_INTERNAL_MALLOC(sz);
2249
2250	while (sz > 0 && (rd = pread(fd, buf, sz, 0)) == sz) {
2251	void *newbuf;
2252	sz <<= 1;
2253	newbuf = KMP_INTERNAL_REALLOC(buf, sz);
2254	buf = newbuf;
2255	}
2256
2257	map = reinterpret_cast<prxmap_t *>(buf);
2258	uaddr = reinterpret_cast<uintptr_t>(addr);
2259
2260	for (cur = map; rd > 0; cur++, rd = -sizeof(*map)) {
2261	if (uaddr >= cur->pr_vaddr && uaddr < cur->pr_vaddr) {
2262	if ((cur->pr_mflags & MA_READ) != 0 && (cur->pr_mflags & MA_WRITE) != 0) {
2263	found = 1;
2264	break;
2265	}
2266	}
2267	}
2268
2269	KMP_INTERNAL_FREE(map);
2270	close(fd);
2271	KMP_INTERNAL_FREE(name);
2272	#elif KMP_OS_DARWIN
2273
2274	/* On OS X*, /proc pseudo filesystem is not available. Try to read memory
2275	using vm interface. */
2276
2277	int buffer;
2278	vm_size_t count;
2279	rc = vm_read_overwrite(
2280	mach_task_self(), // Task to read memory of.
2281	(vm_address_t)(addr), // Address to read from.
2282	1, // Number of bytes to be read.
2283	(vm_address_t)(&buffer), // Address of buffer to save read bytes in.
2284	&count // Address of var to save number of read bytes in.
2285	);
2286	if (rc == 0) {
2287	// Memory successfully read.
2288	found = 1;
2289	}
2290
2291	#elif KMP_OS_NETBSD
2292
2293	int mib[5];
2294	mib[0] = CTL_VM;
2295	mib[1] = VM_PROC;
2296	mib[2] = VM_PROC_MAP;
2297	mib[3] = getpid();
2298	mib[4] = sizeof(struct kinfo_vmentry);
2299
2300	size_t size;
2301	rc = sysctl(mib, __arraycount(mib), NULL, &size, NULL, 0);
2302	KMP_ASSERT(!rc);
2303	KMP_ASSERT(size);
2304
2305	size = size * 4 / 3;
2306	struct kinfo_vmentry *kiv = (struct kinfo_vmentry *)KMP_INTERNAL_MALLOC(size);
2307	KMP_ASSERT(kiv);
2308
2309	rc = sysctl(mib, __arraycount(mib), kiv, &size, NULL, 0);
2310	KMP_ASSERT(!rc);
2311	KMP_ASSERT(size);
2312
2313	for (size_t i = 0; i < size; i++) {
2314	if (kiv[i].kve_start >= (uint64_t)addr &&
2315	kiv[i].kve_end <= (uint64_t)addr) {
2316	found = 1;
2317	break;
2318	}
2319	}
2320	KMP_INTERNAL_FREE(kiv);
2321	#elif KMP_OS_OPENBSD
2322
2323	int mib[3];
2324	mib[0] = CTL_KERN;
2325	mib[1] = KERN_PROC_VMMAP;
2326	mib[2] = getpid();
2327
2328	size_t size;
2329	uint64_t end;
2330	rc = sysctl(mib, 3, NULL, &size, NULL, 0);
2331	KMP_ASSERT(!rc);
2332	KMP_ASSERT(size);
2333	end = size;
2334
2335	struct kinfo_vmentry kiv = {.kve_start = 0};
2336
2337	while ((rc = sysctl(mib, 3, &kiv, &size, NULL, 0)) == 0) {
2338	KMP_ASSERT(size);
2339	if (kiv.kve_end == end)
2340	break;
2341
2342	if (kiv.kve_start >= (uint64_t)addr && kiv.kve_end <= (uint64_t)addr) {
2343	found = 1;
2344	break;
2345	}
2346	kiv.kve_start += 1;
2347	}
2348	#elif KMP_OS_WASI
2349	found = (int)addr < (__builtin_wasm_memory_size(0) * PAGESIZE);
2350	#elif KMP_OS_AIX
2351
2352	uint32_t loadQueryBufSize = 4096u; // Default loadquery buffer size.
2353	char *loadQueryBuf;
2354
2355	for (;;) {
2356	loadQueryBuf = (char *)KMP_INTERNAL_MALLOC(loadQueryBufSize);
2357	if (loadQueryBuf == NULL) {
2358	return 0;
2359	}
2360
2361	rc = loadquery(L_GETXINFO | L_IGNOREUNLOAD, loadQueryBuf, loadQueryBufSize);
2362	if (rc < 0) {
2363	KMP_INTERNAL_FREE(loadQueryBuf);
2364	if (errno != ENOMEM) {
2365	return 0;
2366	}
2367	// errno == ENOMEM; double the size.
2368	loadQueryBufSize <<= 1;
2369	continue;
2370	}
2371	// Obtained the load info successfully.
2372	break;
2373	}
2374
2375	struct ld_xinfo *curLdInfo = (struct ld_xinfo *)loadQueryBuf;
2376
2377	// Loop through the load info to find if there is a match.
2378	for (;;) {
2379	uintptr_t curDataStart = (uintptr_t)curLdInfo->ldinfo_dataorg;
2380	uintptr_t curDataEnd = curDataStart + curLdInfo->ldinfo_datasize;
2381
2382	// The data segment is readable and writable.
2383	if (curDataStart <= (uintptr_t)addr && (uintptr_t)addr < curDataEnd) {
2384	found = 1;
2385	break;
2386	}
2387	if (curLdInfo->ldinfo_next == 0u) {
2388	// Reached the end of load info.
2389	break;
2390	}
2391	curLdInfo = (struct ld_xinfo *)((char *)curLdInfo + curLdInfo->ldinfo_next);
2392	}
2393	KMP_INTERNAL_FREE(loadQueryBuf);
2394
2395	#elif KMP_OS_HAIKU
2396
2397	found = 1;
2398	#else
2399
2400	#error "Unknown or unsupported OS"
2401
2402	#endif
2403
2404	return found;
2405
2406	} // __kmp_is_address_mapped
2407
2408	#ifdef USE_LOAD_BALANCE
2409
2410	#if KMP_OS_DARWIN || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
2411	KMP_OS_OPENBSD || KMP_OS_SOLARIS
2412
2413	// The function returns the rounded value of the system load average
2414	// during given time interval which depends on the value of
2415	// __kmp_load_balance_interval variable (default is 60 sec, other values
2416	// may be 300 sec or 900 sec).
2417	// It returns -1 in case of error.
2418	int __kmp_get_load_balance(int max) {
2419	double averages[3];
2420	int ret_avg = 0;
2421
2422	int res = getloadavg(averages, 3);
2423
2424	// Check __kmp_load_balance_interval to determine which of averages to use.
2425	// getloadavg() may return the number of samples less than requested that is
2426	// less than 3.
2427	if (__kmp_load_balance_interval < 180 && (res >= 1)) {
2428	ret_avg = (int)averages[0]; // 1 min
2429	} else if ((__kmp_load_balance_interval >= 180 &&
2430	__kmp_load_balance_interval < 600) &&
2431	(res >= 2)) {
2432	ret_avg = (int)averages[1]; // 5 min
2433	} else if ((__kmp_load_balance_interval >= 600) && (res == 3)) {
2434	ret_avg = (int)averages[2]; // 15 min
2435	} else { // Error occurred
2436	return -1;
2437	}
2438
2439	return ret_avg;
2440	}
2441
2442	#elif KMP_OS_AIX
2443
2444	// The function returns number of running (not sleeping) threads, or -1 in case
2445	// of error.
2446	int __kmp_get_load_balance(int max) {
2447
2448	static int glb_running_threads = 0; // Saved count of the running threads for
2449	// the thread balance algorithm.
2450	static double glb_call_time = 0; // Thread balance algorithm call time.
2451	int running_threads = 0; // Number of running threads in the system.
2452
2453	double call_time = 0.0;
2454
2455	__kmp_elapsed(&call_time);
2456
2457	if (glb_call_time &&
2458	(call_time - glb_call_time < __kmp_load_balance_interval))
2459	return glb_running_threads;
2460
2461	glb_call_time = call_time;
2462
2463	if (max <= 0) {
2464	max = INT_MAX;
2465	}
2466
2467	// Check how many perfstat_cpu_t structures are available.
2468	int logical_cpus = perfstat_cpu(NULL, NULL, sizeof(perfstat_cpu_t), 0);
2469	if (logical_cpus <= 0) {
2470	glb_call_time = -1;
2471	return -1;
2472	}
2473
2474	perfstat_cpu_t *cpu_stat = (perfstat_cpu_t *)KMP_INTERNAL_MALLOC(
2475	logical_cpus * sizeof(perfstat_cpu_t));
2476	if (cpu_stat == NULL) {
2477	glb_call_time = -1;
2478	return -1;
2479	}
2480
2481	// Set first CPU as the name of the first logical CPU for which the info is
2482	// desired.
2483	perfstat_id_t first_cpu_name;
2484	strcpy(first_cpu_name.name, FIRST_CPU);
2485
2486	// Get the stat info of logical CPUs.
2487	int rc = perfstat_cpu(&first_cpu_name, cpu_stat, sizeof(perfstat_cpu_t),
2488	logical_cpus);
2489	KMP_DEBUG_ASSERT(rc == logical_cpus);
2490	if (rc <= 0) {
2491	KMP_INTERNAL_FREE(cpu_stat);
2492	glb_call_time = -1;
2493	return -1;
2494	}
2495	for (int i = 0; i < logical_cpus; ++i) {
2496	running_threads += cpu_stat[i].runque;
2497	if (running_threads >= max)
2498	break;
2499	}
2500
2501	// There _might_ be a timing hole where the thread executing this
2502	// code gets skipped in the load balance, and running_threads is 0.
2503	// Assert in the debug builds only!!!
2504	KMP_DEBUG_ASSERT(running_threads > 0);
2505	if (running_threads <= 0)
2506	running_threads = 1;
2507
2508	KMP_INTERNAL_FREE(cpu_stat);
2509
2510	glb_running_threads = running_threads;
2511
2512	return running_threads;
2513	}
2514
2515	#else // Linux* OS
2516
2517	// The function returns number of running (not sleeping) threads, or -1 in case
2518	// of error. Error could be reported if Linux* OS kernel too old (without
2519	// "/proc" support). Counting running threads stops if max running threads
2520	// encountered.
2521	int __kmp_get_load_balance(int max) {
2522	static int permanent_error = 0;
2523	static int glb_running_threads = 0; // Saved count of the running threads for
2524	// the thread balance algorithm
2525	static double glb_call_time = 0; /* Thread balance algorithm call time */
2526
2527	int running_threads = 0; // Number of running threads in the system.
2528
2529	DIR *proc_dir = NULL; // Handle of "/proc/" directory.
2530	struct dirent *proc_entry = NULL;
2531
2532	kmp_str_buf_t task_path; // "/proc/<pid>/task/<tid>/" path.
2533	DIR *task_dir = NULL; // Handle of "/proc/<pid>/task/<tid>/" directory.
2534	struct dirent *task_entry = NULL;
2535	int task_path_fixed_len;
2536
2537	kmp_str_buf_t stat_path; // "/proc/<pid>/task/<tid>/stat" path.
2538	int stat_file = -1;
2539	int stat_path_fixed_len;
2540
2541	#ifdef KMP_DEBUG
2542	int total_processes = 0; // Total number of processes in system.
2543	#endif
2544
2545	double call_time = 0.0;
2546
2547	__kmp_str_buf_init(&task_path);
2548	__kmp_str_buf_init(&stat_path);
2549
2550	__kmp_elapsed(t: &call_time);
2551
2552	if (glb_call_time &&
2553	(call_time - glb_call_time < __kmp_load_balance_interval)) {
2554	running_threads = glb_running_threads;
2555	goto finish;
2556	}
2557
2558	glb_call_time = call_time;
2559
2560	// Do not spend time on scanning "/proc/" if we have a permanent error.
2561	if (permanent_error) {
2562	running_threads = -1;
2563	goto finish;
2564	}
2565
2566	if (max <= 0) {
2567	max = INT_MAX;
2568	}
2569
2570	// Open "/proc/" directory.
2571	proc_dir = opendir(name: "/proc");
2572	if (proc_dir == NULL) {
2573	// Cannot open "/proc/". Probably the kernel does not support it. Return an
2574	// error now and in subsequent calls.
2575	running_threads = -1;
2576	permanent_error = 1;
2577	goto finish;
2578	}
2579
2580	// Initialize fixed part of task_path. This part will not change.
2581	__kmp_str_buf_cat(buffer: &task_path, str: "/proc/", len: 6);
2582	task_path_fixed_len = task_path.used; // Remember number of used characters.
2583
2584	proc_entry = readdir(dirp: proc_dir);
2585	while (proc_entry != NULL) {
2586	// Proc entry is a directory and name starts with a digit. Assume it is a
2587	// process' directory.
2588	if (proc_entry->d_type == DT_DIR && isdigit(proc_entry->d_name[0])) {
2589
2590	#ifdef KMP_DEBUG
2591	++total_processes;
2592	#endif
2593	// Make sure init process is the very first in "/proc", so we can replace
2594	// strcmp( proc_entry->d_name, "1" ) == 0 with simpler total_processes ==
2595	// 1. We are going to check that total_processes == 1 => d_name == "1" is
2596	// true (where "=>" is implication). Since C++ does not have => operator,
2597	// let us replace it with its equivalent: a => b == ! a || b.
2598	KMP_DEBUG_ASSERT(total_processes != 1 ||
2599	strcmp(proc_entry->d_name, "1") == 0);
2600
2601	// Construct task_path.
2602	task_path.used = task_path_fixed_len; // Reset task_path to "/proc/".
2603	__kmp_str_buf_cat(buffer: &task_path, str: proc_entry->d_name,
2604	KMP_STRLEN(s: proc_entry->d_name));
2605	__kmp_str_buf_cat(buffer: &task_path, str: "/task", len: 5);
2606
2607	task_dir = opendir(name: task_path.str);
2608	if (task_dir == NULL) {
2609	// Process can finish between reading "/proc/" directory entry and
2610	// opening process' "task/" directory. So, in general case we should not
2611	// complain, but have to skip this process and read the next one. But on
2612	// systems with no "task/" support we will spend lot of time to scan
2613	// "/proc/" tree again and again without any benefit. "init" process
2614	// (its pid is 1) should exist always, so, if we cannot open
2615	// "/proc/1/task/" directory, it means "task/" is not supported by
2616	// kernel. Report an error now and in the future.
2617	if (strcmp(s1: proc_entry->d_name, s2: "1") == 0) {
2618	running_threads = -1;
2619	permanent_error = 1;
2620	goto finish;
2621	}
2622	} else {
2623	// Construct fixed part of stat file path.
2624	__kmp_str_buf_clear(buffer: &stat_path);
2625	__kmp_str_buf_cat(buffer: &stat_path, str: task_path.str, len: task_path.used);
2626	__kmp_str_buf_cat(buffer: &stat_path, str: "/", len: 1);
2627	stat_path_fixed_len = stat_path.used;
2628
2629	task_entry = readdir(dirp: task_dir);
2630	while (task_entry != NULL) {
2631	// It is a directory and name starts with a digit.
2632	if (proc_entry->d_type == DT_DIR && isdigit(task_entry->d_name[0])) {
2633
2634	// Construct complete stat file path. Easiest way would be:
2635	// __kmp_str_buf_print( & stat_path, "%s/%s/stat", task_path.str,
2636	// task_entry->d_name );
2637	// but seriae of __kmp_str_buf_cat works a bit faster.
2638	stat_path.used =
2639	stat_path_fixed_len; // Reset stat path to its fixed part.
2640	__kmp_str_buf_cat(buffer: &stat_path, str: task_entry->d_name,
2641	KMP_STRLEN(s: task_entry->d_name));
2642	__kmp_str_buf_cat(buffer: &stat_path, str: "/stat", len: 5);
2643
2644	// Note: Low-level API (open/read/close) is used. High-level API
2645	// (fopen/fclose) works ~ 30 % slower.
2646	stat_file = open(file: stat_path.str, O_RDONLY);
2647	if (stat_file == -1) {
2648	// We cannot report an error because task (thread) can terminate
2649	// just before reading this file.
2650	} else {
2651	/* Content of "stat" file looks like:
2652	24285 (program) S ...
2653
2654	It is a single line (if program name does not include funny
2655	symbols). First number is a thread id, then name of executable
2656	file name in paretheses, then state of the thread. We need just
2657	thread state.
2658
2659	Good news: Length of program name is 15 characters max. Longer
2660	names are truncated.
2661
2662	Thus, we need rather short buffer: 15 chars for program name +
2663	2 parenthesis, + 3 spaces + ~7 digits of pid = 37.
2664
2665	Bad news: Program name may contain special symbols like space,
2666	closing parenthesis, or even new line. This makes parsing
2667	"stat" file not 100 % reliable. In case of fanny program names
2668	parsing may fail (report incorrect thread state).
2669
2670	Parsing "status" file looks more promissing (due to different
2671	file structure and escaping special symbols) but reading and
2672	parsing of "status" file works slower.
2673	-- ln
2674	*/
2675	char buffer[65];
2676	ssize_t len;
2677	len = read(fd: stat_file, buf: buffer, nbytes: sizeof(buffer) - 1);
2678	if (len >= 0) {
2679	buffer[len] = 0;
2680	// Using scanf:
2681	// sscanf( buffer, "%*d (%*s) %c ", & state );
2682	// looks very nice, but searching for a closing parenthesis
2683	// works a bit faster.
2684	char *close_parent = strstr(haystack: buffer, needle: ") ");
2685	if (close_parent != NULL) {
2686	char state = *(close_parent + 2);
2687	if (state == 'R') {
2688	++running_threads;
2689	if (running_threads >= max) {
2690	goto finish;
2691	}
2692	}
2693	}
2694	}
2695	close(fd: stat_file);
2696	stat_file = -1;
2697	}
2698	}
2699	task_entry = readdir(dirp: task_dir);
2700	}
2701	closedir(dirp: task_dir);
2702	task_dir = NULL;
2703	}
2704	}
2705	proc_entry = readdir(dirp: proc_dir);
2706	}
2707
2708	// There _might_ be a timing hole where the thread executing this
2709	// code get skipped in the load balance, and running_threads is 0.
2710	// Assert in the debug builds only!!!
2711	KMP_DEBUG_ASSERT(running_threads > 0);
2712	if (running_threads <= 0) {
2713	running_threads = 1;
2714	}
2715
2716	finish: // Clean up and exit.
2717	if (proc_dir != NULL) {
2718	closedir(dirp: proc_dir);
2719	}
2720	__kmp_str_buf_free(buffer: &task_path);
2721	if (task_dir != NULL) {
2722	closedir(dirp: task_dir);
2723	}
2724	__kmp_str_buf_free(buffer: &stat_path);
2725	if (stat_file != -1) {
2726	close(fd: stat_file);
2727	}
2728
2729	glb_running_threads = running_threads;
2730
2731	return running_threads;
2732
2733	} // __kmp_get_load_balance
2734
2735	#endif // KMP_OS_DARWIN
2736
2737	#endif // USE_LOAD_BALANCE
2738
2739	#if !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_MIC || \
2740	((KMP_OS_LINUX || KMP_OS_DARWIN) && KMP_ARCH_AARCH64) || \
2741	KMP_ARCH_PPC64 || KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 || \
2742	KMP_ARCH_ARM || KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_PPC_XCOFF || \
2743	KMP_ARCH_AARCH64_32)
2744
2745	// Because WebAssembly will use `call_indirect` to invoke the microtask and
2746	// WebAssembly indirect calls check that the called signature is a precise
2747	// match, we need to cast each microtask function pointer back from `void *` to
2748	// its original type.
2749	typedef void (*microtask_t0)(int *, int *);
2750	typedef void (*microtask_t1)(int *, int *, void *);
2751	typedef void (*microtask_t2)(int *, int *, void *, void *);
2752	typedef void (*microtask_t3)(int *, int *, void *, void *, void *);
2753	typedef void (*microtask_t4)(int *, int *, void *, void *, void *, void *);
2754	typedef void (*microtask_t5)(int *, int *, void *, void *, void *, void *,
2755	void *);
2756	typedef void (*microtask_t6)(int *, int *, void *, void *, void *, void *,
2757	void *, void *);
2758	typedef void (*microtask_t7)(int *, int *, void *, void *, void *, void *,
2759	void *, void *, void *);
2760	typedef void (*microtask_t8)(int *, int *, void *, void *, void *, void *,
2761	void *, void *, void *, void *);
2762	typedef void (*microtask_t9)(int *, int *, void *, void *, void *, void *,
2763	void *, void *, void *, void *, void *);
2764	typedef void (*microtask_t10)(int *, int *, void *, void *, void *, void *,
2765	void *, void *, void *, void *, void *, void *);
2766	typedef void (*microtask_t11)(int *, int *, void *, void *, void *, void *,
2767	void *, void *, void *, void *, void *, void *,
2768	void *);
2769	typedef void (*microtask_t12)(int *, int *, void *, void *, void *, void *,
2770	void *, void *, void *, void *, void *, void *,
2771	void *, void *);
2772	typedef void (*microtask_t13)(int *, int *, void *, void *, void *, void *,
2773	void *, void *, void *, void *, void *, void *,
2774	void *, void *, void *);
2775	typedef void (*microtask_t14)(int *, int *, void *, void *, void *, void *,
2776	void *, void *, void *, void *, void *, void *,
2777	void *, void *, void *, void *);
2778	typedef void (*microtask_t15)(int *, int *, void *, void *, void *, void *,
2779	void *, void *, void *, void *, void *, void *,
2780	void *, void *, void *, void *, void *);
2781
2782	// we really only need the case with 1 argument, because CLANG always build
2783	// a struct of pointers to shared variables referenced in the outlined function
2784	int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc,
2785	void *p_argv[]
2786	#if OMPT_SUPPORT
2787	,
2788	void **exit_frame_ptr
2789	#endif
2790	) {
2791	#if OMPT_SUPPORT
2792	*exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);
2793	#endif
2794
2795	switch (argc) {
2796	default:
2797	fprintf(stderr, "Too many args to microtask: %d!\n", argc);
2798	fflush(stderr);
2799	exit(-1);
2800	case 0:
2801	(*(microtask_t0)pkfn)(&gtid, &tid);
2802	break;
2803	case 1:
2804	(*(microtask_t1)pkfn)(&gtid, &tid, p_argv[0]);
2805	break;
2806	case 2:
2807	(*(microtask_t2)pkfn)(&gtid, &tid, p_argv[0], p_argv[1]);
2808	break;
2809	case 3:
2810	(*(microtask_t3)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2]);
2811	break;
2812	case 4:
2813	(*(microtask_t4)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2814	p_argv[3]);
2815	break;
2816	case 5:
2817	(*(microtask_t5)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2818	p_argv[3], p_argv[4]);
2819	break;
2820	case 6:
2821	(*(microtask_t6)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2822	p_argv[3], p_argv[4], p_argv[5]);
2823	break;
2824	case 7:
2825	(*(microtask_t7)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2826	p_argv[3], p_argv[4], p_argv[5], p_argv[6]);
2827	break;
2828	case 8:
2829	(*(microtask_t8)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2830	p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2831	p_argv[7]);
2832	break;
2833	case 9:
2834	(*(microtask_t9)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2835	p_argv[3], p_argv[4], p_argv[5], p_argv[6], p_argv[7],
2836	p_argv[8]);
2837	break;
2838	case 10:
2839	(*(microtask_t10)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2840	p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2841	p_argv[7], p_argv[8], p_argv[9]);
2842	break;
2843	case 11:
2844	(*(microtask_t11)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2845	p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2846	p_argv[7], p_argv[8], p_argv[9], p_argv[10]);
2847	break;
2848	case 12:
2849	(*(microtask_t12)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2850	p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2851	p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2852	p_argv[11]);
2853	break;
2854	case 13:
2855	(*(microtask_t13)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2856	p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2857	p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2858	p_argv[11], p_argv[12]);
2859	break;
2860	case 14:
2861	(*(microtask_t14)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2862	p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2863	p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2864	p_argv[11], p_argv[12], p_argv[13]);
2865	break;
2866	case 15:
2867	(*(microtask_t15)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2868	p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2869	p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2870	p_argv[11], p_argv[12], p_argv[13], p_argv[14]);
2871	break;
2872	}
2873
2874	return 1;
2875	}
2876
2877	#endif
2878
2879	#if KMP_OS_LINUX
2880	// Functions for hidden helper task
2881	namespace {
2882	// Condition variable for initializing hidden helper team
2883	pthread_cond_t hidden_helper_threads_initz_cond_var;
2884	pthread_mutex_t hidden_helper_threads_initz_lock;
2885	volatile int hidden_helper_initz_signaled = FALSE;
2886
2887	// Condition variable for deinitializing hidden helper team
2888	pthread_cond_t hidden_helper_threads_deinitz_cond_var;
2889	pthread_mutex_t hidden_helper_threads_deinitz_lock;
2890	volatile int hidden_helper_deinitz_signaled = FALSE;
2891
2892	// Condition variable for the wrapper function of main thread
2893	pthread_cond_t hidden_helper_main_thread_cond_var;
2894	pthread_mutex_t hidden_helper_main_thread_lock;
2895	volatile int hidden_helper_main_thread_signaled = FALSE;
2896
2897	// Semaphore for worker threads. We don't use condition variable here in case
2898	// that when multiple signals are sent at the same time, only one thread might
2899	// be waken.
2900	sem_t hidden_helper_task_sem;
2901	} // namespace
2902
2903	void __kmp_hidden_helper_worker_thread_wait() {
2904	int status = sem_wait(sem: &hidden_helper_task_sem);
2905	KMP_CHECK_SYSFAIL("sem_wait", status);
2906	}
2907
2908	void __kmp_do_initialize_hidden_helper_threads() {
2909	// Initialize condition variable
2910	int status =
2911	pthread_cond_init(cond: &hidden_helper_threads_initz_cond_var, cond_attr: nullptr);
2912	KMP_CHECK_SYSFAIL("pthread_cond_init", status);
2913
2914	status = pthread_cond_init(cond: &hidden_helper_threads_deinitz_cond_var, cond_attr: nullptr);
2915	KMP_CHECK_SYSFAIL("pthread_cond_init", status);
2916
2917	status = pthread_cond_init(cond: &hidden_helper_main_thread_cond_var, cond_attr: nullptr);
2918	KMP_CHECK_SYSFAIL("pthread_cond_init", status);
2919
2920	status = pthread_mutex_init(mutex: &hidden_helper_threads_initz_lock, mutexattr: nullptr);
2921	KMP_CHECK_SYSFAIL("pthread_mutex_init", status);
2922
2923	status = pthread_mutex_init(mutex: &hidden_helper_threads_deinitz_lock, mutexattr: nullptr);
2924	KMP_CHECK_SYSFAIL("pthread_mutex_init", status);
2925
2926	status = pthread_mutex_init(mutex: &hidden_helper_main_thread_lock, mutexattr: nullptr);
2927	KMP_CHECK_SYSFAIL("pthread_mutex_init", status);
2928
2929	// Initialize the semaphore
2930	status = sem_init(sem: &hidden_helper_task_sem, pshared: 0, value: 0);
2931	KMP_CHECK_SYSFAIL("sem_init", status);
2932
2933	// Create a new thread to finish initialization
2934	pthread_t handle;
2935	status = pthread_create(
2936	newthread: &handle, attr: nullptr,
2937	start_routine: [](void *) -> void * {
2938	__kmp_hidden_helper_threads_initz_routine();
2939	return nullptr;
2940	},
2941	arg: nullptr);
2942	KMP_CHECK_SYSFAIL("pthread_create", status);
2943	}
2944
2945	void __kmp_hidden_helper_threads_initz_wait() {
2946	// Initial thread waits here for the completion of the initialization. The
2947	// condition variable will be notified by main thread of hidden helper teams.
2948	int status = pthread_mutex_lock(mutex: &hidden_helper_threads_initz_lock);
2949	KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
2950
2951	if (!TCR_4(hidden_helper_initz_signaled)) {
2952	status = pthread_cond_wait(cond: &hidden_helper_threads_initz_cond_var,
2953	mutex: &hidden_helper_threads_initz_lock);
2954	KMP_CHECK_SYSFAIL("pthread_cond_wait", status);
2955	}
2956
2957	status = pthread_mutex_unlock(mutex: &hidden_helper_threads_initz_lock);
2958	KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
2959	}
2960
2961	void __kmp_hidden_helper_initz_release() {
2962	// After all initialization, reset __kmp_init_hidden_helper_threads to false.
2963	int status = pthread_mutex_lock(mutex: &hidden_helper_threads_initz_lock);
2964	KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
2965
2966	status = pthread_cond_signal(cond: &hidden_helper_threads_initz_cond_var);
2967	KMP_CHECK_SYSFAIL("pthread_cond_wait", status);
2968
2969	TCW_SYNC_4(hidden_helper_initz_signaled, TRUE);
2970
2971	status = pthread_mutex_unlock(mutex: &hidden_helper_threads_initz_lock);
2972	KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
2973	}
2974
2975	void __kmp_hidden_helper_main_thread_wait() {
2976	// The main thread of hidden helper team will be blocked here. The
2977	// condition variable can only be signal in the destructor of RTL.
2978	int status = pthread_mutex_lock(mutex: &hidden_helper_main_thread_lock);
2979	KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
2980
2981	if (!TCR_4(hidden_helper_main_thread_signaled)) {
2982	status = pthread_cond_wait(cond: &hidden_helper_main_thread_cond_var,
2983	mutex: &hidden_helper_main_thread_lock);
2984	KMP_CHECK_SYSFAIL("pthread_cond_wait", status);
2985	}
2986
2987	status = pthread_mutex_unlock(mutex: &hidden_helper_main_thread_lock);
2988	KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
2989	}
2990
2991	void __kmp_hidden_helper_main_thread_release() {
2992	// The initial thread of OpenMP RTL should call this function to wake up the
2993	// main thread of hidden helper team.
2994	int status = pthread_mutex_lock(mutex: &hidden_helper_main_thread_lock);
2995	KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
2996
2997	status = pthread_cond_signal(cond: &hidden_helper_main_thread_cond_var);
2998	KMP_CHECK_SYSFAIL("pthread_cond_signal", status);
2999
3000	// The hidden helper team is done here
3001	TCW_SYNC_4(hidden_helper_main_thread_signaled, TRUE);
3002
3003	status = pthread_mutex_unlock(mutex: &hidden_helper_main_thread_lock);
3004	KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
3005	}
3006
3007	void __kmp_hidden_helper_worker_thread_signal() {
3008	int status = sem_post(sem: &hidden_helper_task_sem);
3009	KMP_CHECK_SYSFAIL("sem_post", status);
3010	}
3011
3012	void __kmp_hidden_helper_threads_deinitz_wait() {
3013	// Initial thread waits here for the completion of the deinitialization. The
3014	// condition variable will be notified by main thread of hidden helper teams.
3015	int status = pthread_mutex_lock(mutex: &hidden_helper_threads_deinitz_lock);
3016	KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
3017
3018	if (!TCR_4(hidden_helper_deinitz_signaled)) {
3019	status = pthread_cond_wait(cond: &hidden_helper_threads_deinitz_cond_var,
3020	mutex: &hidden_helper_threads_deinitz_lock);
3021	KMP_CHECK_SYSFAIL("pthread_cond_wait", status);
3022	}
3023
3024	status = pthread_mutex_unlock(mutex: &hidden_helper_threads_deinitz_lock);
3025	KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
3026	}
3027
3028	void __kmp_hidden_helper_threads_deinitz_release() {
3029	int status = pthread_mutex_lock(mutex: &hidden_helper_threads_deinitz_lock);
3030	KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
3031
3032	status = pthread_cond_signal(cond: &hidden_helper_threads_deinitz_cond_var);
3033	KMP_CHECK_SYSFAIL("pthread_cond_wait", status);
3034
3035	TCW_SYNC_4(hidden_helper_deinitz_signaled, TRUE);
3036
3037	status = pthread_mutex_unlock(mutex: &hidden_helper_threads_deinitz_lock);
3038	KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
3039	}
3040	#else // KMP_OS_LINUX
3041	void __kmp_hidden_helper_worker_thread_wait() {
3042	KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3043	}
3044
3045	void __kmp_do_initialize_hidden_helper_threads() {
3046	KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3047	}
3048
3049	void __kmp_hidden_helper_threads_initz_wait() {
3050	KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3051	}
3052
3053	void __kmp_hidden_helper_initz_release() {
3054	KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3055	}
3056
3057	void __kmp_hidden_helper_main_thread_wait() {
3058	KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3059	}
3060
3061	void __kmp_hidden_helper_main_thread_release() {
3062	KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3063	}
3064
3065	void __kmp_hidden_helper_worker_thread_signal() {
3066	KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3067	}
3068
3069	void __kmp_hidden_helper_threads_deinitz_wait() {
3070	KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3071	}
3072
3073	void __kmp_hidden_helper_threads_deinitz_release() {
3074	KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3075	}
3076	#endif // KMP_OS_LINUX
3077
3078	bool __kmp_detect_shm() {
3079	DIR *dir = opendir(name: "/dev/shm");
3080	if (dir) { // /dev/shm exists
3081	closedir(dirp: dir);
3082	return true;
3083	} else if (ENOENT == errno) { // /dev/shm does not exist
3084	return false;
3085	} else { // opendir() failed
3086	return false;
3087	}
3088	}
3089
3090	bool __kmp_detect_tmp() {
3091	DIR *dir = opendir(name: "/tmp");
3092	if (dir) { // /tmp exists
3093	closedir(dirp: dir);
3094	return true;
3095	} else if (ENOENT == errno) { // /tmp does not exist
3096	return false;
3097	} else { // opendir() failed
3098	return false;
3099	}
3100	}
3101
3102	// end of file //
3103