1// RUN: %libomp-compile && env LIBOMP_NUM_HIDDEN_HELPER_THREADS=0 OMP_PROC_BIND=close OMP_PLACES=cores KMP_AFFINITY=verbose %libomp-run 8 1 4
2// REQUIRES: linux
3//
4// This test pthread_creates 8 root threads before any OpenMP
5// runtime entry is ever called. We have all the root threads
6// register with the runtime by calling omp_set_num_threads(),
7// but this does not initialize their affinity. The fourth root thread
8// then calls a parallel region and we make sure its affinity
9// is correct. We also make sure all the other root threads are
10// free-floating since they have not called into a parallel region.
11
12#define _GNU_SOURCE
13#include <stdio.h>
14#include <stdlib.h>
15#include <omp.h>
16#include <pthread.h>
17#include <unistd.h>
18#include <assert.h>
19#include <sys/types.h>
20#include <sys/syscall.h>
21#include "libomp_test_affinity.h"
22
23volatile int entry_flag = 0;
24volatile int flag = 0;
25volatile int num_roots_arrived = 0;
26int num_roots;
27int spawner = 0;
28pthread_mutex_t lock;
29int register_workers = 0; // boolean
30affinity_mask_t *full_mask;
31
32int __kmpc_global_thread_num(void*);
33
34int get_os_thread_id() {
35 return (int)syscall(SYS_gettid);
36}
37
38int place_and_affinity_match() {
39 int i, max_cpu;
40 char buf[512];
41 affinity_mask_t *mask = affinity_mask_alloc();
42 int place = omp_get_place_num();
43 int num_procs = omp_get_place_num_procs(place);
44 int *ids = (int*)malloc(size: sizeof(int) * num_procs);
45 omp_get_place_proc_ids(place, ids);
46 get_thread_affinity(mask);
47 affinity_mask_snprintf(buf, bufsize: sizeof(buf), mask);
48 printf(format: "Primary Thread Place: %d\n", place);
49 printf(format: "Primary Thread mask: %s\n", buf);
50
51 for (i = 0; i < num_procs; ++i) {
52 int cpu = ids[i];
53 if (!affinity_mask_isset(mask, cpu))
54 return 0;
55 }
56
57 max_cpu = AFFINITY_MAX_CPUS;
58 for (i = 0; i < max_cpu; ++i) {
59 int cpu = i;
60 if (affinity_mask_isset(mask, cpu)) {
61 int j, found = 0;
62 for (j = 0; j < num_procs; ++j) {
63 if (ids[j] == cpu) {
64 found = 1;
65 break;
66 }
67 }
68 if (!found)
69 return 0;
70 }
71 }
72
73 affinity_mask_free(mask);
74 free(ptr: ids);
75 return 1;
76}
77
78void* thread_func(void *arg) {
79 int place, nplaces;
80 int root_id = *((int*)arg);
81 int pid = getpid();
82 int tid = get_os_thread_id();
83
84 // Order how the root threads are assigned a gtid in the runtime
85 // i.e., root_id = gtid
86 while (1) {
87 int v = entry_flag;
88 if (v == root_id)
89 break;
90 }
91
92 // If main root thread
93 if (root_id == spawner) {
94 printf(format: "Initial application thread (pid=%d, tid=%d, spawner=%d) reached thread_func (will call OpenMP)\n", pid, tid, spawner);
95 omp_set_num_threads(4);
96 #pragma omp atomic
97 entry_flag++;
98 // Wait for the workers to signal their arrival before #pragma omp parallel
99 while (num_roots_arrived < num_roots - 1) {}
100 // This will trigger the output for KMP_AFFINITY in this case
101 #pragma omp parallel
102 {
103 int gtid = __kmpc_global_thread_num(NULL);
104 #pragma omp single
105 {
106 printf(format: "Exactly %d threads in the #pragma omp parallel\n",
107 omp_get_num_threads());
108 }
109 #pragma omp critical
110 {
111 printf(format: "OpenMP thread %d: gtid=%d\n", omp_get_thread_num(), gtid);
112 }
113 }
114 flag = 1;
115 if (!place_and_affinity_match()) {
116 fprintf(stderr, format: "error: place and affinity mask do not match for primary thread\n");
117 exit (EXIT_FAILURE);
118 }
119
120 } else { // If worker root thread
121 // Worker root threads, register with OpenMP through omp_set_num_threads()
122 // if designated to, signal their arrival and then wait for the main root
123 // thread to signal them to exit.
124 printf(format: "New root pthread (pid=%d, tid=%d) reached thread_func\n", pid, tid);
125 if (register_workers)
126 omp_set_num_threads(4);
127 #pragma omp atomic
128 entry_flag++;
129
130 pthread_mutex_lock(mutex: &lock);
131 num_roots_arrived++;
132 pthread_mutex_unlock(mutex: &lock);
133 while (flag == 0) {}
134
135 // Main check whether root threads' mask is equal to the
136 // initial affinity mask
137 affinity_mask_t *mask = affinity_mask_alloc();
138 get_thread_affinity(mask);
139 if (!affinity_mask_equal(mask1: mask, mask2: full_mask)) {
140 char buf[1024];
141 printf(format: "root thread %d mask: ", root_id);
142 affinity_mask_snprintf(buf, bufsize: sizeof(buf), mask);
143 printf(format: "initial affinity mask: %s\n", buf);
144 fprintf(stderr, format: "error: root thread %d affinity mask not equal"
145 " to initial full mask\n", root_id);
146 affinity_mask_free(mask);
147 exit(EXIT_FAILURE);
148 }
149 affinity_mask_free(mask);
150 }
151 return NULL;
152}
153
154int main(int argc, char** argv) {
155 int i;
156 if (argc != 3 && argc != 4) {
157 fprintf(stderr, format: "usage: %s <num_roots> <register_workers_bool> [<spawn_root_number>]\n", argv[0]);
158 exit(EXIT_FAILURE);
159 }
160
161 // Initialize pthread mutex
162 pthread_mutex_init(mutex: &lock, NULL);
163
164 // Get initial full mask
165 full_mask = affinity_mask_alloc();
166 get_thread_affinity(mask: full_mask);
167
168 // Get the number of root pthreads to create and allocate resources for them
169 num_roots = atoi(nptr: argv[1]);
170 pthread_t *roots = (pthread_t*)malloc(size: sizeof(pthread_t) * num_roots);
171 int *root_ids = (int*)malloc(size: sizeof(int) * num_roots);
172
173 // Get the flag indicating whether to have root pthreads call omp_set_num_threads() or not
174 register_workers = atoi(nptr: argv[2]);
175
176 if (argc == 4)
177 spawner = atoi(nptr: argv[3]);
178
179 // Spawn worker root threads
180 for (i = 1; i < num_roots; ++i) {
181 *(root_ids + i) = i;
182 pthread_create(newthread: roots + i, NULL, start_routine: thread_func, arg: root_ids + i);
183 }
184 // Have main root thread (root 0) go into thread_func
185 *root_ids = 0;
186 thread_func(arg: root_ids);
187
188 // Cleanup all resources
189 for (i = 1; i < num_roots; ++i) {
190 void *status;
191 pthread_join(th: roots[i], thread_return: &status);
192 }
193 free(ptr: roots);
194 free(ptr: root_ids);
195 pthread_mutex_destroy(mutex: &lock);
196 return EXIT_SUCCESS;
197}
198

source code of openmp/runtime/test/affinity/root-threads-affinity.c