1 | /* SPDX-License-Identifier: LGPL-2.1 OR MIT */ |
2 | /* |
3 | * rseq.h |
4 | * |
5 | * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> |
6 | */ |
7 | |
8 | #ifndef RSEQ_H |
9 | #define RSEQ_H |
10 | |
11 | #include <stdint.h> |
12 | #include <stdbool.h> |
13 | #include <pthread.h> |
14 | #include <signal.h> |
15 | #include <sched.h> |
16 | #include <errno.h> |
17 | #include <stdio.h> |
18 | #include <stdlib.h> |
19 | #include <stddef.h> |
20 | #include "rseq-abi.h" |
21 | #include "compiler.h" |
22 | |
23 | #ifndef rseq_sizeof_field |
24 | #define rseq_sizeof_field(TYPE, MEMBER) sizeof((((TYPE *)0)->MEMBER)) |
25 | #endif |
26 | |
27 | #ifndef rseq_offsetofend |
28 | #define rseq_offsetofend(TYPE, MEMBER) \ |
29 | (offsetof(TYPE, MEMBER) + rseq_sizeof_field(TYPE, MEMBER)) |
30 | #endif |
31 | |
32 | /* |
33 | * Empty code injection macros, override when testing. |
34 | * It is important to consider that the ASM injection macros need to be |
35 | * fully reentrant (e.g. do not modify the stack). |
36 | */ |
37 | #ifndef RSEQ_INJECT_ASM |
38 | #define RSEQ_INJECT_ASM(n) |
39 | #endif |
40 | |
41 | #ifndef RSEQ_INJECT_C |
42 | #define RSEQ_INJECT_C(n) |
43 | #endif |
44 | |
45 | #ifndef RSEQ_INJECT_INPUT |
46 | #define RSEQ_INJECT_INPUT |
47 | #endif |
48 | |
49 | #ifndef RSEQ_INJECT_CLOBBER |
50 | #define RSEQ_INJECT_CLOBBER |
51 | #endif |
52 | |
53 | #ifndef RSEQ_INJECT_FAILED |
54 | #define RSEQ_INJECT_FAILED |
55 | #endif |
56 | |
57 | #include "rseq-thread-pointer.h" |
58 | |
59 | /* Offset from the thread pointer to the rseq area. */ |
60 | extern ptrdiff_t rseq_offset; |
61 | |
62 | /* |
63 | * Size of the registered rseq area. 0 if the registration was |
64 | * unsuccessful. |
65 | */ |
66 | extern unsigned int rseq_size; |
67 | |
68 | /* Flags used during rseq registration. */ |
69 | extern unsigned int rseq_flags; |
70 | |
71 | /* |
72 | * rseq feature size supported by the kernel. 0 if the registration was |
73 | * unsuccessful. |
74 | */ |
75 | extern unsigned int rseq_feature_size; |
76 | |
77 | enum rseq_mo { |
78 | RSEQ_MO_RELAXED = 0, |
79 | RSEQ_MO_CONSUME = 1, /* Unused */ |
80 | RSEQ_MO_ACQUIRE = 2, /* Unused */ |
81 | RSEQ_MO_RELEASE = 3, |
82 | RSEQ_MO_ACQ_REL = 4, /* Unused */ |
83 | RSEQ_MO_SEQ_CST = 5, /* Unused */ |
84 | }; |
85 | |
86 | enum rseq_percpu_mode { |
87 | RSEQ_PERCPU_CPU_ID = 0, |
88 | RSEQ_PERCPU_MM_CID = 1, |
89 | }; |
90 | |
91 | static inline struct rseq_abi *rseq_get_abi(void) |
92 | { |
93 | return (struct rseq_abi *) ((uintptr_t) rseq_thread_pointer() + rseq_offset); |
94 | } |
95 | |
96 | #define rseq_likely(x) __builtin_expect(!!(x), 1) |
97 | #define rseq_unlikely(x) __builtin_expect(!!(x), 0) |
98 | #define rseq_barrier() __asm__ __volatile__("" : : : "memory") |
99 | |
100 | #define RSEQ_ACCESS_ONCE(x) (*(__volatile__ __typeof__(x) *)&(x)) |
101 | #define RSEQ_WRITE_ONCE(x, v) __extension__ ({ RSEQ_ACCESS_ONCE(x) = (v); }) |
102 | #define RSEQ_READ_ONCE(x) RSEQ_ACCESS_ONCE(x) |
103 | |
104 | #define __rseq_str_1(x) #x |
105 | #define __rseq_str(x) __rseq_str_1(x) |
106 | |
107 | #define rseq_log(fmt, args...) \ |
108 | fprintf(stderr, fmt "(in %s() at " __FILE__ ":" __rseq_str(__LINE__)"\n", \ |
109 | ## args, __func__) |
110 | |
111 | #define rseq_bug(fmt, args...) \ |
112 | do { \ |
113 | rseq_log(fmt, ##args); \ |
114 | abort(); \ |
115 | } while (0) |
116 | |
117 | #if defined(__x86_64__) || defined(__i386__) |
118 | #include <rseq-x86.h> |
119 | #elif defined(__ARMEL__) |
120 | #include <rseq-arm.h> |
121 | #elif defined (__AARCH64EL__) |
122 | #include <rseq-arm64.h> |
123 | #elif defined(__PPC__) |
124 | #include <rseq-ppc.h> |
125 | #elif defined(__mips__) |
126 | #include <rseq-mips.h> |
127 | #elif defined(__s390__) |
128 | #include <rseq-s390.h> |
129 | #elif defined(__riscv) |
130 | #include <rseq-riscv.h> |
131 | #else |
132 | #error unsupported target |
133 | #endif |
134 | |
135 | /* |
136 | * Register rseq for the current thread. This needs to be called once |
137 | * by any thread which uses restartable sequences, before they start |
138 | * using restartable sequences, to ensure restartable sequences |
139 | * succeed. A restartable sequence executed from a non-registered |
140 | * thread will always fail. |
141 | */ |
142 | int rseq_register_current_thread(void); |
143 | |
144 | /* |
145 | * Unregister rseq for current thread. |
146 | */ |
147 | int rseq_unregister_current_thread(void); |
148 | |
149 | /* |
150 | * Restartable sequence fallback for reading the current CPU number. |
151 | */ |
152 | int32_t rseq_fallback_current_cpu(void); |
153 | |
154 | /* |
155 | * Restartable sequence fallback for reading the current node number. |
156 | */ |
157 | int32_t rseq_fallback_current_node(void); |
158 | |
159 | /* |
160 | * Values returned can be either the current CPU number, -1 (rseq is |
161 | * uninitialized), or -2 (rseq initialization has failed). |
162 | */ |
163 | static inline int32_t rseq_current_cpu_raw(void) |
164 | { |
165 | return RSEQ_ACCESS_ONCE(rseq_get_abi()->cpu_id); |
166 | } |
167 | |
168 | /* |
169 | * Returns a possible CPU number, which is typically the current CPU. |
170 | * The returned CPU number can be used to prepare for an rseq critical |
171 | * section, which will confirm whether the cpu number is indeed the |
172 | * current one, and whether rseq is initialized. |
173 | * |
174 | * The CPU number returned by rseq_cpu_start should always be validated |
175 | * by passing it to a rseq asm sequence, or by comparing it to the |
176 | * return value of rseq_current_cpu_raw() if the rseq asm sequence |
177 | * does not need to be invoked. |
178 | */ |
179 | static inline uint32_t rseq_cpu_start(void) |
180 | { |
181 | return RSEQ_ACCESS_ONCE(rseq_get_abi()->cpu_id_start); |
182 | } |
183 | |
184 | static inline uint32_t rseq_current_cpu(void) |
185 | { |
186 | int32_t cpu; |
187 | |
188 | cpu = rseq_current_cpu_raw(); |
189 | if (rseq_unlikely(cpu < 0)) |
190 | cpu = rseq_fallback_current_cpu(); |
191 | return cpu; |
192 | } |
193 | |
194 | static inline bool rseq_node_id_available(void) |
195 | { |
196 | return (int) rseq_feature_size >= rseq_offsetofend(struct rseq_abi, node_id); |
197 | } |
198 | |
199 | /* |
200 | * Current NUMA node number. |
201 | */ |
202 | static inline uint32_t rseq_current_node_id(void) |
203 | { |
204 | assert(rseq_node_id_available()); |
205 | return RSEQ_ACCESS_ONCE(rseq_get_abi()->node_id); |
206 | } |
207 | |
208 | static inline bool rseq_mm_cid_available(void) |
209 | { |
210 | return (int) rseq_feature_size >= rseq_offsetofend(struct rseq_abi, mm_cid); |
211 | } |
212 | |
213 | static inline uint32_t rseq_current_mm_cid(void) |
214 | { |
215 | return RSEQ_ACCESS_ONCE(rseq_get_abi()->mm_cid); |
216 | } |
217 | |
218 | static inline void rseq_clear_rseq_cs(void) |
219 | { |
220 | RSEQ_WRITE_ONCE(rseq_get_abi()->rseq_cs.arch.ptr, 0); |
221 | } |
222 | |
223 | /* |
224 | * rseq_prepare_unload() should be invoked by each thread executing a rseq |
225 | * critical section at least once between their last critical section and |
226 | * library unload of the library defining the rseq critical section (struct |
227 | * rseq_cs) or the code referred to by the struct rseq_cs start_ip and |
228 | * post_commit_offset fields. This also applies to use of rseq in code |
229 | * generated by JIT: rseq_prepare_unload() should be invoked at least once by |
230 | * each thread executing a rseq critical section before reclaim of the memory |
231 | * holding the struct rseq_cs or reclaim of the code pointed to by struct |
232 | * rseq_cs start_ip and post_commit_offset fields. |
233 | */ |
234 | static inline void rseq_prepare_unload(void) |
235 | { |
236 | rseq_clear_rseq_cs(); |
237 | } |
238 | |
239 | static inline __attribute__((always_inline)) |
240 | int rseq_cmpeqv_storev(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, |
241 | intptr_t *v, intptr_t expect, |
242 | intptr_t newv, int cpu) |
243 | { |
244 | if (rseq_mo != RSEQ_MO_RELAXED) |
245 | return -1; |
246 | switch (percpu_mode) { |
247 | case RSEQ_PERCPU_CPU_ID: |
248 | return rseq_cmpeqv_storev_relaxed_cpu_id(v, expect, newv, cpu); |
249 | case RSEQ_PERCPU_MM_CID: |
250 | return rseq_cmpeqv_storev_relaxed_mm_cid(v, expect, newv, cpu); |
251 | } |
252 | return -1; |
253 | } |
254 | |
255 | /* |
256 | * Compare @v against @expectnot. When it does _not_ match, load @v |
257 | * into @load, and store the content of *@v + voffp into @v. |
258 | */ |
259 | static inline __attribute__((always_inline)) |
260 | int rseq_cmpnev_storeoffp_load(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, |
261 | intptr_t *v, intptr_t expectnot, long voffp, intptr_t *load, |
262 | int cpu) |
263 | { |
264 | if (rseq_mo != RSEQ_MO_RELAXED) |
265 | return -1; |
266 | switch (percpu_mode) { |
267 | case RSEQ_PERCPU_CPU_ID: |
268 | return rseq_cmpnev_storeoffp_load_relaxed_cpu_id(v, expectnot, voffp, load, cpu); |
269 | case RSEQ_PERCPU_MM_CID: |
270 | return rseq_cmpnev_storeoffp_load_relaxed_mm_cid(v, expectnot, voffp, load, cpu); |
271 | } |
272 | return -1; |
273 | } |
274 | |
275 | static inline __attribute__((always_inline)) |
276 | int rseq_addv(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, |
277 | intptr_t *v, intptr_t count, int cpu) |
278 | { |
279 | if (rseq_mo != RSEQ_MO_RELAXED) |
280 | return -1; |
281 | switch (percpu_mode) { |
282 | case RSEQ_PERCPU_CPU_ID: |
283 | return rseq_addv_relaxed_cpu_id(v, count, cpu); |
284 | case RSEQ_PERCPU_MM_CID: |
285 | return rseq_addv_relaxed_mm_cid(v, count, cpu); |
286 | } |
287 | return -1; |
288 | } |
289 | |
290 | #ifdef RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV |
291 | /* |
292 | * pval = *(ptr+off) |
293 | * *pval += inc; |
294 | */ |
295 | static inline __attribute__((always_inline)) |
296 | int rseq_offset_deref_addv(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, |
297 | intptr_t *ptr, long off, intptr_t inc, int cpu) |
298 | { |
299 | if (rseq_mo != RSEQ_MO_RELAXED) |
300 | return -1; |
301 | switch (percpu_mode) { |
302 | case RSEQ_PERCPU_CPU_ID: |
303 | return rseq_offset_deref_addv_relaxed_cpu_id(ptr, off, inc, cpu); |
304 | case RSEQ_PERCPU_MM_CID: |
305 | return rseq_offset_deref_addv_relaxed_mm_cid(ptr, off, inc, cpu); |
306 | } |
307 | return -1; |
308 | } |
309 | #endif |
310 | |
311 | static inline __attribute__((always_inline)) |
312 | int rseq_cmpeqv_trystorev_storev(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, |
313 | intptr_t *v, intptr_t expect, |
314 | intptr_t *v2, intptr_t newv2, |
315 | intptr_t newv, int cpu) |
316 | { |
317 | switch (rseq_mo) { |
318 | case RSEQ_MO_RELAXED: |
319 | switch (percpu_mode) { |
320 | case RSEQ_PERCPU_CPU_ID: |
321 | return rseq_cmpeqv_trystorev_storev_relaxed_cpu_id(v, expect, v2, newv2, newv, cpu); |
322 | case RSEQ_PERCPU_MM_CID: |
323 | return rseq_cmpeqv_trystorev_storev_relaxed_mm_cid(v, expect, v2, newv2, newv, cpu); |
324 | } |
325 | return -1; |
326 | case RSEQ_MO_RELEASE: |
327 | switch (percpu_mode) { |
328 | case RSEQ_PERCPU_CPU_ID: |
329 | return rseq_cmpeqv_trystorev_storev_release_cpu_id(v, expect, v2, newv2, newv, cpu); |
330 | case RSEQ_PERCPU_MM_CID: |
331 | return rseq_cmpeqv_trystorev_storev_release_mm_cid(v, expect, v2, newv2, newv, cpu); |
332 | } |
333 | return -1; |
334 | default: |
335 | return -1; |
336 | } |
337 | } |
338 | |
339 | static inline __attribute__((always_inline)) |
340 | int rseq_cmpeqv_cmpeqv_storev(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, |
341 | intptr_t *v, intptr_t expect, |
342 | intptr_t *v2, intptr_t expect2, |
343 | intptr_t newv, int cpu) |
344 | { |
345 | if (rseq_mo != RSEQ_MO_RELAXED) |
346 | return -1; |
347 | switch (percpu_mode) { |
348 | case RSEQ_PERCPU_CPU_ID: |
349 | return rseq_cmpeqv_cmpeqv_storev_relaxed_cpu_id(v, expect, v2, expect2, newv, cpu); |
350 | case RSEQ_PERCPU_MM_CID: |
351 | return rseq_cmpeqv_cmpeqv_storev_relaxed_mm_cid(v, expect, v2, expect2, newv, cpu); |
352 | } |
353 | return -1; |
354 | } |
355 | |
356 | static inline __attribute__((always_inline)) |
357 | int rseq_cmpeqv_trymemcpy_storev(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, |
358 | intptr_t *v, intptr_t expect, |
359 | void *dst, void *src, size_t len, |
360 | intptr_t newv, int cpu) |
361 | { |
362 | switch (rseq_mo) { |
363 | case RSEQ_MO_RELAXED: |
364 | switch (percpu_mode) { |
365 | case RSEQ_PERCPU_CPU_ID: |
366 | return rseq_cmpeqv_trymemcpy_storev_relaxed_cpu_id(v, expect, dst, src, len, newv, cpu); |
367 | case RSEQ_PERCPU_MM_CID: |
368 | return rseq_cmpeqv_trymemcpy_storev_relaxed_mm_cid(v, expect, dst, src, len, newv, cpu); |
369 | } |
370 | return -1; |
371 | case RSEQ_MO_RELEASE: |
372 | switch (percpu_mode) { |
373 | case RSEQ_PERCPU_CPU_ID: |
374 | return rseq_cmpeqv_trymemcpy_storev_release_cpu_id(v, expect, dst, src, len, newv, cpu); |
375 | case RSEQ_PERCPU_MM_CID: |
376 | return rseq_cmpeqv_trymemcpy_storev_release_mm_cid(v, expect, dst, src, len, newv, cpu); |
377 | } |
378 | return -1; |
379 | default: |
380 | return -1; |
381 | } |
382 | } |
383 | |
384 | #endif /* RSEQ_H_ */ |
385 | |