1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2/*
3 * rseq-x86-bits.h
4 *
5 * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 */
7
8#include "rseq-bits-template.h"
9
10#ifdef __x86_64__
11
12#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
13 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
14
15static inline __attribute__((always_inline))
16int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
17{
18 RSEQ_INJECT_C(9)
19
20 __asm__ __volatile__ goto (
21 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
22 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
23#ifdef RSEQ_COMPARE_TWICE
24 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
25 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
26#endif
27 /* Start rseq by storing table entry pointer into rseq_cs. */
28 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
29 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
30 RSEQ_INJECT_ASM(3)
31 "cmpq %[v], %[expect]\n\t"
32 "jnz %l[cmpfail]\n\t"
33 RSEQ_INJECT_ASM(4)
34#ifdef RSEQ_COMPARE_TWICE
35 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
36 "cmpq %[v], %[expect]\n\t"
37 "jnz %l[error2]\n\t"
38#endif
39 /* final store */
40 "movq %[newv], %[v]\n\t"
41 "2:\n\t"
42 RSEQ_INJECT_ASM(5)
43 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
44 : /* gcc asm goto does not allow outputs */
45 : [cpu_id] "r" (cpu),
46 [rseq_offset] "r" (rseq_offset),
47 [v] "m" (*v),
48 [expect] "r" (expect),
49 [newv] "r" (newv)
50 : "memory", "cc", "rax"
51 RSEQ_INJECT_CLOBBER
52 : abort, cmpfail
53#ifdef RSEQ_COMPARE_TWICE
54 , error1, error2
55#endif
56 );
57 rseq_after_asm_goto();
58 return 0;
59abort:
60 rseq_after_asm_goto();
61 RSEQ_INJECT_FAILED
62 return -1;
63cmpfail:
64 rseq_after_asm_goto();
65 return 1;
66#ifdef RSEQ_COMPARE_TWICE
67error1:
68 rseq_after_asm_goto();
69 rseq_bug("cpu_id comparison failed");
70error2:
71 rseq_after_asm_goto();
72 rseq_bug("expected value comparison failed");
73#endif
74}
75
76/*
77 * Compare @v against @expectnot. When it does _not_ match, load @v
78 * into @load, and store the content of *@v + voffp into @v.
79 */
80static inline __attribute__((always_inline))
81int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
82 long voffp, intptr_t *load, int cpu)
83{
84 RSEQ_INJECT_C(9)
85
86 __asm__ __volatile__ goto (
87 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
88 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
89#ifdef RSEQ_COMPARE_TWICE
90 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
91 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
92#endif
93 /* Start rseq by storing table entry pointer into rseq_cs. */
94 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
95 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
96 RSEQ_INJECT_ASM(3)
97 "movq %[v], %%rbx\n\t"
98 "cmpq %%rbx, %[expectnot]\n\t"
99 "je %l[cmpfail]\n\t"
100 RSEQ_INJECT_ASM(4)
101#ifdef RSEQ_COMPARE_TWICE
102 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
103 "movq %[v], %%rbx\n\t"
104 "cmpq %%rbx, %[expectnot]\n\t"
105 "je %l[error2]\n\t"
106#endif
107 "movq %%rbx, %[load]\n\t"
108 "addq %[voffp], %%rbx\n\t"
109 "movq (%%rbx), %%rbx\n\t"
110 /* final store */
111 "movq %%rbx, %[v]\n\t"
112 "2:\n\t"
113 RSEQ_INJECT_ASM(5)
114 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
115 : /* gcc asm goto does not allow outputs */
116 : [cpu_id] "r" (cpu),
117 [rseq_offset] "r" (rseq_offset),
118 /* final store input */
119 [v] "m" (*v),
120 [expectnot] "r" (expectnot),
121 [voffp] "er" (voffp),
122 [load] "m" (*load)
123 : "memory", "cc", "rax", "rbx"
124 RSEQ_INJECT_CLOBBER
125 : abort, cmpfail
126#ifdef RSEQ_COMPARE_TWICE
127 , error1, error2
128#endif
129 );
130 rseq_after_asm_goto();
131 return 0;
132abort:
133 rseq_after_asm_goto();
134 RSEQ_INJECT_FAILED
135 return -1;
136cmpfail:
137 rseq_after_asm_goto();
138 return 1;
139#ifdef RSEQ_COMPARE_TWICE
140error1:
141 rseq_after_asm_goto();
142 rseq_bug("cpu_id comparison failed");
143error2:
144 rseq_after_asm_goto();
145 rseq_bug("expected value comparison failed");
146#endif
147}
148
149static inline __attribute__((always_inline))
150int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
151{
152 RSEQ_INJECT_C(9)
153
154 __asm__ __volatile__ goto (
155 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
156#ifdef RSEQ_COMPARE_TWICE
157 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
158#endif
159 /* Start rseq by storing table entry pointer into rseq_cs. */
160 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
161 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
162 RSEQ_INJECT_ASM(3)
163#ifdef RSEQ_COMPARE_TWICE
164 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
165#endif
166 /* final store */
167 "addq %[count], %[v]\n\t"
168 "2:\n\t"
169 RSEQ_INJECT_ASM(4)
170 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
171 : /* gcc asm goto does not allow outputs */
172 : [cpu_id] "r" (cpu),
173 [rseq_offset] "r" (rseq_offset),
174 /* final store input */
175 [v] "m" (*v),
176 [count] "er" (count)
177 : "memory", "cc", "rax"
178 RSEQ_INJECT_CLOBBER
179 : abort
180#ifdef RSEQ_COMPARE_TWICE
181 , error1
182#endif
183 );
184 rseq_after_asm_goto();
185 return 0;
186abort:
187 rseq_after_asm_goto();
188 RSEQ_INJECT_FAILED
189 return -1;
190#ifdef RSEQ_COMPARE_TWICE
191error1:
192 rseq_after_asm_goto();
193 rseq_bug("cpu_id comparison failed");
194#endif
195}
196
197#define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
198
199/*
200 * pval = *(ptr+off)
201 * *pval += inc;
202 */
203static inline __attribute__((always_inline))
204int RSEQ_TEMPLATE_IDENTIFIER(rseq_offset_deref_addv)(intptr_t *ptr, long off, intptr_t inc, int cpu)
205{
206 RSEQ_INJECT_C(9)
207
208 __asm__ __volatile__ goto (
209 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
210#ifdef RSEQ_COMPARE_TWICE
211 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
212#endif
213 /* Start rseq by storing table entry pointer into rseq_cs. */
214 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
215 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
216 RSEQ_INJECT_ASM(3)
217#ifdef RSEQ_COMPARE_TWICE
218 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
219#endif
220 /* get p+v */
221 "movq %[ptr], %%rbx\n\t"
222 "addq %[off], %%rbx\n\t"
223 /* get pv */
224 "movq (%%rbx), %%rcx\n\t"
225 /* *pv += inc */
226 "addq %[inc], (%%rcx)\n\t"
227 "2:\n\t"
228 RSEQ_INJECT_ASM(4)
229 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
230 : /* gcc asm goto does not allow outputs */
231 : [cpu_id] "r" (cpu),
232 [rseq_offset] "r" (rseq_offset),
233 /* final store input */
234 [ptr] "m" (*ptr),
235 [off] "er" (off),
236 [inc] "er" (inc)
237 : "memory", "cc", "rax", "rbx", "rcx"
238 RSEQ_INJECT_CLOBBER
239 : abort
240#ifdef RSEQ_COMPARE_TWICE
241 , error1
242#endif
243 );
244 return 0;
245abort:
246 RSEQ_INJECT_FAILED
247 return -1;
248#ifdef RSEQ_COMPARE_TWICE
249error1:
250 rseq_bug("cpu_id comparison failed");
251#endif
252}
253
254static inline __attribute__((always_inline))
255int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
256 intptr_t *v2, intptr_t expect2,
257 intptr_t newv, int cpu)
258{
259 RSEQ_INJECT_C(9)
260
261 __asm__ __volatile__ goto (
262 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
263 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
264#ifdef RSEQ_COMPARE_TWICE
265 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
266 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
267 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
268#endif
269 /* Start rseq by storing table entry pointer into rseq_cs. */
270 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
271 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
272 RSEQ_INJECT_ASM(3)
273 "cmpq %[v], %[expect]\n\t"
274 "jnz %l[cmpfail]\n\t"
275 RSEQ_INJECT_ASM(4)
276 "cmpq %[v2], %[expect2]\n\t"
277 "jnz %l[cmpfail]\n\t"
278 RSEQ_INJECT_ASM(5)
279#ifdef RSEQ_COMPARE_TWICE
280 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
281 "cmpq %[v], %[expect]\n\t"
282 "jnz %l[error2]\n\t"
283 "cmpq %[v2], %[expect2]\n\t"
284 "jnz %l[error3]\n\t"
285#endif
286 /* final store */
287 "movq %[newv], %[v]\n\t"
288 "2:\n\t"
289 RSEQ_INJECT_ASM(6)
290 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
291 : /* gcc asm goto does not allow outputs */
292 : [cpu_id] "r" (cpu),
293 [rseq_offset] "r" (rseq_offset),
294 /* cmp2 input */
295 [v2] "m" (*v2),
296 [expect2] "r" (expect2),
297 /* final store input */
298 [v] "m" (*v),
299 [expect] "r" (expect),
300 [newv] "r" (newv)
301 : "memory", "cc", "rax"
302 RSEQ_INJECT_CLOBBER
303 : abort, cmpfail
304#ifdef RSEQ_COMPARE_TWICE
305 , error1, error2, error3
306#endif
307 );
308 rseq_after_asm_goto();
309 return 0;
310abort:
311 rseq_after_asm_goto();
312 RSEQ_INJECT_FAILED
313 return -1;
314cmpfail:
315 rseq_after_asm_goto();
316 return 1;
317#ifdef RSEQ_COMPARE_TWICE
318error1:
319 rseq_after_asm_goto();
320 rseq_bug("cpu_id comparison failed");
321error2:
322 rseq_after_asm_goto();
323 rseq_bug("1st expected value comparison failed");
324error3:
325 rseq_after_asm_goto();
326 rseq_bug("2nd expected value comparison failed");
327#endif
328}
329
330#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
331 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
332
333#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
334 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
335
336static inline __attribute__((always_inline))
337int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
338 intptr_t *v2, intptr_t newv2,
339 intptr_t newv, int cpu)
340{
341 RSEQ_INJECT_C(9)
342
343 __asm__ __volatile__ goto (
344 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
345 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
346#ifdef RSEQ_COMPARE_TWICE
347 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
348 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
349#endif
350 /* Start rseq by storing table entry pointer into rseq_cs. */
351 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
352 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
353 RSEQ_INJECT_ASM(3)
354 "cmpq %[v], %[expect]\n\t"
355 "jnz %l[cmpfail]\n\t"
356 RSEQ_INJECT_ASM(4)
357#ifdef RSEQ_COMPARE_TWICE
358 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
359 "cmpq %[v], %[expect]\n\t"
360 "jnz %l[error2]\n\t"
361#endif
362 /* try store */
363 "movq %[newv2], %[v2]\n\t"
364 RSEQ_INJECT_ASM(5)
365 /* final store */
366 "movq %[newv], %[v]\n\t"
367 "2:\n\t"
368 RSEQ_INJECT_ASM(6)
369 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
370 : /* gcc asm goto does not allow outputs */
371 : [cpu_id] "r" (cpu),
372 [rseq_offset] "r" (rseq_offset),
373 /* try store input */
374 [v2] "m" (*v2),
375 [newv2] "r" (newv2),
376 /* final store input */
377 [v] "m" (*v),
378 [expect] "r" (expect),
379 [newv] "r" (newv)
380 : "memory", "cc", "rax"
381 RSEQ_INJECT_CLOBBER
382 : abort, cmpfail
383#ifdef RSEQ_COMPARE_TWICE
384 , error1, error2
385#endif
386 );
387 rseq_after_asm_goto();
388 return 0;
389abort:
390 rseq_after_asm_goto();
391 RSEQ_INJECT_FAILED
392 return -1;
393cmpfail:
394 rseq_after_asm_goto();
395 return 1;
396#ifdef RSEQ_COMPARE_TWICE
397error1:
398 rseq_after_asm_goto();
399 rseq_bug("cpu_id comparison failed");
400error2:
401 rseq_after_asm_goto();
402 rseq_bug("expected value comparison failed");
403#endif
404}
405
406static inline __attribute__((always_inline))
407int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
408 void *dst, void *src, size_t len,
409 intptr_t newv, int cpu)
410{
411 uint64_t rseq_scratch[3];
412
413 RSEQ_INJECT_C(9)
414
415 __asm__ __volatile__ goto (
416 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
417 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
418#ifdef RSEQ_COMPARE_TWICE
419 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
420 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
421#endif
422 "movq %[src], %[rseq_scratch0]\n\t"
423 "movq %[dst], %[rseq_scratch1]\n\t"
424 "movq %[len], %[rseq_scratch2]\n\t"
425 /* Start rseq by storing table entry pointer into rseq_cs. */
426 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
427 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
428 RSEQ_INJECT_ASM(3)
429 "cmpq %[v], %[expect]\n\t"
430 "jnz 5f\n\t"
431 RSEQ_INJECT_ASM(4)
432#ifdef RSEQ_COMPARE_TWICE
433 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 6f)
434 "cmpq %[v], %[expect]\n\t"
435 "jnz 7f\n\t"
436#endif
437 /* try memcpy */
438 "test %[len], %[len]\n\t" \
439 "jz 333f\n\t" \
440 "222:\n\t" \
441 "movb (%[src]), %%al\n\t" \
442 "movb %%al, (%[dst])\n\t" \
443 "inc %[src]\n\t" \
444 "inc %[dst]\n\t" \
445 "dec %[len]\n\t" \
446 "jnz 222b\n\t" \
447 "333:\n\t" \
448 RSEQ_INJECT_ASM(5)
449 /* final store */
450 "movq %[newv], %[v]\n\t"
451 "2:\n\t"
452 RSEQ_INJECT_ASM(6)
453 /* teardown */
454 "movq %[rseq_scratch2], %[len]\n\t"
455 "movq %[rseq_scratch1], %[dst]\n\t"
456 "movq %[rseq_scratch0], %[src]\n\t"
457 RSEQ_ASM_DEFINE_ABORT(4,
458 "movq %[rseq_scratch2], %[len]\n\t"
459 "movq %[rseq_scratch1], %[dst]\n\t"
460 "movq %[rseq_scratch0], %[src]\n\t",
461 abort)
462 RSEQ_ASM_DEFINE_CMPFAIL(5,
463 "movq %[rseq_scratch2], %[len]\n\t"
464 "movq %[rseq_scratch1], %[dst]\n\t"
465 "movq %[rseq_scratch0], %[src]\n\t",
466 cmpfail)
467#ifdef RSEQ_COMPARE_TWICE
468 RSEQ_ASM_DEFINE_CMPFAIL(6,
469 "movq %[rseq_scratch2], %[len]\n\t"
470 "movq %[rseq_scratch1], %[dst]\n\t"
471 "movq %[rseq_scratch0], %[src]\n\t",
472 error1)
473 RSEQ_ASM_DEFINE_CMPFAIL(7,
474 "movq %[rseq_scratch2], %[len]\n\t"
475 "movq %[rseq_scratch1], %[dst]\n\t"
476 "movq %[rseq_scratch0], %[src]\n\t",
477 error2)
478#endif
479 : /* gcc asm goto does not allow outputs */
480 : [cpu_id] "r" (cpu),
481 [rseq_offset] "r" (rseq_offset),
482 /* final store input */
483 [v] "m" (*v),
484 [expect] "r" (expect),
485 [newv] "r" (newv),
486 /* try memcpy input */
487 [dst] "r" (dst),
488 [src] "r" (src),
489 [len] "r" (len),
490 [rseq_scratch0] "m" (rseq_scratch[0]),
491 [rseq_scratch1] "m" (rseq_scratch[1]),
492 [rseq_scratch2] "m" (rseq_scratch[2])
493 : "memory", "cc", "rax"
494 RSEQ_INJECT_CLOBBER
495 : abort, cmpfail
496#ifdef RSEQ_COMPARE_TWICE
497 , error1, error2
498#endif
499 );
500 rseq_after_asm_goto();
501 return 0;
502abort:
503 rseq_after_asm_goto();
504 RSEQ_INJECT_FAILED
505 return -1;
506cmpfail:
507 rseq_after_asm_goto();
508 return 1;
509#ifdef RSEQ_COMPARE_TWICE
510error1:
511 rseq_after_asm_goto();
512 rseq_bug("cpu_id comparison failed");
513error2:
514 rseq_after_asm_goto();
515 rseq_bug("expected value comparison failed");
516#endif
517}
518
519#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
520 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
521
522#elif defined(__i386__)
523
524#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
525 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
526
527static inline __attribute__((always_inline))
528int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
529{
530 RSEQ_INJECT_C(9)
531
532 __asm__ __volatile__ goto (
533 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
534 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
535#ifdef RSEQ_COMPARE_TWICE
536 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
537 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
538#endif
539 /* Start rseq by storing table entry pointer into rseq_cs. */
540 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
541 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
542 RSEQ_INJECT_ASM(3)
543 "cmpl %[v], %[expect]\n\t"
544 "jnz %l[cmpfail]\n\t"
545 RSEQ_INJECT_ASM(4)
546#ifdef RSEQ_COMPARE_TWICE
547 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
548 "cmpl %[v], %[expect]\n\t"
549 "jnz %l[error2]\n\t"
550#endif
551 /* final store */
552 "movl %[newv], %[v]\n\t"
553 "2:\n\t"
554 RSEQ_INJECT_ASM(5)
555 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
556 : /* gcc asm goto does not allow outputs */
557 : [cpu_id] "r" (cpu),
558 [rseq_offset] "r" (rseq_offset),
559 [v] "m" (*v),
560 [expect] "r" (expect),
561 [newv] "r" (newv)
562 : "memory", "cc", "eax"
563 RSEQ_INJECT_CLOBBER
564 : abort, cmpfail
565#ifdef RSEQ_COMPARE_TWICE
566 , error1, error2
567#endif
568 );
569 rseq_after_asm_goto();
570 return 0;
571abort:
572 rseq_after_asm_goto();
573 RSEQ_INJECT_FAILED
574 return -1;
575cmpfail:
576 rseq_after_asm_goto();
577 return 1;
578#ifdef RSEQ_COMPARE_TWICE
579error1:
580 rseq_after_asm_goto();
581 rseq_bug("cpu_id comparison failed");
582error2:
583 rseq_after_asm_goto();
584 rseq_bug("expected value comparison failed");
585#endif
586}
587
588/*
589 * Compare @v against @expectnot. When it does _not_ match, load @v
590 * into @load, and store the content of *@v + voffp into @v.
591 */
592static inline __attribute__((always_inline))
593int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
594 long voffp, intptr_t *load, int cpu)
595{
596 RSEQ_INJECT_C(9)
597
598 __asm__ __volatile__ goto (
599 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
600 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
601#ifdef RSEQ_COMPARE_TWICE
602 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
603 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
604#endif
605 /* Start rseq by storing table entry pointer into rseq_cs. */
606 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
607 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
608 RSEQ_INJECT_ASM(3)
609 "movl %[v], %%ebx\n\t"
610 "cmpl %%ebx, %[expectnot]\n\t"
611 "je %l[cmpfail]\n\t"
612 RSEQ_INJECT_ASM(4)
613#ifdef RSEQ_COMPARE_TWICE
614 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
615 "movl %[v], %%ebx\n\t"
616 "cmpl %%ebx, %[expectnot]\n\t"
617 "je %l[error2]\n\t"
618#endif
619 "movl %%ebx, %[load]\n\t"
620 "addl %[voffp], %%ebx\n\t"
621 "movl (%%ebx), %%ebx\n\t"
622 /* final store */
623 "movl %%ebx, %[v]\n\t"
624 "2:\n\t"
625 RSEQ_INJECT_ASM(5)
626 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
627 : /* gcc asm goto does not allow outputs */
628 : [cpu_id] "r" (cpu),
629 [rseq_offset] "r" (rseq_offset),
630 /* final store input */
631 [v] "m" (*v),
632 [expectnot] "r" (expectnot),
633 [voffp] "ir" (voffp),
634 [load] "m" (*load)
635 : "memory", "cc", "eax", "ebx"
636 RSEQ_INJECT_CLOBBER
637 : abort, cmpfail
638#ifdef RSEQ_COMPARE_TWICE
639 , error1, error2
640#endif
641 );
642 rseq_after_asm_goto();
643 return 0;
644abort:
645 rseq_after_asm_goto();
646 RSEQ_INJECT_FAILED
647 return -1;
648cmpfail:
649 rseq_after_asm_goto();
650 return 1;
651#ifdef RSEQ_COMPARE_TWICE
652error1:
653 rseq_after_asm_goto();
654 rseq_bug("cpu_id comparison failed");
655error2:
656 rseq_after_asm_goto();
657 rseq_bug("expected value comparison failed");
658#endif
659}
660
661static inline __attribute__((always_inline))
662int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
663{
664 RSEQ_INJECT_C(9)
665
666 __asm__ __volatile__ goto (
667 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
668#ifdef RSEQ_COMPARE_TWICE
669 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
670#endif
671 /* Start rseq by storing table entry pointer into rseq_cs. */
672 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
673 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
674 RSEQ_INJECT_ASM(3)
675#ifdef RSEQ_COMPARE_TWICE
676 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
677#endif
678 /* final store */
679 "addl %[count], %[v]\n\t"
680 "2:\n\t"
681 RSEQ_INJECT_ASM(4)
682 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
683 : /* gcc asm goto does not allow outputs */
684 : [cpu_id] "r" (cpu),
685 [rseq_offset] "r" (rseq_offset),
686 /* final store input */
687 [v] "m" (*v),
688 [count] "ir" (count)
689 : "memory", "cc", "eax"
690 RSEQ_INJECT_CLOBBER
691 : abort
692#ifdef RSEQ_COMPARE_TWICE
693 , error1
694#endif
695 );
696 rseq_after_asm_goto();
697 return 0;
698abort:
699 rseq_after_asm_goto();
700 RSEQ_INJECT_FAILED
701 return -1;
702#ifdef RSEQ_COMPARE_TWICE
703error1:
704 rseq_after_asm_goto();
705 rseq_bug("cpu_id comparison failed");
706#endif
707}
708
709static inline __attribute__((always_inline))
710int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
711 intptr_t *v2, intptr_t expect2,
712 intptr_t newv, int cpu)
713{
714 RSEQ_INJECT_C(9)
715
716 __asm__ __volatile__ goto (
717 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
718 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
719#ifdef RSEQ_COMPARE_TWICE
720 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
721 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
722 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
723#endif
724 /* Start rseq by storing table entry pointer into rseq_cs. */
725 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
726 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
727 RSEQ_INJECT_ASM(3)
728 "cmpl %[v], %[expect]\n\t"
729 "jnz %l[cmpfail]\n\t"
730 RSEQ_INJECT_ASM(4)
731 "cmpl %[expect2], %[v2]\n\t"
732 "jnz %l[cmpfail]\n\t"
733 RSEQ_INJECT_ASM(5)
734#ifdef RSEQ_COMPARE_TWICE
735 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
736 "cmpl %[v], %[expect]\n\t"
737 "jnz %l[error2]\n\t"
738 "cmpl %[expect2], %[v2]\n\t"
739 "jnz %l[error3]\n\t"
740#endif
741 "movl %[newv], %%eax\n\t"
742 /* final store */
743 "movl %%eax, %[v]\n\t"
744 "2:\n\t"
745 RSEQ_INJECT_ASM(6)
746 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
747 : /* gcc asm goto does not allow outputs */
748 : [cpu_id] "r" (cpu),
749 [rseq_offset] "r" (rseq_offset),
750 /* cmp2 input */
751 [v2] "m" (*v2),
752 [expect2] "r" (expect2),
753 /* final store input */
754 [v] "m" (*v),
755 [expect] "r" (expect),
756 [newv] "m" (newv)
757 : "memory", "cc", "eax"
758 RSEQ_INJECT_CLOBBER
759 : abort, cmpfail
760#ifdef RSEQ_COMPARE_TWICE
761 , error1, error2, error3
762#endif
763 );
764 rseq_after_asm_goto();
765 return 0;
766abort:
767 rseq_after_asm_goto();
768 RSEQ_INJECT_FAILED
769 return -1;
770cmpfail:
771 rseq_after_asm_goto();
772 return 1;
773#ifdef RSEQ_COMPARE_TWICE
774error1:
775 rseq_after_asm_goto();
776 rseq_bug("cpu_id comparison failed");
777error2:
778 rseq_after_asm_goto();
779 rseq_bug("1st expected value comparison failed");
780error3:
781 rseq_after_asm_goto();
782 rseq_bug("2nd expected value comparison failed");
783#endif
784}
785
786#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
787 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
788
789#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
790 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
791
792static inline __attribute__((always_inline))
793int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
794 intptr_t *v2, intptr_t newv2,
795 intptr_t newv, int cpu)
796{
797 RSEQ_INJECT_C(9)
798
799 __asm__ __volatile__ goto (
800 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
801 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
802#ifdef RSEQ_COMPARE_TWICE
803 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
804 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
805#endif
806 /* Start rseq by storing table entry pointer into rseq_cs. */
807 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
808 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
809 RSEQ_INJECT_ASM(3)
810 "movl %[expect], %%eax\n\t"
811 "cmpl %[v], %%eax\n\t"
812 "jnz %l[cmpfail]\n\t"
813 RSEQ_INJECT_ASM(4)
814#ifdef RSEQ_COMPARE_TWICE
815 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
816 "movl %[expect], %%eax\n\t"
817 "cmpl %[v], %%eax\n\t"
818 "jnz %l[error2]\n\t"
819#endif
820 /* try store */
821 "movl %[newv2], %[v2]\n\t"
822 RSEQ_INJECT_ASM(5)
823#ifdef RSEQ_TEMPLATE_MO_RELEASE
824 "lock; addl $0,-128(%%esp)\n\t"
825#endif
826 /* final store */
827 "movl %[newv], %[v]\n\t"
828 "2:\n\t"
829 RSEQ_INJECT_ASM(6)
830 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
831 : /* gcc asm goto does not allow outputs */
832 : [cpu_id] "r" (cpu),
833 [rseq_offset] "r" (rseq_offset),
834 /* try store input */
835 [v2] "m" (*v2),
836 [newv2] "r" (newv2),
837 /* final store input */
838 [v] "m" (*v),
839 [expect] "m" (expect),
840 [newv] "r" (newv)
841 : "memory", "cc", "eax"
842 RSEQ_INJECT_CLOBBER
843 : abort, cmpfail
844#ifdef RSEQ_COMPARE_TWICE
845 , error1, error2
846#endif
847 );
848 rseq_after_asm_goto();
849 return 0;
850abort:
851 rseq_after_asm_goto();
852 RSEQ_INJECT_FAILED
853 return -1;
854cmpfail:
855 rseq_after_asm_goto();
856 return 1;
857#ifdef RSEQ_COMPARE_TWICE
858error1:
859 rseq_after_asm_goto();
860 rseq_bug("cpu_id comparison failed");
861error2:
862 rseq_after_asm_goto();
863 rseq_bug("expected value comparison failed");
864#endif
865
866}
867
868/* TODO: implement a faster memcpy. */
869static inline __attribute__((always_inline))
870int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
871 void *dst, void *src, size_t len,
872 intptr_t newv, int cpu)
873{
874 uint32_t rseq_scratch[3];
875
876 RSEQ_INJECT_C(9)
877
878 __asm__ __volatile__ goto (
879 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
880 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
881#ifdef RSEQ_COMPARE_TWICE
882 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
883 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
884#endif
885 "movl %[src], %[rseq_scratch0]\n\t"
886 "movl %[dst], %[rseq_scratch1]\n\t"
887 "movl %[len], %[rseq_scratch2]\n\t"
888 /* Start rseq by storing table entry pointer into rseq_cs. */
889 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
890 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
891 RSEQ_INJECT_ASM(3)
892 "movl %[expect], %%eax\n\t"
893 "cmpl %%eax, %[v]\n\t"
894 "jnz 5f\n\t"
895 RSEQ_INJECT_ASM(4)
896#ifdef RSEQ_COMPARE_TWICE
897 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 6f)
898 "movl %[expect], %%eax\n\t"
899 "cmpl %%eax, %[v]\n\t"
900 "jnz 7f\n\t"
901#endif
902 /* try memcpy */
903 "test %[len], %[len]\n\t" \
904 "jz 333f\n\t" \
905 "222:\n\t" \
906 "movb (%[src]), %%al\n\t" \
907 "movb %%al, (%[dst])\n\t" \
908 "inc %[src]\n\t" \
909 "inc %[dst]\n\t" \
910 "dec %[len]\n\t" \
911 "jnz 222b\n\t" \
912 "333:\n\t" \
913 RSEQ_INJECT_ASM(5)
914#ifdef RSEQ_TEMPLATE_MO_RELEASE
915 "lock; addl $0,-128(%%esp)\n\t"
916#endif
917 "movl %[newv], %%eax\n\t"
918 /* final store */
919 "movl %%eax, %[v]\n\t"
920 "2:\n\t"
921 RSEQ_INJECT_ASM(6)
922 /* teardown */
923 "movl %[rseq_scratch2], %[len]\n\t"
924 "movl %[rseq_scratch1], %[dst]\n\t"
925 "movl %[rseq_scratch0], %[src]\n\t"
926 RSEQ_ASM_DEFINE_ABORT(4,
927 "movl %[rseq_scratch2], %[len]\n\t"
928 "movl %[rseq_scratch1], %[dst]\n\t"
929 "movl %[rseq_scratch0], %[src]\n\t",
930 abort)
931 RSEQ_ASM_DEFINE_CMPFAIL(5,
932 "movl %[rseq_scratch2], %[len]\n\t"
933 "movl %[rseq_scratch1], %[dst]\n\t"
934 "movl %[rseq_scratch0], %[src]\n\t",
935 cmpfail)
936#ifdef RSEQ_COMPARE_TWICE
937 RSEQ_ASM_DEFINE_CMPFAIL(6,
938 "movl %[rseq_scratch2], %[len]\n\t"
939 "movl %[rseq_scratch1], %[dst]\n\t"
940 "movl %[rseq_scratch0], %[src]\n\t",
941 error1)
942 RSEQ_ASM_DEFINE_CMPFAIL(7,
943 "movl %[rseq_scratch2], %[len]\n\t"
944 "movl %[rseq_scratch1], %[dst]\n\t"
945 "movl %[rseq_scratch0], %[src]\n\t",
946 error2)
947#endif
948 : /* gcc asm goto does not allow outputs */
949 : [cpu_id] "r" (cpu),
950 [rseq_offset] "r" (rseq_offset),
951 /* final store input */
952 [v] "m" (*v),
953 [expect] "m" (expect),
954 [newv] "m" (newv),
955 /* try memcpy input */
956 [dst] "r" (dst),
957 [src] "r" (src),
958 [len] "r" (len),
959 [rseq_scratch0] "m" (rseq_scratch[0]),
960 [rseq_scratch1] "m" (rseq_scratch[1]),
961 [rseq_scratch2] "m" (rseq_scratch[2])
962 : "memory", "cc", "eax"
963 RSEQ_INJECT_CLOBBER
964 : abort, cmpfail
965#ifdef RSEQ_COMPARE_TWICE
966 , error1, error2
967#endif
968 );
969 rseq_after_asm_goto();
970 return 0;
971abort:
972 rseq_after_asm_goto();
973 RSEQ_INJECT_FAILED
974 return -1;
975cmpfail:
976 rseq_after_asm_goto();
977 return 1;
978#ifdef RSEQ_COMPARE_TWICE
979error1:
980 rseq_after_asm_goto();
981 rseq_bug("cpu_id comparison failed");
982error2:
983 rseq_after_asm_goto();
984 rseq_bug("expected value comparison failed");
985#endif
986}
987
988#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
989 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
990
991#endif
992
993#include "rseq-bits-reset.h"
994

source code of linux/tools/testing/selftests/rseq/rseq-x86-bits.h