1 | /* SPDX-License-Identifier: LGPL-2.1 OR MIT */ |
2 | /* |
3 | * rseq-x86-bits.h |
4 | * |
5 | * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> |
6 | */ |
7 | |
8 | #include "rseq-bits-template.h" |
9 | |
10 | #ifdef __x86_64__ |
11 | |
12 | #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ |
13 | (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) |
14 | |
15 | static inline __attribute__((always_inline)) |
16 | int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) |
17 | { |
18 | RSEQ_INJECT_C(9) |
19 | |
20 | __asm__ __volatile__ goto ( |
21 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ |
22 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
23 | #ifdef RSEQ_COMPARE_TWICE |
24 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) |
25 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) |
26 | #endif |
27 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
28 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
29 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) |
30 | RSEQ_INJECT_ASM(3) |
31 | "cmpq %[v], %[expect]\n\t" |
32 | "jnz %l[cmpfail]\n\t" |
33 | RSEQ_INJECT_ASM(4) |
34 | #ifdef RSEQ_COMPARE_TWICE |
35 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) |
36 | "cmpq %[v], %[expect]\n\t" |
37 | "jnz %l[error2]\n\t" |
38 | #endif |
39 | /* final store */ |
40 | "movq %[newv], %[v]\n\t" |
41 | "2:\n\t" |
42 | RSEQ_INJECT_ASM(5) |
43 | RSEQ_ASM_DEFINE_ABORT(4, "" , abort) |
44 | : /* gcc asm goto does not allow outputs */ |
45 | : [cpu_id] "r" (cpu), |
46 | [rseq_offset] "r" (rseq_offset), |
47 | [v] "m" (*v), |
48 | [expect] "r" (expect), |
49 | [newv] "r" (newv) |
50 | : "memory" , "cc" , "rax" |
51 | RSEQ_INJECT_CLOBBER |
52 | : abort, cmpfail |
53 | #ifdef RSEQ_COMPARE_TWICE |
54 | , error1, error2 |
55 | #endif |
56 | ); |
57 | rseq_after_asm_goto(); |
58 | return 0; |
59 | abort: |
60 | rseq_after_asm_goto(); |
61 | RSEQ_INJECT_FAILED |
62 | return -1; |
63 | cmpfail: |
64 | rseq_after_asm_goto(); |
65 | return 1; |
66 | #ifdef RSEQ_COMPARE_TWICE |
67 | error1: |
68 | rseq_after_asm_goto(); |
69 | rseq_bug("cpu_id comparison failed" ); |
70 | error2: |
71 | rseq_after_asm_goto(); |
72 | rseq_bug("expected value comparison failed" ); |
73 | #endif |
74 | } |
75 | |
76 | /* |
77 | * Compare @v against @expectnot. When it does _not_ match, load @v |
78 | * into @load, and store the content of *@v + voffp into @v. |
79 | */ |
80 | static inline __attribute__((always_inline)) |
81 | int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, |
82 | long voffp, intptr_t *load, int cpu) |
83 | { |
84 | RSEQ_INJECT_C(9) |
85 | |
86 | __asm__ __volatile__ goto ( |
87 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ |
88 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
89 | #ifdef RSEQ_COMPARE_TWICE |
90 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) |
91 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) |
92 | #endif |
93 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
94 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
95 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) |
96 | RSEQ_INJECT_ASM(3) |
97 | "movq %[v], %%rbx\n\t" |
98 | "cmpq %%rbx, %[expectnot]\n\t" |
99 | "je %l[cmpfail]\n\t" |
100 | RSEQ_INJECT_ASM(4) |
101 | #ifdef RSEQ_COMPARE_TWICE |
102 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) |
103 | "movq %[v], %%rbx\n\t" |
104 | "cmpq %%rbx, %[expectnot]\n\t" |
105 | "je %l[error2]\n\t" |
106 | #endif |
107 | "movq %%rbx, %[load]\n\t" |
108 | "addq %[voffp], %%rbx\n\t" |
109 | "movq (%%rbx), %%rbx\n\t" |
110 | /* final store */ |
111 | "movq %%rbx, %[v]\n\t" |
112 | "2:\n\t" |
113 | RSEQ_INJECT_ASM(5) |
114 | RSEQ_ASM_DEFINE_ABORT(4, "" , abort) |
115 | : /* gcc asm goto does not allow outputs */ |
116 | : [cpu_id] "r" (cpu), |
117 | [rseq_offset] "r" (rseq_offset), |
118 | /* final store input */ |
119 | [v] "m" (*v), |
120 | [expectnot] "r" (expectnot), |
121 | [voffp] "er" (voffp), |
122 | [load] "m" (*load) |
123 | : "memory" , "cc" , "rax" , "rbx" |
124 | RSEQ_INJECT_CLOBBER |
125 | : abort, cmpfail |
126 | #ifdef RSEQ_COMPARE_TWICE |
127 | , error1, error2 |
128 | #endif |
129 | ); |
130 | rseq_after_asm_goto(); |
131 | return 0; |
132 | abort: |
133 | rseq_after_asm_goto(); |
134 | RSEQ_INJECT_FAILED |
135 | return -1; |
136 | cmpfail: |
137 | rseq_after_asm_goto(); |
138 | return 1; |
139 | #ifdef RSEQ_COMPARE_TWICE |
140 | error1: |
141 | rseq_after_asm_goto(); |
142 | rseq_bug("cpu_id comparison failed" ); |
143 | error2: |
144 | rseq_after_asm_goto(); |
145 | rseq_bug("expected value comparison failed" ); |
146 | #endif |
147 | } |
148 | |
149 | static inline __attribute__((always_inline)) |
150 | int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) |
151 | { |
152 | RSEQ_INJECT_C(9) |
153 | |
154 | __asm__ __volatile__ goto ( |
155 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ |
156 | #ifdef RSEQ_COMPARE_TWICE |
157 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) |
158 | #endif |
159 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
160 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
161 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) |
162 | RSEQ_INJECT_ASM(3) |
163 | #ifdef RSEQ_COMPARE_TWICE |
164 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) |
165 | #endif |
166 | /* final store */ |
167 | "addq %[count], %[v]\n\t" |
168 | "2:\n\t" |
169 | RSEQ_INJECT_ASM(4) |
170 | RSEQ_ASM_DEFINE_ABORT(4, "" , abort) |
171 | : /* gcc asm goto does not allow outputs */ |
172 | : [cpu_id] "r" (cpu), |
173 | [rseq_offset] "r" (rseq_offset), |
174 | /* final store input */ |
175 | [v] "m" (*v), |
176 | [count] "er" (count) |
177 | : "memory" , "cc" , "rax" |
178 | RSEQ_INJECT_CLOBBER |
179 | : abort |
180 | #ifdef RSEQ_COMPARE_TWICE |
181 | , error1 |
182 | #endif |
183 | ); |
184 | rseq_after_asm_goto(); |
185 | return 0; |
186 | abort: |
187 | rseq_after_asm_goto(); |
188 | RSEQ_INJECT_FAILED |
189 | return -1; |
190 | #ifdef RSEQ_COMPARE_TWICE |
191 | error1: |
192 | rseq_after_asm_goto(); |
193 | rseq_bug("cpu_id comparison failed" ); |
194 | #endif |
195 | } |
196 | |
197 | #define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV |
198 | |
199 | /* |
200 | * pval = *(ptr+off) |
201 | * *pval += inc; |
202 | */ |
203 | static inline __attribute__((always_inline)) |
204 | int RSEQ_TEMPLATE_IDENTIFIER(rseq_offset_deref_addv)(intptr_t *ptr, long off, intptr_t inc, int cpu) |
205 | { |
206 | RSEQ_INJECT_C(9) |
207 | |
208 | __asm__ __volatile__ goto ( |
209 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ |
210 | #ifdef RSEQ_COMPARE_TWICE |
211 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) |
212 | #endif |
213 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
214 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
215 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) |
216 | RSEQ_INJECT_ASM(3) |
217 | #ifdef RSEQ_COMPARE_TWICE |
218 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) |
219 | #endif |
220 | /* get p+v */ |
221 | "movq %[ptr], %%rbx\n\t" |
222 | "addq %[off], %%rbx\n\t" |
223 | /* get pv */ |
224 | "movq (%%rbx), %%rcx\n\t" |
225 | /* *pv += inc */ |
226 | "addq %[inc], (%%rcx)\n\t" |
227 | "2:\n\t" |
228 | RSEQ_INJECT_ASM(4) |
229 | RSEQ_ASM_DEFINE_ABORT(4, "" , abort) |
230 | : /* gcc asm goto does not allow outputs */ |
231 | : [cpu_id] "r" (cpu), |
232 | [rseq_offset] "r" (rseq_offset), |
233 | /* final store input */ |
234 | [ptr] "m" (*ptr), |
235 | [off] "er" (off), |
236 | [inc] "er" (inc) |
237 | : "memory" , "cc" , "rax" , "rbx" , "rcx" |
238 | RSEQ_INJECT_CLOBBER |
239 | : abort |
240 | #ifdef RSEQ_COMPARE_TWICE |
241 | , error1 |
242 | #endif |
243 | ); |
244 | return 0; |
245 | abort: |
246 | RSEQ_INJECT_FAILED |
247 | return -1; |
248 | #ifdef RSEQ_COMPARE_TWICE |
249 | error1: |
250 | rseq_bug("cpu_id comparison failed" ); |
251 | #endif |
252 | } |
253 | |
254 | static inline __attribute__((always_inline)) |
255 | int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, |
256 | intptr_t *v2, intptr_t expect2, |
257 | intptr_t newv, int cpu) |
258 | { |
259 | RSEQ_INJECT_C(9) |
260 | |
261 | __asm__ __volatile__ goto ( |
262 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ |
263 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
264 | #ifdef RSEQ_COMPARE_TWICE |
265 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) |
266 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) |
267 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) |
268 | #endif |
269 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
270 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
271 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) |
272 | RSEQ_INJECT_ASM(3) |
273 | "cmpq %[v], %[expect]\n\t" |
274 | "jnz %l[cmpfail]\n\t" |
275 | RSEQ_INJECT_ASM(4) |
276 | "cmpq %[v2], %[expect2]\n\t" |
277 | "jnz %l[cmpfail]\n\t" |
278 | RSEQ_INJECT_ASM(5) |
279 | #ifdef RSEQ_COMPARE_TWICE |
280 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) |
281 | "cmpq %[v], %[expect]\n\t" |
282 | "jnz %l[error2]\n\t" |
283 | "cmpq %[v2], %[expect2]\n\t" |
284 | "jnz %l[error3]\n\t" |
285 | #endif |
286 | /* final store */ |
287 | "movq %[newv], %[v]\n\t" |
288 | "2:\n\t" |
289 | RSEQ_INJECT_ASM(6) |
290 | RSEQ_ASM_DEFINE_ABORT(4, "" , abort) |
291 | : /* gcc asm goto does not allow outputs */ |
292 | : [cpu_id] "r" (cpu), |
293 | [rseq_offset] "r" (rseq_offset), |
294 | /* cmp2 input */ |
295 | [v2] "m" (*v2), |
296 | [expect2] "r" (expect2), |
297 | /* final store input */ |
298 | [v] "m" (*v), |
299 | [expect] "r" (expect), |
300 | [newv] "r" (newv) |
301 | : "memory" , "cc" , "rax" |
302 | RSEQ_INJECT_CLOBBER |
303 | : abort, cmpfail |
304 | #ifdef RSEQ_COMPARE_TWICE |
305 | , error1, error2, error3 |
306 | #endif |
307 | ); |
308 | rseq_after_asm_goto(); |
309 | return 0; |
310 | abort: |
311 | rseq_after_asm_goto(); |
312 | RSEQ_INJECT_FAILED |
313 | return -1; |
314 | cmpfail: |
315 | rseq_after_asm_goto(); |
316 | return 1; |
317 | #ifdef RSEQ_COMPARE_TWICE |
318 | error1: |
319 | rseq_after_asm_goto(); |
320 | rseq_bug("cpu_id comparison failed" ); |
321 | error2: |
322 | rseq_after_asm_goto(); |
323 | rseq_bug("1st expected value comparison failed" ); |
324 | error3: |
325 | rseq_after_asm_goto(); |
326 | rseq_bug("2nd expected value comparison failed" ); |
327 | #endif |
328 | } |
329 | |
330 | #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && |
331 | (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ |
332 | |
333 | #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ |
334 | (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) |
335 | |
336 | static inline __attribute__((always_inline)) |
337 | int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, |
338 | intptr_t *v2, intptr_t newv2, |
339 | intptr_t newv, int cpu) |
340 | { |
341 | RSEQ_INJECT_C(9) |
342 | |
343 | __asm__ __volatile__ goto ( |
344 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ |
345 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
346 | #ifdef RSEQ_COMPARE_TWICE |
347 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) |
348 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) |
349 | #endif |
350 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
351 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
352 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) |
353 | RSEQ_INJECT_ASM(3) |
354 | "cmpq %[v], %[expect]\n\t" |
355 | "jnz %l[cmpfail]\n\t" |
356 | RSEQ_INJECT_ASM(4) |
357 | #ifdef RSEQ_COMPARE_TWICE |
358 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) |
359 | "cmpq %[v], %[expect]\n\t" |
360 | "jnz %l[error2]\n\t" |
361 | #endif |
362 | /* try store */ |
363 | "movq %[newv2], %[v2]\n\t" |
364 | RSEQ_INJECT_ASM(5) |
365 | /* final store */ |
366 | "movq %[newv], %[v]\n\t" |
367 | "2:\n\t" |
368 | RSEQ_INJECT_ASM(6) |
369 | RSEQ_ASM_DEFINE_ABORT(4, "" , abort) |
370 | : /* gcc asm goto does not allow outputs */ |
371 | : [cpu_id] "r" (cpu), |
372 | [rseq_offset] "r" (rseq_offset), |
373 | /* try store input */ |
374 | [v2] "m" (*v2), |
375 | [newv2] "r" (newv2), |
376 | /* final store input */ |
377 | [v] "m" (*v), |
378 | [expect] "r" (expect), |
379 | [newv] "r" (newv) |
380 | : "memory" , "cc" , "rax" |
381 | RSEQ_INJECT_CLOBBER |
382 | : abort, cmpfail |
383 | #ifdef RSEQ_COMPARE_TWICE |
384 | , error1, error2 |
385 | #endif |
386 | ); |
387 | rseq_after_asm_goto(); |
388 | return 0; |
389 | abort: |
390 | rseq_after_asm_goto(); |
391 | RSEQ_INJECT_FAILED |
392 | return -1; |
393 | cmpfail: |
394 | rseq_after_asm_goto(); |
395 | return 1; |
396 | #ifdef RSEQ_COMPARE_TWICE |
397 | error1: |
398 | rseq_after_asm_goto(); |
399 | rseq_bug("cpu_id comparison failed" ); |
400 | error2: |
401 | rseq_after_asm_goto(); |
402 | rseq_bug("expected value comparison failed" ); |
403 | #endif |
404 | } |
405 | |
406 | static inline __attribute__((always_inline)) |
407 | int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, |
408 | void *dst, void *src, size_t len, |
409 | intptr_t newv, int cpu) |
410 | { |
411 | uint64_t rseq_scratch[3]; |
412 | |
413 | RSEQ_INJECT_C(9) |
414 | |
415 | __asm__ __volatile__ goto ( |
416 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ |
417 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
418 | #ifdef RSEQ_COMPARE_TWICE |
419 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) |
420 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) |
421 | #endif |
422 | "movq %[src], %[rseq_scratch0]\n\t" |
423 | "movq %[dst], %[rseq_scratch1]\n\t" |
424 | "movq %[len], %[rseq_scratch2]\n\t" |
425 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
426 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
427 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) |
428 | RSEQ_INJECT_ASM(3) |
429 | "cmpq %[v], %[expect]\n\t" |
430 | "jnz 5f\n\t" |
431 | RSEQ_INJECT_ASM(4) |
432 | #ifdef RSEQ_COMPARE_TWICE |
433 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 6f) |
434 | "cmpq %[v], %[expect]\n\t" |
435 | "jnz 7f\n\t" |
436 | #endif |
437 | /* try memcpy */ |
438 | "test %[len], %[len]\n\t" \ |
439 | "jz 333f\n\t" \ |
440 | "222:\n\t" \ |
441 | "movb (%[src]), %%al\n\t" \ |
442 | "movb %%al, (%[dst])\n\t" \ |
443 | "inc %[src]\n\t" \ |
444 | "inc %[dst]\n\t" \ |
445 | "dec %[len]\n\t" \ |
446 | "jnz 222b\n\t" \ |
447 | "333:\n\t" \ |
448 | RSEQ_INJECT_ASM(5) |
449 | /* final store */ |
450 | "movq %[newv], %[v]\n\t" |
451 | "2:\n\t" |
452 | RSEQ_INJECT_ASM(6) |
453 | /* teardown */ |
454 | "movq %[rseq_scratch2], %[len]\n\t" |
455 | "movq %[rseq_scratch1], %[dst]\n\t" |
456 | "movq %[rseq_scratch0], %[src]\n\t" |
457 | RSEQ_ASM_DEFINE_ABORT(4, |
458 | "movq %[rseq_scratch2], %[len]\n\t" |
459 | "movq %[rseq_scratch1], %[dst]\n\t" |
460 | "movq %[rseq_scratch0], %[src]\n\t" , |
461 | abort) |
462 | RSEQ_ASM_DEFINE_CMPFAIL(5, |
463 | "movq %[rseq_scratch2], %[len]\n\t" |
464 | "movq %[rseq_scratch1], %[dst]\n\t" |
465 | "movq %[rseq_scratch0], %[src]\n\t" , |
466 | cmpfail) |
467 | #ifdef RSEQ_COMPARE_TWICE |
468 | RSEQ_ASM_DEFINE_CMPFAIL(6, |
469 | "movq %[rseq_scratch2], %[len]\n\t" |
470 | "movq %[rseq_scratch1], %[dst]\n\t" |
471 | "movq %[rseq_scratch0], %[src]\n\t" , |
472 | error1) |
473 | RSEQ_ASM_DEFINE_CMPFAIL(7, |
474 | "movq %[rseq_scratch2], %[len]\n\t" |
475 | "movq %[rseq_scratch1], %[dst]\n\t" |
476 | "movq %[rseq_scratch0], %[src]\n\t" , |
477 | error2) |
478 | #endif |
479 | : /* gcc asm goto does not allow outputs */ |
480 | : [cpu_id] "r" (cpu), |
481 | [rseq_offset] "r" (rseq_offset), |
482 | /* final store input */ |
483 | [v] "m" (*v), |
484 | [expect] "r" (expect), |
485 | [newv] "r" (newv), |
486 | /* try memcpy input */ |
487 | [dst] "r" (dst), |
488 | [src] "r" (src), |
489 | [len] "r" (len), |
490 | [rseq_scratch0] "m" (rseq_scratch[0]), |
491 | [rseq_scratch1] "m" (rseq_scratch[1]), |
492 | [rseq_scratch2] "m" (rseq_scratch[2]) |
493 | : "memory" , "cc" , "rax" |
494 | RSEQ_INJECT_CLOBBER |
495 | : abort, cmpfail |
496 | #ifdef RSEQ_COMPARE_TWICE |
497 | , error1, error2 |
498 | #endif |
499 | ); |
500 | rseq_after_asm_goto(); |
501 | return 0; |
502 | abort: |
503 | rseq_after_asm_goto(); |
504 | RSEQ_INJECT_FAILED |
505 | return -1; |
506 | cmpfail: |
507 | rseq_after_asm_goto(); |
508 | return 1; |
509 | #ifdef RSEQ_COMPARE_TWICE |
510 | error1: |
511 | rseq_after_asm_goto(); |
512 | rseq_bug("cpu_id comparison failed" ); |
513 | error2: |
514 | rseq_after_asm_goto(); |
515 | rseq_bug("expected value comparison failed" ); |
516 | #endif |
517 | } |
518 | |
519 | #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && |
520 | (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ |
521 | |
522 | #elif defined(__i386__) |
523 | |
524 | #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ |
525 | (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) |
526 | |
527 | static inline __attribute__((always_inline)) |
528 | int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) |
529 | { |
530 | RSEQ_INJECT_C(9) |
531 | |
532 | __asm__ __volatile__ goto ( |
533 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ |
534 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
535 | #ifdef RSEQ_COMPARE_TWICE |
536 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) |
537 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) |
538 | #endif |
539 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
540 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
541 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) |
542 | RSEQ_INJECT_ASM(3) |
543 | "cmpl %[v], %[expect]\n\t" |
544 | "jnz %l[cmpfail]\n\t" |
545 | RSEQ_INJECT_ASM(4) |
546 | #ifdef RSEQ_COMPARE_TWICE |
547 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) |
548 | "cmpl %[v], %[expect]\n\t" |
549 | "jnz %l[error2]\n\t" |
550 | #endif |
551 | /* final store */ |
552 | "movl %[newv], %[v]\n\t" |
553 | "2:\n\t" |
554 | RSEQ_INJECT_ASM(5) |
555 | RSEQ_ASM_DEFINE_ABORT(4, "" , abort) |
556 | : /* gcc asm goto does not allow outputs */ |
557 | : [cpu_id] "r" (cpu), |
558 | [rseq_offset] "r" (rseq_offset), |
559 | [v] "m" (*v), |
560 | [expect] "r" (expect), |
561 | [newv] "r" (newv) |
562 | : "memory" , "cc" , "eax" |
563 | RSEQ_INJECT_CLOBBER |
564 | : abort, cmpfail |
565 | #ifdef RSEQ_COMPARE_TWICE |
566 | , error1, error2 |
567 | #endif |
568 | ); |
569 | rseq_after_asm_goto(); |
570 | return 0; |
571 | abort: |
572 | rseq_after_asm_goto(); |
573 | RSEQ_INJECT_FAILED |
574 | return -1; |
575 | cmpfail: |
576 | rseq_after_asm_goto(); |
577 | return 1; |
578 | #ifdef RSEQ_COMPARE_TWICE |
579 | error1: |
580 | rseq_after_asm_goto(); |
581 | rseq_bug("cpu_id comparison failed" ); |
582 | error2: |
583 | rseq_after_asm_goto(); |
584 | rseq_bug("expected value comparison failed" ); |
585 | #endif |
586 | } |
587 | |
588 | /* |
589 | * Compare @v against @expectnot. When it does _not_ match, load @v |
590 | * into @load, and store the content of *@v + voffp into @v. |
591 | */ |
592 | static inline __attribute__((always_inline)) |
593 | int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, |
594 | long voffp, intptr_t *load, int cpu) |
595 | { |
596 | RSEQ_INJECT_C(9) |
597 | |
598 | __asm__ __volatile__ goto ( |
599 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ |
600 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
601 | #ifdef RSEQ_COMPARE_TWICE |
602 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) |
603 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) |
604 | #endif |
605 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
606 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
607 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) |
608 | RSEQ_INJECT_ASM(3) |
609 | "movl %[v], %%ebx\n\t" |
610 | "cmpl %%ebx, %[expectnot]\n\t" |
611 | "je %l[cmpfail]\n\t" |
612 | RSEQ_INJECT_ASM(4) |
613 | #ifdef RSEQ_COMPARE_TWICE |
614 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) |
615 | "movl %[v], %%ebx\n\t" |
616 | "cmpl %%ebx, %[expectnot]\n\t" |
617 | "je %l[error2]\n\t" |
618 | #endif |
619 | "movl %%ebx, %[load]\n\t" |
620 | "addl %[voffp], %%ebx\n\t" |
621 | "movl (%%ebx), %%ebx\n\t" |
622 | /* final store */ |
623 | "movl %%ebx, %[v]\n\t" |
624 | "2:\n\t" |
625 | RSEQ_INJECT_ASM(5) |
626 | RSEQ_ASM_DEFINE_ABORT(4, "" , abort) |
627 | : /* gcc asm goto does not allow outputs */ |
628 | : [cpu_id] "r" (cpu), |
629 | [rseq_offset] "r" (rseq_offset), |
630 | /* final store input */ |
631 | [v] "m" (*v), |
632 | [expectnot] "r" (expectnot), |
633 | [voffp] "ir" (voffp), |
634 | [load] "m" (*load) |
635 | : "memory" , "cc" , "eax" , "ebx" |
636 | RSEQ_INJECT_CLOBBER |
637 | : abort, cmpfail |
638 | #ifdef RSEQ_COMPARE_TWICE |
639 | , error1, error2 |
640 | #endif |
641 | ); |
642 | rseq_after_asm_goto(); |
643 | return 0; |
644 | abort: |
645 | rseq_after_asm_goto(); |
646 | RSEQ_INJECT_FAILED |
647 | return -1; |
648 | cmpfail: |
649 | rseq_after_asm_goto(); |
650 | return 1; |
651 | #ifdef RSEQ_COMPARE_TWICE |
652 | error1: |
653 | rseq_after_asm_goto(); |
654 | rseq_bug("cpu_id comparison failed" ); |
655 | error2: |
656 | rseq_after_asm_goto(); |
657 | rseq_bug("expected value comparison failed" ); |
658 | #endif |
659 | } |
660 | |
661 | static inline __attribute__((always_inline)) |
662 | int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) |
663 | { |
664 | RSEQ_INJECT_C(9) |
665 | |
666 | __asm__ __volatile__ goto ( |
667 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ |
668 | #ifdef RSEQ_COMPARE_TWICE |
669 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) |
670 | #endif |
671 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
672 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
673 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) |
674 | RSEQ_INJECT_ASM(3) |
675 | #ifdef RSEQ_COMPARE_TWICE |
676 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) |
677 | #endif |
678 | /* final store */ |
679 | "addl %[count], %[v]\n\t" |
680 | "2:\n\t" |
681 | RSEQ_INJECT_ASM(4) |
682 | RSEQ_ASM_DEFINE_ABORT(4, "" , abort) |
683 | : /* gcc asm goto does not allow outputs */ |
684 | : [cpu_id] "r" (cpu), |
685 | [rseq_offset] "r" (rseq_offset), |
686 | /* final store input */ |
687 | [v] "m" (*v), |
688 | [count] "ir" (count) |
689 | : "memory" , "cc" , "eax" |
690 | RSEQ_INJECT_CLOBBER |
691 | : abort |
692 | #ifdef RSEQ_COMPARE_TWICE |
693 | , error1 |
694 | #endif |
695 | ); |
696 | rseq_after_asm_goto(); |
697 | return 0; |
698 | abort: |
699 | rseq_after_asm_goto(); |
700 | RSEQ_INJECT_FAILED |
701 | return -1; |
702 | #ifdef RSEQ_COMPARE_TWICE |
703 | error1: |
704 | rseq_after_asm_goto(); |
705 | rseq_bug("cpu_id comparison failed" ); |
706 | #endif |
707 | } |
708 | |
709 | static inline __attribute__((always_inline)) |
710 | int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, |
711 | intptr_t *v2, intptr_t expect2, |
712 | intptr_t newv, int cpu) |
713 | { |
714 | RSEQ_INJECT_C(9) |
715 | |
716 | __asm__ __volatile__ goto ( |
717 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ |
718 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
719 | #ifdef RSEQ_COMPARE_TWICE |
720 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) |
721 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) |
722 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) |
723 | #endif |
724 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
725 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
726 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) |
727 | RSEQ_INJECT_ASM(3) |
728 | "cmpl %[v], %[expect]\n\t" |
729 | "jnz %l[cmpfail]\n\t" |
730 | RSEQ_INJECT_ASM(4) |
731 | "cmpl %[expect2], %[v2]\n\t" |
732 | "jnz %l[cmpfail]\n\t" |
733 | RSEQ_INJECT_ASM(5) |
734 | #ifdef RSEQ_COMPARE_TWICE |
735 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) |
736 | "cmpl %[v], %[expect]\n\t" |
737 | "jnz %l[error2]\n\t" |
738 | "cmpl %[expect2], %[v2]\n\t" |
739 | "jnz %l[error3]\n\t" |
740 | #endif |
741 | "movl %[newv], %%eax\n\t" |
742 | /* final store */ |
743 | "movl %%eax, %[v]\n\t" |
744 | "2:\n\t" |
745 | RSEQ_INJECT_ASM(6) |
746 | RSEQ_ASM_DEFINE_ABORT(4, "" , abort) |
747 | : /* gcc asm goto does not allow outputs */ |
748 | : [cpu_id] "r" (cpu), |
749 | [rseq_offset] "r" (rseq_offset), |
750 | /* cmp2 input */ |
751 | [v2] "m" (*v2), |
752 | [expect2] "r" (expect2), |
753 | /* final store input */ |
754 | [v] "m" (*v), |
755 | [expect] "r" (expect), |
756 | [newv] "m" (newv) |
757 | : "memory" , "cc" , "eax" |
758 | RSEQ_INJECT_CLOBBER |
759 | : abort, cmpfail |
760 | #ifdef RSEQ_COMPARE_TWICE |
761 | , error1, error2, error3 |
762 | #endif |
763 | ); |
764 | rseq_after_asm_goto(); |
765 | return 0; |
766 | abort: |
767 | rseq_after_asm_goto(); |
768 | RSEQ_INJECT_FAILED |
769 | return -1; |
770 | cmpfail: |
771 | rseq_after_asm_goto(); |
772 | return 1; |
773 | #ifdef RSEQ_COMPARE_TWICE |
774 | error1: |
775 | rseq_after_asm_goto(); |
776 | rseq_bug("cpu_id comparison failed" ); |
777 | error2: |
778 | rseq_after_asm_goto(); |
779 | rseq_bug("1st expected value comparison failed" ); |
780 | error3: |
781 | rseq_after_asm_goto(); |
782 | rseq_bug("2nd expected value comparison failed" ); |
783 | #endif |
784 | } |
785 | |
786 | #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && |
787 | (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ |
788 | |
789 | #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ |
790 | (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) |
791 | |
792 | static inline __attribute__((always_inline)) |
793 | int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, |
794 | intptr_t *v2, intptr_t newv2, |
795 | intptr_t newv, int cpu) |
796 | { |
797 | RSEQ_INJECT_C(9) |
798 | |
799 | __asm__ __volatile__ goto ( |
800 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ |
801 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
802 | #ifdef RSEQ_COMPARE_TWICE |
803 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) |
804 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) |
805 | #endif |
806 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
807 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
808 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) |
809 | RSEQ_INJECT_ASM(3) |
810 | "movl %[expect], %%eax\n\t" |
811 | "cmpl %[v], %%eax\n\t" |
812 | "jnz %l[cmpfail]\n\t" |
813 | RSEQ_INJECT_ASM(4) |
814 | #ifdef RSEQ_COMPARE_TWICE |
815 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) |
816 | "movl %[expect], %%eax\n\t" |
817 | "cmpl %[v], %%eax\n\t" |
818 | "jnz %l[error2]\n\t" |
819 | #endif |
820 | /* try store */ |
821 | "movl %[newv2], %[v2]\n\t" |
822 | RSEQ_INJECT_ASM(5) |
823 | #ifdef RSEQ_TEMPLATE_MO_RELEASE |
824 | "lock; addl $0,-128(%%esp)\n\t" |
825 | #endif |
826 | /* final store */ |
827 | "movl %[newv], %[v]\n\t" |
828 | "2:\n\t" |
829 | RSEQ_INJECT_ASM(6) |
830 | RSEQ_ASM_DEFINE_ABORT(4, "" , abort) |
831 | : /* gcc asm goto does not allow outputs */ |
832 | : [cpu_id] "r" (cpu), |
833 | [rseq_offset] "r" (rseq_offset), |
834 | /* try store input */ |
835 | [v2] "m" (*v2), |
836 | [newv2] "r" (newv2), |
837 | /* final store input */ |
838 | [v] "m" (*v), |
839 | [expect] "m" (expect), |
840 | [newv] "r" (newv) |
841 | : "memory" , "cc" , "eax" |
842 | RSEQ_INJECT_CLOBBER |
843 | : abort, cmpfail |
844 | #ifdef RSEQ_COMPARE_TWICE |
845 | , error1, error2 |
846 | #endif |
847 | ); |
848 | rseq_after_asm_goto(); |
849 | return 0; |
850 | abort: |
851 | rseq_after_asm_goto(); |
852 | RSEQ_INJECT_FAILED |
853 | return -1; |
854 | cmpfail: |
855 | rseq_after_asm_goto(); |
856 | return 1; |
857 | #ifdef RSEQ_COMPARE_TWICE |
858 | error1: |
859 | rseq_after_asm_goto(); |
860 | rseq_bug("cpu_id comparison failed" ); |
861 | error2: |
862 | rseq_after_asm_goto(); |
863 | rseq_bug("expected value comparison failed" ); |
864 | #endif |
865 | |
866 | } |
867 | |
868 | /* TODO: implement a faster memcpy. */ |
869 | static inline __attribute__((always_inline)) |
870 | int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, |
871 | void *dst, void *src, size_t len, |
872 | intptr_t newv, int cpu) |
873 | { |
874 | uint32_t rseq_scratch[3]; |
875 | |
876 | RSEQ_INJECT_C(9) |
877 | |
878 | __asm__ __volatile__ goto ( |
879 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ |
880 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
881 | #ifdef RSEQ_COMPARE_TWICE |
882 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) |
883 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) |
884 | #endif |
885 | "movl %[src], %[rseq_scratch0]\n\t" |
886 | "movl %[dst], %[rseq_scratch1]\n\t" |
887 | "movl %[len], %[rseq_scratch2]\n\t" |
888 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
889 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
890 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) |
891 | RSEQ_INJECT_ASM(3) |
892 | "movl %[expect], %%eax\n\t" |
893 | "cmpl %%eax, %[v]\n\t" |
894 | "jnz 5f\n\t" |
895 | RSEQ_INJECT_ASM(4) |
896 | #ifdef RSEQ_COMPARE_TWICE |
897 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 6f) |
898 | "movl %[expect], %%eax\n\t" |
899 | "cmpl %%eax, %[v]\n\t" |
900 | "jnz 7f\n\t" |
901 | #endif |
902 | /* try memcpy */ |
903 | "test %[len], %[len]\n\t" \ |
904 | "jz 333f\n\t" \ |
905 | "222:\n\t" \ |
906 | "movb (%[src]), %%al\n\t" \ |
907 | "movb %%al, (%[dst])\n\t" \ |
908 | "inc %[src]\n\t" \ |
909 | "inc %[dst]\n\t" \ |
910 | "dec %[len]\n\t" \ |
911 | "jnz 222b\n\t" \ |
912 | "333:\n\t" \ |
913 | RSEQ_INJECT_ASM(5) |
914 | #ifdef RSEQ_TEMPLATE_MO_RELEASE |
915 | "lock; addl $0,-128(%%esp)\n\t" |
916 | #endif |
917 | "movl %[newv], %%eax\n\t" |
918 | /* final store */ |
919 | "movl %%eax, %[v]\n\t" |
920 | "2:\n\t" |
921 | RSEQ_INJECT_ASM(6) |
922 | /* teardown */ |
923 | "movl %[rseq_scratch2], %[len]\n\t" |
924 | "movl %[rseq_scratch1], %[dst]\n\t" |
925 | "movl %[rseq_scratch0], %[src]\n\t" |
926 | RSEQ_ASM_DEFINE_ABORT(4, |
927 | "movl %[rseq_scratch2], %[len]\n\t" |
928 | "movl %[rseq_scratch1], %[dst]\n\t" |
929 | "movl %[rseq_scratch0], %[src]\n\t" , |
930 | abort) |
931 | RSEQ_ASM_DEFINE_CMPFAIL(5, |
932 | "movl %[rseq_scratch2], %[len]\n\t" |
933 | "movl %[rseq_scratch1], %[dst]\n\t" |
934 | "movl %[rseq_scratch0], %[src]\n\t" , |
935 | cmpfail) |
936 | #ifdef RSEQ_COMPARE_TWICE |
937 | RSEQ_ASM_DEFINE_CMPFAIL(6, |
938 | "movl %[rseq_scratch2], %[len]\n\t" |
939 | "movl %[rseq_scratch1], %[dst]\n\t" |
940 | "movl %[rseq_scratch0], %[src]\n\t" , |
941 | error1) |
942 | RSEQ_ASM_DEFINE_CMPFAIL(7, |
943 | "movl %[rseq_scratch2], %[len]\n\t" |
944 | "movl %[rseq_scratch1], %[dst]\n\t" |
945 | "movl %[rseq_scratch0], %[src]\n\t" , |
946 | error2) |
947 | #endif |
948 | : /* gcc asm goto does not allow outputs */ |
949 | : [cpu_id] "r" (cpu), |
950 | [rseq_offset] "r" (rseq_offset), |
951 | /* final store input */ |
952 | [v] "m" (*v), |
953 | [expect] "m" (expect), |
954 | [newv] "m" (newv), |
955 | /* try memcpy input */ |
956 | [dst] "r" (dst), |
957 | [src] "r" (src), |
958 | [len] "r" (len), |
959 | [rseq_scratch0] "m" (rseq_scratch[0]), |
960 | [rseq_scratch1] "m" (rseq_scratch[1]), |
961 | [rseq_scratch2] "m" (rseq_scratch[2]) |
962 | : "memory" , "cc" , "eax" |
963 | RSEQ_INJECT_CLOBBER |
964 | : abort, cmpfail |
965 | #ifdef RSEQ_COMPARE_TWICE |
966 | , error1, error2 |
967 | #endif |
968 | ); |
969 | rseq_after_asm_goto(); |
970 | return 0; |
971 | abort: |
972 | rseq_after_asm_goto(); |
973 | RSEQ_INJECT_FAILED |
974 | return -1; |
975 | cmpfail: |
976 | rseq_after_asm_goto(); |
977 | return 1; |
978 | #ifdef RSEQ_COMPARE_TWICE |
979 | error1: |
980 | rseq_after_asm_goto(); |
981 | rseq_bug("cpu_id comparison failed" ); |
982 | error2: |
983 | rseq_after_asm_goto(); |
984 | rseq_bug("expected value comparison failed" ); |
985 | #endif |
986 | } |
987 | |
988 | #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && |
989 | (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ |
990 | |
991 | #endif |
992 | |
993 | #include "rseq-bits-reset.h" |
994 | |