1 | /* SPDX-License-Identifier: LGPL-2.1 OR MIT */ |
2 | /* |
3 | * rseq-x86.h |
4 | * |
5 | * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> |
6 | */ |
7 | |
8 | #ifndef RSEQ_H |
9 | #error "Never use <rseq-x86.h> directly; include <rseq.h> instead." |
10 | #endif |
11 | |
12 | #include <stdint.h> |
13 | |
14 | /* |
15 | * RSEQ_SIG is used with the following reserved undefined instructions, which |
16 | * trap in user-space: |
17 | * |
18 | * x86-32: 0f b9 3d 53 30 05 53 ud1 0x53053053,%edi |
19 | * x86-64: 0f b9 3d 53 30 05 53 ud1 0x53053053(%rip),%edi |
20 | */ |
21 | #define RSEQ_SIG 0x53053053 |
22 | |
23 | /* |
24 | * Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input |
25 | * operands, we cannot use "m" input operands, and rather pass the __rseq_abi |
26 | * address through a "r" input operand. |
27 | */ |
28 | |
29 | /* Offset of cpu_id, rseq_cs, and mm_cid fields in struct rseq. */ |
30 | #define RSEQ_CPU_ID_OFFSET 4 |
31 | #define RSEQ_CS_OFFSET 8 |
32 | #define RSEQ_MM_CID_OFFSET 24 |
33 | |
34 | #ifdef __x86_64__ |
35 | |
36 | #define RSEQ_ASM_TP_SEGMENT %%fs |
37 | |
38 | #define rseq_smp_mb() \ |
39 | __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc") |
40 | #define rseq_smp_rmb() rseq_barrier() |
41 | #define rseq_smp_wmb() rseq_barrier() |
42 | |
43 | #define rseq_smp_load_acquire(p) \ |
44 | __extension__ ({ \ |
45 | rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p)); \ |
46 | rseq_barrier(); \ |
47 | ____p1; \ |
48 | }) |
49 | |
50 | #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() |
51 | |
52 | #define rseq_smp_store_release(p, v) \ |
53 | do { \ |
54 | rseq_barrier(); \ |
55 | RSEQ_WRITE_ONCE(*(p), v); \ |
56 | } while (0) |
57 | |
58 | #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ |
59 | start_ip, post_commit_offset, abort_ip) \ |
60 | ".pushsection __rseq_cs, \"aw\"\n\t" \ |
61 | ".balign 32\n\t" \ |
62 | __rseq_str(label) ":\n\t" \ |
63 | ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ |
64 | ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \ |
65 | ".popsection\n\t" \ |
66 | ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ |
67 | ".quad " __rseq_str(label) "b\n\t" \ |
68 | ".popsection\n\t" |
69 | |
70 | |
71 | #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ |
72 | __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ |
73 | (post_commit_ip - start_ip), abort_ip) |
74 | |
75 | /* |
76 | * Exit points of a rseq critical section consist of all instructions outside |
77 | * of the critical section where a critical section can either branch to or |
78 | * reach through the normal course of its execution. The abort IP and the |
79 | * post-commit IP are already part of the __rseq_cs section and should not be |
80 | * explicitly defined as additional exit points. Knowing all exit points is |
81 | * useful to assist debuggers stepping over the critical section. |
82 | */ |
83 | #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ |
84 | ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ |
85 | ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \ |
86 | ".popsection\n\t" |
87 | |
88 | #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ |
89 | RSEQ_INJECT_ASM(1) \ |
90 | "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t" \ |
91 | "movq %%rax, " __rseq_str(rseq_cs) "\n\t" \ |
92 | __rseq_str(label) ":\n\t" |
93 | |
94 | #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ |
95 | RSEQ_INJECT_ASM(2) \ |
96 | "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \ |
97 | "jnz " __rseq_str(label) "\n\t" |
98 | |
99 | #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \ |
100 | ".pushsection __rseq_failure, \"ax\"\n\t" \ |
101 | /* Disassembler-friendly signature: ud1 <sig>(%rip),%edi. */ \ |
102 | ".byte 0x0f, 0xb9, 0x3d\n\t" \ |
103 | ".long " __rseq_str(RSEQ_SIG) "\n\t" \ |
104 | __rseq_str(label) ":\n\t" \ |
105 | teardown \ |
106 | "jmp %l[" __rseq_str(abort_label) "]\n\t" \ |
107 | ".popsection\n\t" |
108 | |
109 | #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \ |
110 | ".pushsection __rseq_failure, \"ax\"\n\t" \ |
111 | __rseq_str(label) ":\n\t" \ |
112 | teardown \ |
113 | "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \ |
114 | ".popsection\n\t" |
115 | |
116 | #elif defined(__i386__) |
117 | |
118 | #define RSEQ_ASM_TP_SEGMENT %%gs |
119 | |
120 | #define rseq_smp_mb() \ |
121 | __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc") |
122 | #define rseq_smp_rmb() \ |
123 | __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc") |
124 | #define rseq_smp_wmb() \ |
125 | __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc") |
126 | |
127 | #define rseq_smp_load_acquire(p) \ |
128 | __extension__ ({ \ |
129 | __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ |
130 | rseq_smp_mb(); \ |
131 | ____p1; \ |
132 | }) |
133 | |
134 | #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() |
135 | |
136 | #define rseq_smp_store_release(p, v) \ |
137 | do { \ |
138 | rseq_smp_mb(); \ |
139 | RSEQ_WRITE_ONCE(*p, v); \ |
140 | } while (0) |
141 | |
142 | /* |
143 | * Use eax as scratch register and take memory operands as input to |
144 | * lessen register pressure. Especially needed when compiling in O0. |
145 | */ |
146 | #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ |
147 | start_ip, post_commit_offset, abort_ip) \ |
148 | ".pushsection __rseq_cs, \"aw\"\n\t" \ |
149 | ".balign 32\n\t" \ |
150 | __rseq_str(label) ":\n\t" \ |
151 | ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ |
152 | ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \ |
153 | ".popsection\n\t" \ |
154 | ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ |
155 | ".long " __rseq_str(label) "b, 0x0\n\t" \ |
156 | ".popsection\n\t" |
157 | |
158 | #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ |
159 | __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ |
160 | (post_commit_ip - start_ip), abort_ip) |
161 | |
162 | /* |
163 | * Exit points of a rseq critical section consist of all instructions outside |
164 | * of the critical section where a critical section can either branch to or |
165 | * reach through the normal course of its execution. The abort IP and the |
166 | * post-commit IP are already part of the __rseq_cs section and should not be |
167 | * explicitly defined as additional exit points. Knowing all exit points is |
168 | * useful to assist debuggers stepping over the critical section. |
169 | */ |
170 | #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ |
171 | ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ |
172 | ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \ |
173 | ".popsection\n\t" |
174 | |
175 | #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ |
176 | RSEQ_INJECT_ASM(1) \ |
177 | "movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t" \ |
178 | __rseq_str(label) ":\n\t" |
179 | |
180 | #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ |
181 | RSEQ_INJECT_ASM(2) \ |
182 | "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \ |
183 | "jnz " __rseq_str(label) "\n\t" |
184 | |
185 | #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \ |
186 | ".pushsection __rseq_failure, \"ax\"\n\t" \ |
187 | /* Disassembler-friendly signature: ud1 <sig>,%edi. */ \ |
188 | ".byte 0x0f, 0xb9, 0x3d\n\t" \ |
189 | ".long " __rseq_str(RSEQ_SIG) "\n\t" \ |
190 | __rseq_str(label) ":\n\t" \ |
191 | teardown \ |
192 | "jmp %l[" __rseq_str(abort_label) "]\n\t" \ |
193 | ".popsection\n\t" |
194 | |
195 | #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \ |
196 | ".pushsection __rseq_failure, \"ax\"\n\t" \ |
197 | __rseq_str(label) ":\n\t" \ |
198 | teardown \ |
199 | "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \ |
200 | ".popsection\n\t" |
201 | |
202 | #endif |
203 | |
204 | /* Per-cpu-id indexing. */ |
205 | |
206 | #define RSEQ_TEMPLATE_CPU_ID |
207 | #define RSEQ_TEMPLATE_MO_RELAXED |
208 | #include "rseq-x86-bits.h" |
209 | #undef RSEQ_TEMPLATE_MO_RELAXED |
210 | |
211 | #define RSEQ_TEMPLATE_MO_RELEASE |
212 | #include "rseq-x86-bits.h" |
213 | #undef RSEQ_TEMPLATE_MO_RELEASE |
214 | #undef RSEQ_TEMPLATE_CPU_ID |
215 | |
216 | /* Per-mm-cid indexing. */ |
217 | |
218 | #define RSEQ_TEMPLATE_MM_CID |
219 | #define RSEQ_TEMPLATE_MO_RELAXED |
220 | #include "rseq-x86-bits.h" |
221 | #undef RSEQ_TEMPLATE_MO_RELAXED |
222 | |
223 | #define RSEQ_TEMPLATE_MO_RELEASE |
224 | #include "rseq-x86-bits.h" |
225 | #undef RSEQ_TEMPLATE_MO_RELEASE |
226 | #undef RSEQ_TEMPLATE_MM_CID |
227 | |
228 | /* APIs which are not based on cpu ids. */ |
229 | |
230 | #define RSEQ_TEMPLATE_CPU_ID_NONE |
231 | #define RSEQ_TEMPLATE_MO_RELAXED |
232 | #include "rseq-x86-bits.h" |
233 | #undef RSEQ_TEMPLATE_MO_RELAXED |
234 | #undef RSEQ_TEMPLATE_CPU_ID_NONE |
235 | |