1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | #ifndef _LINUX_STATIC_CALL_H |
3 | #define _LINUX_STATIC_CALL_H |
4 | |
5 | /* |
6 | * Static call support |
7 | * |
8 | * Static calls use code patching to hard-code function pointers into direct |
9 | * branch instructions. They give the flexibility of function pointers, but |
10 | * with improved performance. This is especially important for cases where |
11 | * retpolines would otherwise be used, as retpolines can significantly impact |
12 | * performance. |
13 | * |
14 | * |
15 | * API overview: |
16 | * |
17 | * DECLARE_STATIC_CALL(name, func); |
18 | * DEFINE_STATIC_CALL(name, func); |
19 | * DEFINE_STATIC_CALL_NULL(name, typename); |
20 | * DEFINE_STATIC_CALL_RET0(name, typename); |
21 | * |
22 | * __static_call_return0; |
23 | * |
24 | * static_call(name)(args...); |
25 | * static_call_cond(name)(args...); |
26 | * static_call_update(name, func); |
27 | * static_call_query(name); |
28 | * |
29 | * EXPORT_STATIC_CALL{,_TRAMP}{,_GPL}() |
30 | * |
31 | * Usage example: |
32 | * |
33 | * # Start with the following functions (with identical prototypes): |
34 | * int func_a(int arg1, int arg2); |
35 | * int func_b(int arg1, int arg2); |
36 | * |
37 | * # Define a 'my_name' reference, associated with func_a() by default |
38 | * DEFINE_STATIC_CALL(my_name, func_a); |
39 | * |
40 | * # Call func_a() |
41 | * static_call(my_name)(arg1, arg2); |
42 | * |
43 | * # Update 'my_name' to point to func_b() |
44 | * static_call_update(my_name, &func_b); |
45 | * |
46 | * # Call func_b() |
47 | * static_call(my_name)(arg1, arg2); |
48 | * |
49 | * |
50 | * Implementation details: |
51 | * |
52 | * This requires some arch-specific code (CONFIG_HAVE_STATIC_CALL). |
53 | * Otherwise basic indirect calls are used (with function pointers). |
54 | * |
55 | * Each static_call() site calls into a trampoline associated with the name. |
56 | * The trampoline has a direct branch to the default function. Updates to a |
57 | * name will modify the trampoline's branch destination. |
58 | * |
59 | * If the arch has CONFIG_HAVE_STATIC_CALL_INLINE, then the call sites |
60 | * themselves will be patched at runtime to call the functions directly, |
61 | * rather than calling through the trampoline. This requires objtool or a |
62 | * compiler plugin to detect all the static_call() sites and annotate them |
63 | * in the .static_call_sites section. |
64 | * |
65 | * |
66 | * Notes on NULL function pointers: |
67 | * |
68 | * Static_call()s support NULL functions, with many of the caveats that |
69 | * regular function pointers have. |
70 | * |
71 | * Clearly calling a NULL function pointer is 'BAD', so too for |
72 | * static_call()s (although when HAVE_STATIC_CALL it might not be immediately |
73 | * fatal). A NULL static_call can be the result of: |
74 | * |
75 | * DECLARE_STATIC_CALL_NULL(my_static_call, void (*)(int)); |
76 | * |
77 | * which is equivalent to declaring a NULL function pointer with just a |
78 | * typename: |
79 | * |
80 | * void (*my_func_ptr)(int arg1) = NULL; |
81 | * |
82 | * or using static_call_update() with a NULL function. In both cases the |
83 | * HAVE_STATIC_CALL implementation will patch the trampoline with a RET |
84 | * instruction, instead of an immediate tail-call JMP. HAVE_STATIC_CALL_INLINE |
85 | * architectures can patch the trampoline call to a NOP. |
86 | * |
87 | * In all cases, any argument evaluation is unconditional. Unlike a regular |
88 | * conditional function pointer call: |
89 | * |
90 | * if (my_func_ptr) |
91 | * my_func_ptr(arg1) |
92 | * |
93 | * where the argument evaludation also depends on the pointer value. |
94 | * |
95 | * When calling a static_call that can be NULL, use: |
96 | * |
97 | * static_call_cond(name)(arg1); |
98 | * |
99 | * which will include the required value tests to avoid NULL-pointer |
100 | * dereferences. |
101 | * |
102 | * To query which function is currently set to be called, use: |
103 | * |
104 | * func = static_call_query(name); |
105 | * |
106 | * |
107 | * DEFINE_STATIC_CALL_RET0 / __static_call_return0: |
108 | * |
109 | * Just like how DEFINE_STATIC_CALL_NULL() / static_call_cond() optimize the |
110 | * conditional void function call, DEFINE_STATIC_CALL_RET0 / |
111 | * __static_call_return0 optimize the do nothing return 0 function. |
112 | * |
113 | * This feature is strictly UB per the C standard (since it casts a function |
114 | * pointer to a different signature) and relies on the architecture ABI to |
115 | * make things work. In particular it relies on Caller Stack-cleanup and the |
116 | * whole return register being clobbered for short return values. All normal |
117 | * CDECL style ABIs conform. |
118 | * |
119 | * In particular the x86_64 implementation replaces the 5 byte CALL |
120 | * instruction at the callsite with a 5 byte clear of the RAX register, |
121 | * completely eliding any function call overhead. |
122 | * |
123 | * Notably argument setup is unconditional. |
124 | * |
125 | * |
126 | * EXPORT_STATIC_CALL() vs EXPORT_STATIC_CALL_TRAMP(): |
127 | * |
128 | * The difference is that the _TRAMP variant tries to only export the |
129 | * trampoline with the result that a module can use static_call{,_cond}() but |
130 | * not static_call_update(). |
131 | * |
132 | */ |
133 | |
134 | #include <linux/types.h> |
135 | #include <linux/cpu.h> |
136 | #include <linux/static_call_types.h> |
137 | |
138 | #ifdef CONFIG_HAVE_STATIC_CALL |
139 | #include <asm/static_call.h> |
140 | |
141 | /* |
142 | * Either @site or @tramp can be NULL. |
143 | */ |
144 | extern void arch_static_call_transform(void *site, void *tramp, void *func, bool tail); |
145 | |
146 | #define STATIC_CALL_TRAMP_ADDR(name) &STATIC_CALL_TRAMP(name) |
147 | |
148 | #else |
149 | #define STATIC_CALL_TRAMP_ADDR(name) NULL |
150 | #endif |
151 | |
152 | #define static_call_update(name, func) \ |
153 | ({ \ |
154 | typeof(&STATIC_CALL_TRAMP(name)) __F = (func); \ |
155 | __static_call_update(&STATIC_CALL_KEY(name), \ |
156 | STATIC_CALL_TRAMP_ADDR(name), __F); \ |
157 | }) |
158 | |
159 | #define static_call_query(name) (READ_ONCE(STATIC_CALL_KEY(name).func)) |
160 | |
161 | #ifdef CONFIG_HAVE_STATIC_CALL_INLINE |
162 | |
163 | extern int __init static_call_init(void); |
164 | |
165 | extern void static_call_force_reinit(void); |
166 | |
167 | struct static_call_mod { |
168 | struct static_call_mod *next; |
169 | struct module *mod; /* for vmlinux, mod == NULL */ |
170 | struct static_call_site *sites; |
171 | }; |
172 | |
173 | /* For finding the key associated with a trampoline */ |
174 | struct static_call_tramp_key { |
175 | s32 tramp; |
176 | s32 key; |
177 | }; |
178 | |
179 | extern void __static_call_update(struct static_call_key *key, void *tramp, void *func); |
180 | extern int static_call_mod_init(struct module *mod); |
181 | extern int static_call_text_reserved(void *start, void *end); |
182 | |
183 | extern long __static_call_return0(void); |
184 | |
185 | #define DEFINE_STATIC_CALL(name, _func) \ |
186 | DECLARE_STATIC_CALL(name, _func); \ |
187 | struct static_call_key STATIC_CALL_KEY(name) = { \ |
188 | .func = _func, \ |
189 | .type = 1, \ |
190 | }; \ |
191 | ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func) |
192 | |
193 | #define DEFINE_STATIC_CALL_NULL(name, _func) \ |
194 | DECLARE_STATIC_CALL(name, _func); \ |
195 | struct static_call_key STATIC_CALL_KEY(name) = { \ |
196 | .func = NULL, \ |
197 | .type = 1, \ |
198 | }; \ |
199 | ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name) |
200 | |
201 | #define DEFINE_STATIC_CALL_RET0(name, _func) \ |
202 | DECLARE_STATIC_CALL(name, _func); \ |
203 | struct static_call_key STATIC_CALL_KEY(name) = { \ |
204 | .func = __static_call_return0, \ |
205 | .type = 1, \ |
206 | }; \ |
207 | ARCH_DEFINE_STATIC_CALL_RET0_TRAMP(name) |
208 | |
209 | #define static_call_cond(name) (void)__static_call(name) |
210 | |
211 | #define EXPORT_STATIC_CALL(name) \ |
212 | EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \ |
213 | EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)) |
214 | #define EXPORT_STATIC_CALL_GPL(name) \ |
215 | EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)); \ |
216 | EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)) |
217 | |
218 | /* Leave the key unexported, so modules can't change static call targets: */ |
219 | #define EXPORT_STATIC_CALL_TRAMP(name) \ |
220 | EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)); \ |
221 | ARCH_ADD_TRAMP_KEY(name) |
222 | #define EXPORT_STATIC_CALL_TRAMP_GPL(name) \ |
223 | EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)); \ |
224 | ARCH_ADD_TRAMP_KEY(name) |
225 | |
226 | #elif defined(CONFIG_HAVE_STATIC_CALL) |
227 | |
228 | static inline int static_call_init(void) { return 0; } |
229 | |
230 | #define DEFINE_STATIC_CALL(name, _func) \ |
231 | DECLARE_STATIC_CALL(name, _func); \ |
232 | struct static_call_key STATIC_CALL_KEY(name) = { \ |
233 | .func = _func, \ |
234 | }; \ |
235 | ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func) |
236 | |
237 | #define DEFINE_STATIC_CALL_NULL(name, _func) \ |
238 | DECLARE_STATIC_CALL(name, _func); \ |
239 | struct static_call_key STATIC_CALL_KEY(name) = { \ |
240 | .func = NULL, \ |
241 | }; \ |
242 | ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name) |
243 | |
244 | #define DEFINE_STATIC_CALL_RET0(name, _func) \ |
245 | DECLARE_STATIC_CALL(name, _func); \ |
246 | struct static_call_key STATIC_CALL_KEY(name) = { \ |
247 | .func = __static_call_return0, \ |
248 | }; \ |
249 | ARCH_DEFINE_STATIC_CALL_RET0_TRAMP(name) |
250 | |
251 | #define static_call_cond(name) (void)__static_call(name) |
252 | |
253 | static inline |
254 | void __static_call_update(struct static_call_key *key, void *tramp, void *func) |
255 | { |
256 | cpus_read_lock(); |
257 | WRITE_ONCE(key->func, func); |
258 | arch_static_call_transform(NULL, tramp, func, false); |
259 | cpus_read_unlock(); |
260 | } |
261 | |
262 | static inline int static_call_text_reserved(void *start, void *end) |
263 | { |
264 | return 0; |
265 | } |
266 | |
267 | extern long __static_call_return0(void); |
268 | |
269 | #define EXPORT_STATIC_CALL(name) \ |
270 | EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \ |
271 | EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)) |
272 | #define EXPORT_STATIC_CALL_GPL(name) \ |
273 | EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)); \ |
274 | EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)) |
275 | |
276 | /* Leave the key unexported, so modules can't change static call targets: */ |
277 | #define EXPORT_STATIC_CALL_TRAMP(name) \ |
278 | EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)) |
279 | #define EXPORT_STATIC_CALL_TRAMP_GPL(name) \ |
280 | EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)) |
281 | |
282 | #else /* Generic implementation */ |
283 | |
284 | static inline int static_call_init(void) { return 0; } |
285 | |
286 | static inline long __static_call_return0(void) |
287 | { |
288 | return 0; |
289 | } |
290 | |
291 | #define __DEFINE_STATIC_CALL(name, _func, _func_init) \ |
292 | DECLARE_STATIC_CALL(name, _func); \ |
293 | struct static_call_key STATIC_CALL_KEY(name) = { \ |
294 | .func = _func_init, \ |
295 | } |
296 | |
297 | #define DEFINE_STATIC_CALL(name, _func) \ |
298 | __DEFINE_STATIC_CALL(name, _func, _func) |
299 | |
300 | #define DEFINE_STATIC_CALL_NULL(name, _func) \ |
301 | __DEFINE_STATIC_CALL(name, _func, NULL) |
302 | |
303 | #define DEFINE_STATIC_CALL_RET0(name, _func) \ |
304 | __DEFINE_STATIC_CALL(name, _func, __static_call_return0) |
305 | |
306 | static inline void __static_call_nop(void) { } |
307 | |
308 | /* |
309 | * This horrific hack takes care of two things: |
310 | * |
311 | * - it ensures the compiler will only load the function pointer ONCE, |
312 | * which avoids a reload race. |
313 | * |
314 | * - it ensures the argument evaluation is unconditional, similar |
315 | * to the HAVE_STATIC_CALL variant. |
316 | * |
317 | * Sadly current GCC/Clang (10 for both) do not optimize this properly |
318 | * and will emit an indirect call for the NULL case :-( |
319 | */ |
320 | #define __static_call_cond(name) \ |
321 | ({ \ |
322 | void *func = READ_ONCE(STATIC_CALL_KEY(name).func); \ |
323 | if (!func) \ |
324 | func = &__static_call_nop; \ |
325 | (typeof(STATIC_CALL_TRAMP(name))*)func; \ |
326 | }) |
327 | |
328 | #define static_call_cond(name) (void)__static_call_cond(name) |
329 | |
330 | static inline |
331 | void __static_call_update(struct static_call_key *key, void *tramp, void *func) |
332 | { |
333 | WRITE_ONCE(key->func, func); |
334 | } |
335 | |
336 | static inline int static_call_text_reserved(void *start, void *end) |
337 | { |
338 | return 0; |
339 | } |
340 | |
341 | #define EXPORT_STATIC_CALL(name) EXPORT_SYMBOL(STATIC_CALL_KEY(name)) |
342 | #define EXPORT_STATIC_CALL_GPL(name) EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)) |
343 | |
344 | #endif /* CONFIG_HAVE_STATIC_CALL */ |
345 | |
346 | #endif /* _LINUX_STATIC_CALL_H */ |
347 | |