Warning: This file is not a C or C++ file. It does not have highlighting.
1 | #ifndef __BPF_EXPERIMENTAL__ |
---|---|
2 | #define __BPF_EXPERIMENTAL__ |
3 | |
4 | #include <vmlinux.h> |
5 | #include <bpf/bpf_tracing.h> |
6 | #include <bpf/bpf_helpers.h> |
7 | #include <bpf/bpf_core_read.h> |
8 | |
9 | #define __contains(name, node) __attribute__((btf_decl_tag("contains:" #name ":" #node))) |
10 | |
11 | /* Description |
12 | * Allocates an object of the type represented by 'local_type_id' in |
13 | * program BTF. User may use the bpf_core_type_id_local macro to pass the |
14 | * type ID of a struct in program BTF. |
15 | * |
16 | * The 'local_type_id' parameter must be a known constant. |
17 | * The 'meta' parameter is rewritten by the verifier, no need for BPF |
18 | * program to set it. |
19 | * Returns |
20 | * A pointer to an object of the type corresponding to the passed in |
21 | * 'local_type_id', or NULL on failure. |
22 | */ |
23 | extern void *bpf_obj_new_impl(__u64 local_type_id, void *meta) __ksym; |
24 | |
25 | /* Convenience macro to wrap over bpf_obj_new_impl */ |
26 | #define bpf_obj_new(type) ((type *)bpf_obj_new_impl(bpf_core_type_id_local(type), NULL)) |
27 | |
28 | /* Description |
29 | * Free an allocated object. All fields of the object that require |
30 | * destruction will be destructed before the storage is freed. |
31 | * |
32 | * The 'meta' parameter is rewritten by the verifier, no need for BPF |
33 | * program to set it. |
34 | * Returns |
35 | * Void. |
36 | */ |
37 | extern void bpf_obj_drop_impl(void *kptr, void *meta) __ksym; |
38 | |
39 | /* Convenience macro to wrap over bpf_obj_drop_impl */ |
40 | #define bpf_obj_drop(kptr) bpf_obj_drop_impl(kptr, NULL) |
41 | |
42 | /* Description |
43 | * Increment the refcount on a refcounted local kptr, turning the |
44 | * non-owning reference input into an owning reference in the process. |
45 | * |
46 | * The 'meta' parameter is rewritten by the verifier, no need for BPF |
47 | * program to set it. |
48 | * Returns |
49 | * An owning reference to the object pointed to by 'kptr' |
50 | */ |
51 | extern void *bpf_refcount_acquire_impl(void *kptr, void *meta) __ksym; |
52 | |
53 | /* Convenience macro to wrap over bpf_refcount_acquire_impl */ |
54 | #define bpf_refcount_acquire(kptr) bpf_refcount_acquire_impl(kptr, NULL) |
55 | |
56 | /* Description |
57 | * Add a new entry to the beginning of the BPF linked list. |
58 | * |
59 | * The 'meta' and 'off' parameters are rewritten by the verifier, no need |
60 | * for BPF programs to set them |
61 | * Returns |
62 | * 0 if the node was successfully added |
63 | * -EINVAL if the node wasn't added because it's already in a list |
64 | */ |
65 | extern int bpf_list_push_front_impl(struct bpf_list_head *head, |
66 | struct bpf_list_node *node, |
67 | void *meta, __u64 off) __ksym; |
68 | |
69 | /* Convenience macro to wrap over bpf_list_push_front_impl */ |
70 | #define bpf_list_push_front(head, node) bpf_list_push_front_impl(head, node, NULL, 0) |
71 | |
72 | /* Description |
73 | * Add a new entry to the end of the BPF linked list. |
74 | * |
75 | * The 'meta' and 'off' parameters are rewritten by the verifier, no need |
76 | * for BPF programs to set them |
77 | * Returns |
78 | * 0 if the node was successfully added |
79 | * -EINVAL if the node wasn't added because it's already in a list |
80 | */ |
81 | extern int bpf_list_push_back_impl(struct bpf_list_head *head, |
82 | struct bpf_list_node *node, |
83 | void *meta, __u64 off) __ksym; |
84 | |
85 | /* Convenience macro to wrap over bpf_list_push_back_impl */ |
86 | #define bpf_list_push_back(head, node) bpf_list_push_back_impl(head, node, NULL, 0) |
87 | |
88 | /* Description |
89 | * Remove the entry at the beginning of the BPF linked list. |
90 | * Returns |
91 | * Pointer to bpf_list_node of deleted entry, or NULL if list is empty. |
92 | */ |
93 | extern struct bpf_list_node *bpf_list_pop_front(struct bpf_list_head *head) __ksym; |
94 | |
95 | /* Description |
96 | * Remove the entry at the end of the BPF linked list. |
97 | * Returns |
98 | * Pointer to bpf_list_node of deleted entry, or NULL if list is empty. |
99 | */ |
100 | extern struct bpf_list_node *bpf_list_pop_back(struct bpf_list_head *head) __ksym; |
101 | |
102 | /* Description |
103 | * Remove 'node' from rbtree with root 'root' |
104 | * Returns |
105 | * Pointer to the removed node, or NULL if 'root' didn't contain 'node' |
106 | */ |
107 | extern struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root, |
108 | struct bpf_rb_node *node) __ksym; |
109 | |
110 | /* Description |
111 | * Add 'node' to rbtree with root 'root' using comparator 'less' |
112 | * |
113 | * The 'meta' and 'off' parameters are rewritten by the verifier, no need |
114 | * for BPF programs to set them |
115 | * Returns |
116 | * 0 if the node was successfully added |
117 | * -EINVAL if the node wasn't added because it's already in a tree |
118 | */ |
119 | extern int bpf_rbtree_add_impl(struct bpf_rb_root *root, struct bpf_rb_node *node, |
120 | bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b), |
121 | void *meta, __u64 off) __ksym; |
122 | |
123 | /* Convenience macro to wrap over bpf_rbtree_add_impl */ |
124 | #define bpf_rbtree_add(head, node, less) bpf_rbtree_add_impl(head, node, less, NULL, 0) |
125 | |
126 | /* Description |
127 | * Return the first (leftmost) node in input tree |
128 | * Returns |
129 | * Pointer to the node, which is _not_ removed from the tree. If the tree |
130 | * contains no nodes, returns NULL. |
131 | */ |
132 | extern struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) __ksym; |
133 | |
134 | /* Description |
135 | * Allocates a percpu object of the type represented by 'local_type_id' in |
136 | * program BTF. User may use the bpf_core_type_id_local macro to pass the |
137 | * type ID of a struct in program BTF. |
138 | * |
139 | * The 'local_type_id' parameter must be a known constant. |
140 | * The 'meta' parameter is rewritten by the verifier, no need for BPF |
141 | * program to set it. |
142 | * Returns |
143 | * A pointer to a percpu object of the type corresponding to the passed in |
144 | * 'local_type_id', or NULL on failure. |
145 | */ |
146 | extern void *bpf_percpu_obj_new_impl(__u64 local_type_id, void *meta) __ksym; |
147 | |
148 | /* Convenience macro to wrap over bpf_percpu_obj_new_impl */ |
149 | #define bpf_percpu_obj_new(type) ((type __percpu_kptr *)bpf_percpu_obj_new_impl(bpf_core_type_id_local(type), NULL)) |
150 | |
151 | /* Description |
152 | * Free an allocated percpu object. All fields of the object that require |
153 | * destruction will be destructed before the storage is freed. |
154 | * |
155 | * The 'meta' parameter is rewritten by the verifier, no need for BPF |
156 | * program to set it. |
157 | * Returns |
158 | * Void. |
159 | */ |
160 | extern void bpf_percpu_obj_drop_impl(void *kptr, void *meta) __ksym; |
161 | |
162 | struct bpf_iter_task_vma; |
163 | |
164 | extern int bpf_iter_task_vma_new(struct bpf_iter_task_vma *it, |
165 | struct task_struct *task, |
166 | unsigned long addr) __ksym; |
167 | extern struct vm_area_struct *bpf_iter_task_vma_next(struct bpf_iter_task_vma *it) __ksym; |
168 | extern void bpf_iter_task_vma_destroy(struct bpf_iter_task_vma *it) __ksym; |
169 | |
170 | /* Convenience macro to wrap over bpf_obj_drop_impl */ |
171 | #define bpf_percpu_obj_drop(kptr) bpf_percpu_obj_drop_impl(kptr, NULL) |
172 | |
173 | /* Description |
174 | * Throw a BPF exception from the program, immediately terminating its |
175 | * execution and unwinding the stack. The supplied 'cookie' parameter |
176 | * will be the return value of the program when an exception is thrown, |
177 | * and the default exception callback is used. Otherwise, if an exception |
178 | * callback is set using the '__exception_cb(callback)' declaration tag |
179 | * on the main program, the 'cookie' parameter will be the callback's only |
180 | * input argument. |
181 | * |
182 | * Thus, in case of default exception callback, 'cookie' is subjected to |
183 | * constraints on the program's return value (as with R0 on exit). |
184 | * Otherwise, the return value of the marked exception callback will be |
185 | * subjected to the same checks. |
186 | * |
187 | * Note that throwing an exception with lingering resources (locks, |
188 | * references, etc.) will lead to a verification error. |
189 | * |
190 | * Note that callbacks *cannot* call this helper. |
191 | * Returns |
192 | * Never. |
193 | * Throws |
194 | * An exception with the specified 'cookie' value. |
195 | */ |
196 | extern void bpf_throw(u64 cookie) __ksym; |
197 | |
198 | /* This macro must be used to mark the exception callback corresponding to the |
199 | * main program. For example: |
200 | * |
201 | * int exception_cb(u64 cookie) { |
202 | * return cookie; |
203 | * } |
204 | * |
205 | * SEC("tc") |
206 | * __exception_cb(exception_cb) |
207 | * int main_prog(struct __sk_buff *ctx) { |
208 | * ... |
209 | * return TC_ACT_OK; |
210 | * } |
211 | * |
212 | * Here, exception callback for the main program will be 'exception_cb'. Note |
213 | * that this attribute can only be used once, and multiple exception callbacks |
214 | * specified for the main program will lead to verification error. |
215 | */ |
216 | #define __exception_cb(name) __attribute__((btf_decl_tag("exception_callback:" #name))) |
217 | |
218 | #define __bpf_assert_signed(x) _Generic((x), \ |
219 | unsigned long: 0, \ |
220 | unsigned long long: 0, \ |
221 | signed long: 1, \ |
222 | signed long long: 1 \ |
223 | ) |
224 | |
225 | #define __bpf_assert_check(LHS, op, RHS) \ |
226 | _Static_assert(sizeof(&(LHS)), "1st argument must be an lvalue expression"); \ |
227 | _Static_assert(sizeof(LHS) == 8, "Only 8-byte integers are supported\n"); \ |
228 | _Static_assert(__builtin_constant_p(__bpf_assert_signed(LHS)), "internal static assert"); \ |
229 | _Static_assert(__builtin_constant_p((RHS)), "2nd argument must be a constant expression") |
230 | |
231 | #define __bpf_assert(LHS, op, cons, RHS, VAL) \ |
232 | ({ \ |
233 | (void)bpf_throw; \ |
234 | asm volatile ("if %[lhs] " op " %[rhs] goto +2; r1 = %[value]; call bpf_throw" \ |
235 | : : [lhs] "r"(LHS), [rhs] cons(RHS), [value] "ri"(VAL) : ); \ |
236 | }) |
237 | |
238 | #define __bpf_assert_op_sign(LHS, op, cons, RHS, VAL, supp_sign) \ |
239 | ({ \ |
240 | __bpf_assert_check(LHS, op, RHS); \ |
241 | if (__bpf_assert_signed(LHS) && !(supp_sign)) \ |
242 | __bpf_assert(LHS, "s" #op, cons, RHS, VAL); \ |
243 | else \ |
244 | __bpf_assert(LHS, #op, cons, RHS, VAL); \ |
245 | }) |
246 | |
247 | #define __bpf_assert_op(LHS, op, RHS, VAL, supp_sign) \ |
248 | ({ \ |
249 | if (sizeof(typeof(RHS)) == 8) { \ |
250 | const typeof(RHS) rhs_var = (RHS); \ |
251 | __bpf_assert_op_sign(LHS, op, "r", rhs_var, VAL, supp_sign); \ |
252 | } else { \ |
253 | __bpf_assert_op_sign(LHS, op, "i", RHS, VAL, supp_sign); \ |
254 | } \ |
255 | }) |
256 | |
257 | #define __cmp_cannot_be_signed(x) \ |
258 | __builtin_strcmp(#x, "==") == 0 || __builtin_strcmp(#x, "!=") == 0 || \ |
259 | __builtin_strcmp(#x, "&") == 0 |
260 | |
261 | #define __is_signed_type(type) (((type)(-1)) < (type)1) |
262 | |
263 | #define __bpf_cmp(LHS, OP, PRED, RHS, DEFAULT) \ |
264 | ({ \ |
265 | __label__ l_true; \ |
266 | bool ret = DEFAULT; \ |
267 | asm volatile goto("if %[lhs] " OP " %[rhs] goto %l[l_true]" \ |
268 | :: [lhs] "r"((short)LHS), [rhs] PRED (RHS) :: l_true); \ |
269 | ret = !DEFAULT; \ |
270 | l_true: \ |
271 | ret; \ |
272 | }) |
273 | |
274 | /* C type conversions coupled with comparison operator are tricky. |
275 | * Make sure BPF program is compiled with -Wsign-compare then |
276 | * __lhs OP __rhs below will catch the mistake. |
277 | * Be aware that we check only __lhs to figure out the sign of compare. |
278 | */ |
279 | #define _bpf_cmp(LHS, OP, RHS, UNLIKELY) \ |
280 | ({ \ |
281 | typeof(LHS) __lhs = (LHS); \ |
282 | typeof(RHS) __rhs = (RHS); \ |
283 | bool ret; \ |
284 | _Static_assert(sizeof(&(LHS)), "1st argument must be an lvalue expression"); \ |
285 | (void)(__lhs OP __rhs); \ |
286 | if (__cmp_cannot_be_signed(OP) || !__is_signed_type(typeof(__lhs))) { \ |
287 | if (sizeof(__rhs) == 8) \ |
288 | /* "i" will truncate 64-bit constant into s32, \ |
289 | * so we have to use extra register via "r". \ |
290 | */ \ |
291 | ret = __bpf_cmp(__lhs, #OP, "r", __rhs, UNLIKELY); \ |
292 | else \ |
293 | ret = __bpf_cmp(__lhs, #OP, "ri", __rhs, UNLIKELY); \ |
294 | } else { \ |
295 | if (sizeof(__rhs) == 8) \ |
296 | ret = __bpf_cmp(__lhs, "s"#OP, "r", __rhs, UNLIKELY); \ |
297 | else \ |
298 | ret = __bpf_cmp(__lhs, "s"#OP, "ri", __rhs, UNLIKELY); \ |
299 | } \ |
300 | ret; \ |
301 | }) |
302 | |
303 | #ifndef bpf_cmp_unlikely |
304 | #define bpf_cmp_unlikely(LHS, OP, RHS) _bpf_cmp(LHS, OP, RHS, true) |
305 | #endif |
306 | |
307 | #ifndef bpf_cmp_likely |
308 | #define bpf_cmp_likely(LHS, OP, RHS) \ |
309 | ({ \ |
310 | bool ret = 0; \ |
311 | if (__builtin_strcmp(#OP, "==") == 0) \ |
312 | ret = _bpf_cmp(LHS, !=, RHS, false); \ |
313 | else if (__builtin_strcmp(#OP, "!=") == 0) \ |
314 | ret = _bpf_cmp(LHS, ==, RHS, false); \ |
315 | else if (__builtin_strcmp(#OP, "<=") == 0) \ |
316 | ret = _bpf_cmp(LHS, >, RHS, false); \ |
317 | else if (__builtin_strcmp(#OP, "<") == 0) \ |
318 | ret = _bpf_cmp(LHS, >=, RHS, false); \ |
319 | else if (__builtin_strcmp(#OP, ">") == 0) \ |
320 | ret = _bpf_cmp(LHS, <=, RHS, false); \ |
321 | else if (__builtin_strcmp(#OP, ">=") == 0) \ |
322 | ret = _bpf_cmp(LHS, <, RHS, false); \ |
323 | else \ |
324 | asm volatile("r0 " #OP " invalid compare"); \ |
325 | ret; \ |
326 | }) |
327 | #endif |
328 | |
329 | #define cond_break \ |
330 | ({ __label__ l_break, l_continue; \ |
331 | asm volatile goto("1:.byte 0xe5; \ |
332 | .byte 0; \ |
333 | .long ((%l[l_break] - 1b - 8) / 8) & 0xffff; \ |
334 | .short 0" \ |
335 | :::: l_break); \ |
336 | goto l_continue; \ |
337 | l_break: break; \ |
338 | l_continue:; \ |
339 | }) |
340 | |
341 | #ifndef bpf_nop_mov |
342 | #define bpf_nop_mov(var) \ |
343 | asm volatile("%[reg]=%[reg]"::[reg]"r"((short)var)) |
344 | #endif |
345 | |
346 | /* emit instruction: |
347 | * rX = rX .off = BPF_ADDR_SPACE_CAST .imm32 = (dst_as << 16) | src_as |
348 | */ |
349 | #ifndef bpf_addr_space_cast |
350 | #define bpf_addr_space_cast(var, dst_as, src_as)\ |
351 | asm volatile(".byte 0xBF; \ |
352 | .ifc %[reg], r0; \ |
353 | .byte 0x00; \ |
354 | .endif; \ |
355 | .ifc %[reg], r1; \ |
356 | .byte 0x11; \ |
357 | .endif; \ |
358 | .ifc %[reg], r2; \ |
359 | .byte 0x22; \ |
360 | .endif; \ |
361 | .ifc %[reg], r3; \ |
362 | .byte 0x33; \ |
363 | .endif; \ |
364 | .ifc %[reg], r4; \ |
365 | .byte 0x44; \ |
366 | .endif; \ |
367 | .ifc %[reg], r5; \ |
368 | .byte 0x55; \ |
369 | .endif; \ |
370 | .ifc %[reg], r6; \ |
371 | .byte 0x66; \ |
372 | .endif; \ |
373 | .ifc %[reg], r7; \ |
374 | .byte 0x77; \ |
375 | .endif; \ |
376 | .ifc %[reg], r8; \ |
377 | .byte 0x88; \ |
378 | .endif; \ |
379 | .ifc %[reg], r9; \ |
380 | .byte 0x99; \ |
381 | .endif; \ |
382 | .short %[off]; \ |
383 | .long %[as]" \ |
384 | : [reg]"+r"(var) \ |
385 | : [off]"i"(BPF_ADDR_SPACE_CAST) \ |
386 | , [as]"i"((dst_as << 16) | src_as)); |
387 | #endif |
388 | |
389 | /* Description |
390 | * Assert that a conditional expression is true. |
391 | * Returns |
392 | * Void. |
393 | * Throws |
394 | * An exception with the value zero when the assertion fails. |
395 | */ |
396 | #define bpf_assert(cond) if (!(cond)) bpf_throw(0); |
397 | |
398 | /* Description |
399 | * Assert that a conditional expression is true. |
400 | * Returns |
401 | * Void. |
402 | * Throws |
403 | * An exception with the specified value when the assertion fails. |
404 | */ |
405 | #define bpf_assert_with(cond, value) if (!(cond)) bpf_throw(value); |
406 | |
407 | /* Description |
408 | * Assert that LHS is in the range [BEG, END] (inclusive of both). This |
409 | * statement updates the known bounds of LHS during verification. Note |
410 | * that both BEG and END must be constant values, and must fit within the |
411 | * data type of LHS. |
412 | * Returns |
413 | * Void. |
414 | * Throws |
415 | * An exception with the value zero when the assertion fails. |
416 | */ |
417 | #define bpf_assert_range(LHS, BEG, END) \ |
418 | ({ \ |
419 | _Static_assert(BEG <= END, "BEG must be <= END"); \ |
420 | barrier_var(LHS); \ |
421 | __bpf_assert_op(LHS, >=, BEG, 0, false); \ |
422 | __bpf_assert_op(LHS, <=, END, 0, false); \ |
423 | }) |
424 | |
425 | /* Description |
426 | * Assert that LHS is in the range [BEG, END] (inclusive of both). This |
427 | * statement updates the known bounds of LHS during verification. Note |
428 | * that both BEG and END must be constant values, and must fit within the |
429 | * data type of LHS. |
430 | * Returns |
431 | * Void. |
432 | * Throws |
433 | * An exception with the specified value when the assertion fails. |
434 | */ |
435 | #define bpf_assert_range_with(LHS, BEG, END, value) \ |
436 | ({ \ |
437 | _Static_assert(BEG <= END, "BEG must be <= END"); \ |
438 | barrier_var(LHS); \ |
439 | __bpf_assert_op(LHS, >=, BEG, value, false); \ |
440 | __bpf_assert_op(LHS, <=, END, value, false); \ |
441 | }) |
442 | |
443 | struct bpf_iter_css_task; |
444 | struct cgroup_subsys_state; |
445 | extern int bpf_iter_css_task_new(struct bpf_iter_css_task *it, |
446 | struct cgroup_subsys_state *css, unsigned int flags) __weak __ksym; |
447 | extern struct task_struct *bpf_iter_css_task_next(struct bpf_iter_css_task *it) __weak __ksym; |
448 | extern void bpf_iter_css_task_destroy(struct bpf_iter_css_task *it) __weak __ksym; |
449 | |
450 | struct bpf_iter_task; |
451 | extern int bpf_iter_task_new(struct bpf_iter_task *it, |
452 | struct task_struct *task, unsigned int flags) __weak __ksym; |
453 | extern struct task_struct *bpf_iter_task_next(struct bpf_iter_task *it) __weak __ksym; |
454 | extern void bpf_iter_task_destroy(struct bpf_iter_task *it) __weak __ksym; |
455 | |
456 | struct bpf_iter_css; |
457 | extern int bpf_iter_css_new(struct bpf_iter_css *it, |
458 | struct cgroup_subsys_state *start, unsigned int flags) __weak __ksym; |
459 | extern struct cgroup_subsys_state *bpf_iter_css_next(struct bpf_iter_css *it) __weak __ksym; |
460 | extern void bpf_iter_css_destroy(struct bpf_iter_css *it) __weak __ksym; |
461 | |
462 | #endif |
463 |
Warning: This file is not a C or C++ file. It does not have highlighting.