1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | /* Copyright (c) 2023 Isovalent */ |
3 | #ifndef __BPF_MPROG_H |
4 | #define __BPF_MPROG_H |
5 | |
6 | #include <linux/bpf.h> |
7 | |
8 | /* bpf_mprog framework: |
9 | * |
10 | * bpf_mprog is a generic layer for multi-program attachment. In-kernel users |
11 | * of the bpf_mprog don't need to care about the dependency resolution |
12 | * internals, they can just consume it with few API calls. Currently available |
13 | * dependency directives are BPF_F_{BEFORE,AFTER} which enable insertion of |
14 | * a BPF program or BPF link relative to an existing BPF program or BPF link |
15 | * inside the multi-program array as well as prepend and append behavior if |
16 | * no relative object was specified, see corresponding selftests for concrete |
17 | * examples (e.g. tc_links and tc_opts test cases of test_progs). |
18 | * |
19 | * Usage of bpf_mprog_{attach,detach,query}() core APIs with pseudo code: |
20 | * |
21 | * Attach case: |
22 | * |
23 | * struct bpf_mprog_entry *entry, *entry_new; |
24 | * int ret; |
25 | * |
26 | * // bpf_mprog user-side lock |
27 | * // fetch active @entry from attach location |
28 | * [...] |
29 | * ret = bpf_mprog_attach(entry, &entry_new, [...]); |
30 | * if (!ret) { |
31 | * if (entry != entry_new) { |
32 | * // swap @entry to @entry_new at attach location |
33 | * // ensure there are no inflight users of @entry: |
34 | * synchronize_rcu(); |
35 | * } |
36 | * bpf_mprog_commit(entry); |
37 | * } else { |
38 | * // error path, bail out, propagate @ret |
39 | * } |
40 | * // bpf_mprog user-side unlock |
41 | * |
42 | * Detach case: |
43 | * |
44 | * struct bpf_mprog_entry *entry, *entry_new; |
45 | * int ret; |
46 | * |
47 | * // bpf_mprog user-side lock |
48 | * // fetch active @entry from attach location |
49 | * [...] |
50 | * ret = bpf_mprog_detach(entry, &entry_new, [...]); |
51 | * if (!ret) { |
52 | * // all (*) marked is optional and depends on the use-case |
53 | * // whether bpf_mprog_bundle should be freed or not |
54 | * if (!bpf_mprog_total(entry_new)) (*) |
55 | * entry_new = NULL (*) |
56 | * // swap @entry to @entry_new at attach location |
57 | * // ensure there are no inflight users of @entry: |
58 | * synchronize_rcu(); |
59 | * bpf_mprog_commit(entry); |
60 | * if (!entry_new) (*) |
61 | * // free bpf_mprog_bundle (*) |
62 | * } else { |
63 | * // error path, bail out, propagate @ret |
64 | * } |
65 | * // bpf_mprog user-side unlock |
66 | * |
67 | * Query case: |
68 | * |
69 | * struct bpf_mprog_entry *entry; |
70 | * int ret; |
71 | * |
72 | * // bpf_mprog user-side lock |
73 | * // fetch active @entry from attach location |
74 | * [...] |
75 | * ret = bpf_mprog_query(attr, uattr, entry); |
76 | * // bpf_mprog user-side unlock |
77 | * |
78 | * Data/fast path: |
79 | * |
80 | * struct bpf_mprog_entry *entry; |
81 | * struct bpf_mprog_fp *fp; |
82 | * struct bpf_prog *prog; |
83 | * int ret = [...]; |
84 | * |
85 | * rcu_read_lock(); |
86 | * // fetch active @entry from attach location |
87 | * [...] |
88 | * bpf_mprog_foreach_prog(entry, fp, prog) { |
89 | * ret = bpf_prog_run(prog, [...]); |
90 | * // process @ret from program |
91 | * } |
92 | * [...] |
93 | * rcu_read_unlock(); |
94 | * |
95 | * bpf_mprog locking considerations: |
96 | * |
97 | * bpf_mprog_{attach,detach,query}() must be protected by an external lock |
98 | * (like RTNL in case of tcx). |
99 | * |
100 | * bpf_mprog_entry pointer can be an __rcu annotated pointer (in case of tcx |
101 | * the netdevice has tcx_ingress and tcx_egress __rcu pointer) which gets |
102 | * updated via rcu_assign_pointer() pointing to the active bpf_mprog_entry of |
103 | * the bpf_mprog_bundle. |
104 | * |
105 | * Fast path accesses the active bpf_mprog_entry within RCU critical section |
106 | * (in case of tcx it runs in NAPI which provides RCU protection there, |
107 | * other users might need explicit rcu_read_lock()). The bpf_mprog_commit() |
108 | * assumes that for the old bpf_mprog_entry there are no inflight users |
109 | * anymore. |
110 | * |
111 | * The READ_ONCE()/WRITE_ONCE() pairing for bpf_mprog_fp's prog access is for |
112 | * the replacement case where we don't swap the bpf_mprog_entry. |
113 | */ |
114 | |
115 | #define bpf_mprog_foreach_tuple(entry, fp, cp, t) \ |
116 | for (fp = &entry->fp_items[0], cp = &entry->parent->cp_items[0];\ |
117 | ({ \ |
118 | t.prog = READ_ONCE(fp->prog); \ |
119 | t.link = cp->link; \ |
120 | t.prog; \ |
121 | }); \ |
122 | fp++, cp++) |
123 | |
124 | #define bpf_mprog_foreach_prog(entry, fp, p) \ |
125 | for (fp = &entry->fp_items[0]; \ |
126 | (p = READ_ONCE(fp->prog)); \ |
127 | fp++) |
128 | |
129 | #define BPF_MPROG_MAX 64 |
130 | |
131 | struct bpf_mprog_fp { |
132 | struct bpf_prog *prog; |
133 | }; |
134 | |
135 | struct bpf_mprog_cp { |
136 | struct bpf_link *link; |
137 | }; |
138 | |
139 | struct bpf_mprog_entry { |
140 | struct bpf_mprog_fp fp_items[BPF_MPROG_MAX]; |
141 | struct bpf_mprog_bundle *parent; |
142 | }; |
143 | |
144 | struct bpf_mprog_bundle { |
145 | struct bpf_mprog_entry a; |
146 | struct bpf_mprog_entry b; |
147 | struct bpf_mprog_cp cp_items[BPF_MPROG_MAX]; |
148 | struct bpf_prog *ref; |
149 | atomic64_t revision; |
150 | u32 count; |
151 | }; |
152 | |
153 | struct bpf_tuple { |
154 | struct bpf_prog *prog; |
155 | struct bpf_link *link; |
156 | }; |
157 | |
158 | static inline struct bpf_mprog_entry * |
159 | bpf_mprog_peer(const struct bpf_mprog_entry *entry) |
160 | { |
161 | if (entry == &entry->parent->a) |
162 | return &entry->parent->b; |
163 | else |
164 | return &entry->parent->a; |
165 | } |
166 | |
167 | static inline void bpf_mprog_bundle_init(struct bpf_mprog_bundle *bundle) |
168 | { |
169 | BUILD_BUG_ON(sizeof(bundle->a.fp_items[0]) > sizeof(u64)); |
170 | BUILD_BUG_ON(ARRAY_SIZE(bundle->a.fp_items) != |
171 | ARRAY_SIZE(bundle->cp_items)); |
172 | |
173 | memset(bundle, 0, sizeof(*bundle)); |
174 | atomic64_set(v: &bundle->revision, i: 1); |
175 | bundle->a.parent = bundle; |
176 | bundle->b.parent = bundle; |
177 | } |
178 | |
179 | static inline void bpf_mprog_inc(struct bpf_mprog_entry *entry) |
180 | { |
181 | entry->parent->count++; |
182 | } |
183 | |
184 | static inline void bpf_mprog_dec(struct bpf_mprog_entry *entry) |
185 | { |
186 | entry->parent->count--; |
187 | } |
188 | |
189 | static inline int bpf_mprog_max(void) |
190 | { |
191 | return ARRAY_SIZE(((struct bpf_mprog_entry *)NULL)->fp_items) - 1; |
192 | } |
193 | |
194 | static inline int bpf_mprog_total(struct bpf_mprog_entry *entry) |
195 | { |
196 | int total = entry->parent->count; |
197 | |
198 | WARN_ON_ONCE(total > bpf_mprog_max()); |
199 | return total; |
200 | } |
201 | |
202 | static inline bool bpf_mprog_exists(struct bpf_mprog_entry *entry, |
203 | struct bpf_prog *prog) |
204 | { |
205 | const struct bpf_mprog_fp *fp; |
206 | const struct bpf_prog *tmp; |
207 | |
208 | bpf_mprog_foreach_prog(entry, fp, tmp) { |
209 | if (tmp == prog) |
210 | return true; |
211 | } |
212 | return false; |
213 | } |
214 | |
215 | static inline void bpf_mprog_mark_for_release(struct bpf_mprog_entry *entry, |
216 | struct bpf_tuple *tuple) |
217 | { |
218 | WARN_ON_ONCE(entry->parent->ref); |
219 | if (!tuple->link) |
220 | entry->parent->ref = tuple->prog; |
221 | } |
222 | |
223 | static inline void bpf_mprog_complete_release(struct bpf_mprog_entry *entry) |
224 | { |
225 | /* In the non-link case prog deletions can only drop the reference |
226 | * to the prog after the bpf_mprog_entry got swapped and the |
227 | * bpf_mprog ensured that there are no inflight users anymore. |
228 | * |
229 | * Paired with bpf_mprog_mark_for_release(). |
230 | */ |
231 | if (entry->parent->ref) { |
232 | bpf_prog_put(prog: entry->parent->ref); |
233 | entry->parent->ref = NULL; |
234 | } |
235 | } |
236 | |
237 | static inline void bpf_mprog_revision_new(struct bpf_mprog_entry *entry) |
238 | { |
239 | atomic64_inc(v: &entry->parent->revision); |
240 | } |
241 | |
242 | static inline void bpf_mprog_commit(struct bpf_mprog_entry *entry) |
243 | { |
244 | bpf_mprog_complete_release(entry); |
245 | bpf_mprog_revision_new(entry); |
246 | } |
247 | |
248 | static inline u64 bpf_mprog_revision(struct bpf_mprog_entry *entry) |
249 | { |
250 | return atomic64_read(v: &entry->parent->revision); |
251 | } |
252 | |
253 | static inline void bpf_mprog_entry_copy(struct bpf_mprog_entry *dst, |
254 | struct bpf_mprog_entry *src) |
255 | { |
256 | memcpy(dst->fp_items, src->fp_items, sizeof(src->fp_items)); |
257 | } |
258 | |
259 | static inline void bpf_mprog_entry_clear(struct bpf_mprog_entry *dst) |
260 | { |
261 | memset(dst->fp_items, 0, sizeof(dst->fp_items)); |
262 | } |
263 | |
264 | static inline void bpf_mprog_clear_all(struct bpf_mprog_entry *entry, |
265 | struct bpf_mprog_entry **entry_new) |
266 | { |
267 | struct bpf_mprog_entry *peer; |
268 | |
269 | peer = bpf_mprog_peer(entry); |
270 | bpf_mprog_entry_clear(dst: peer); |
271 | peer->parent->count = 0; |
272 | *entry_new = peer; |
273 | } |
274 | |
275 | static inline void bpf_mprog_entry_grow(struct bpf_mprog_entry *entry, int idx) |
276 | { |
277 | int total = bpf_mprog_total(entry); |
278 | |
279 | memmove(entry->fp_items + idx + 1, |
280 | entry->fp_items + idx, |
281 | (total - idx) * sizeof(struct bpf_mprog_fp)); |
282 | |
283 | memmove(entry->parent->cp_items + idx + 1, |
284 | entry->parent->cp_items + idx, |
285 | (total - idx) * sizeof(struct bpf_mprog_cp)); |
286 | } |
287 | |
288 | static inline void bpf_mprog_entry_shrink(struct bpf_mprog_entry *entry, int idx) |
289 | { |
290 | /* Total array size is needed in this case to enure the NULL |
291 | * entry is copied at the end. |
292 | */ |
293 | int total = ARRAY_SIZE(entry->fp_items); |
294 | |
295 | memmove(entry->fp_items + idx, |
296 | entry->fp_items + idx + 1, |
297 | (total - idx - 1) * sizeof(struct bpf_mprog_fp)); |
298 | |
299 | memmove(entry->parent->cp_items + idx, |
300 | entry->parent->cp_items + idx + 1, |
301 | (total - idx - 1) * sizeof(struct bpf_mprog_cp)); |
302 | } |
303 | |
304 | static inline void bpf_mprog_read(struct bpf_mprog_entry *entry, u32 idx, |
305 | struct bpf_mprog_fp **fp, |
306 | struct bpf_mprog_cp **cp) |
307 | { |
308 | *fp = &entry->fp_items[idx]; |
309 | *cp = &entry->parent->cp_items[idx]; |
310 | } |
311 | |
312 | static inline void bpf_mprog_write(struct bpf_mprog_fp *fp, |
313 | struct bpf_mprog_cp *cp, |
314 | struct bpf_tuple *tuple) |
315 | { |
316 | WRITE_ONCE(fp->prog, tuple->prog); |
317 | cp->link = tuple->link; |
318 | } |
319 | |
320 | int bpf_mprog_attach(struct bpf_mprog_entry *entry, |
321 | struct bpf_mprog_entry **entry_new, |
322 | struct bpf_prog *prog_new, struct bpf_link *link, |
323 | struct bpf_prog *prog_old, |
324 | u32 flags, u32 id_or_fd, u64 revision); |
325 | |
326 | int bpf_mprog_detach(struct bpf_mprog_entry *entry, |
327 | struct bpf_mprog_entry **entry_new, |
328 | struct bpf_prog *prog, struct bpf_link *link, |
329 | u32 flags, u32 id_or_fd, u64 revision); |
330 | |
331 | int bpf_mprog_query(const union bpf_attr *attr, union bpf_attr __user *uattr, |
332 | struct bpf_mprog_entry *entry); |
333 | |
334 | static inline bool bpf_mprog_supported(enum bpf_prog_type type) |
335 | { |
336 | switch (type) { |
337 | case BPF_PROG_TYPE_SCHED_CLS: |
338 | return true; |
339 | default: |
340 | return false; |
341 | } |
342 | } |
343 | #endif /* __BPF_MPROG_H */ |
344 | |