1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (c) 2017 Facebook
3 */
4#include <linux/bpf.h>
5#include <linux/btf.h>
6#include <linux/btf_ids.h>
7#include <linux/slab.h>
8#include <linux/init.h>
9#include <linux/vmalloc.h>
10#include <linux/etherdevice.h>
11#include <linux/filter.h>
12#include <linux/rcupdate_trace.h>
13#include <linux/sched/signal.h>
14#include <net/bpf_sk_storage.h>
15#include <net/hotdata.h>
16#include <net/sock.h>
17#include <net/tcp.h>
18#include <net/net_namespace.h>
19#include <net/page_pool/helpers.h>
20#include <linux/error-injection.h>
21#include <linux/smp.h>
22#include <linux/sock_diag.h>
23#include <linux/netfilter.h>
24#include <net/netdev_rx_queue.h>
25#include <net/xdp.h>
26#include <net/netfilter/nf_bpf_link.h>
27
28#define CREATE_TRACE_POINTS
29#include <trace/events/bpf_test_run.h>
30
31struct bpf_test_timer {
32 u32 i;
33 u64 time_start, time_spent;
34};
35
36static void bpf_test_timer_enter(struct bpf_test_timer *t)
37 __acquires(rcu)
38{
39 rcu_read_lock_dont_migrate();
40 t->time_start = ktime_get_ns();
41}
42
43static void bpf_test_timer_leave(struct bpf_test_timer *t)
44 __releases(rcu)
45{
46 t->time_start = 0;
47 rcu_read_unlock_migrate();
48}
49
50static bool bpf_test_timer_continue(struct bpf_test_timer *t, int iterations,
51 u32 repeat, int *err, u32 *duration)
52 __must_hold(rcu)
53{
54 t->i += iterations;
55 if (t->i >= repeat) {
56 /* We're done. */
57 t->time_spent += ktime_get_ns() - t->time_start;
58 do_div(t->time_spent, t->i);
59 *duration = t->time_spent > U32_MAX ? U32_MAX : (u32)t->time_spent;
60 *err = 0;
61 goto reset;
62 }
63
64 if (signal_pending(current)) {
65 /* During iteration: we've been cancelled, abort. */
66 *err = -EINTR;
67 goto reset;
68 }
69
70 if (need_resched()) {
71 /* During iteration: we need to reschedule between runs. */
72 t->time_spent += ktime_get_ns() - t->time_start;
73 bpf_test_timer_leave(t);
74 cond_resched();
75 bpf_test_timer_enter(t);
76 }
77
78 /* Do another round. */
79 return true;
80
81reset:
82 t->i = 0;
83 return false;
84}
85
86/* We put this struct at the head of each page with a context and frame
87 * initialised when the page is allocated, so we don't have to do this on each
88 * repetition of the test run.
89 */
90struct xdp_page_head {
91 struct xdp_buff orig_ctx;
92 struct xdp_buff ctx;
93 union {
94 /* ::data_hard_start starts here */
95 DECLARE_FLEX_ARRAY(struct xdp_frame, frame);
96 DECLARE_FLEX_ARRAY(u8, data);
97 };
98};
99
100struct xdp_test_data {
101 struct xdp_buff *orig_ctx;
102 struct xdp_rxq_info rxq;
103 struct net_device *dev;
104 struct page_pool *pp;
105 struct xdp_frame **frames;
106 struct sk_buff **skbs;
107 struct xdp_mem_info mem;
108 u32 batch_size;
109 u32 frame_cnt;
110};
111
112/* tools/testing/selftests/bpf/prog_tests/xdp_do_redirect.c:%MAX_PKT_SIZE
113 * must be updated accordingly this gets changed, otherwise BPF selftests
114 * will fail.
115 */
116#define TEST_XDP_FRAME_SIZE (PAGE_SIZE - sizeof(struct xdp_page_head))
117#define TEST_XDP_MAX_BATCH 256
118
119static void xdp_test_run_init_page(netmem_ref netmem, void *arg)
120{
121 struct xdp_page_head *head =
122 phys_to_virt(page_to_phys(netmem_to_page(netmem)));
123 struct xdp_buff *new_ctx, *orig_ctx;
124 u32 headroom = XDP_PACKET_HEADROOM;
125 struct xdp_test_data *xdp = arg;
126 size_t frm_len, meta_len;
127 struct xdp_frame *frm;
128 void *data;
129
130 orig_ctx = xdp->orig_ctx;
131 frm_len = orig_ctx->data_end - orig_ctx->data_meta;
132 meta_len = orig_ctx->data - orig_ctx->data_meta;
133 headroom -= meta_len;
134
135 new_ctx = &head->ctx;
136 frm = head->frame;
137 data = head->data;
138 memcpy(data + headroom, orig_ctx->data_meta, frm_len);
139
140 xdp_init_buff(xdp: new_ctx, TEST_XDP_FRAME_SIZE, rxq: &xdp->rxq);
141 xdp_prepare_buff(xdp: new_ctx, hard_start: data, headroom, data_len: frm_len, meta_valid: true);
142 new_ctx->data = new_ctx->data_meta + meta_len;
143
144 xdp_update_frame_from_buff(xdp: new_ctx, xdp_frame: frm);
145 frm->mem_type = new_ctx->rxq->mem.type;
146
147 memcpy(&head->orig_ctx, new_ctx, sizeof(head->orig_ctx));
148}
149
150static int xdp_test_run_setup(struct xdp_test_data *xdp, struct xdp_buff *orig_ctx)
151{
152 struct page_pool *pp;
153 int err = -ENOMEM;
154 struct page_pool_params pp_params = {
155 .order = 0,
156 .flags = 0,
157 .pool_size = xdp->batch_size,
158 .nid = NUMA_NO_NODE,
159 .init_callback = xdp_test_run_init_page,
160 .init_arg = xdp,
161 };
162
163 xdp->frames = kvmalloc_array(xdp->batch_size, sizeof(void *), GFP_KERNEL);
164 if (!xdp->frames)
165 return -ENOMEM;
166
167 xdp->skbs = kvmalloc_array(xdp->batch_size, sizeof(void *), GFP_KERNEL);
168 if (!xdp->skbs)
169 goto err_skbs;
170
171 pp = page_pool_create(params: &pp_params);
172 if (IS_ERR(ptr: pp)) {
173 err = PTR_ERR(ptr: pp);
174 goto err_pp;
175 }
176
177 /* will copy 'mem.id' into pp->xdp_mem_id */
178 err = xdp_reg_mem_model(mem: &xdp->mem, type: MEM_TYPE_PAGE_POOL, allocator: pp);
179 if (err)
180 goto err_mmodel;
181
182 xdp->pp = pp;
183
184 /* We create a 'fake' RXQ referencing the original dev, but with an
185 * xdp_mem_info pointing to our page_pool
186 */
187 xdp_rxq_info_reg(xdp_rxq: &xdp->rxq, dev: orig_ctx->rxq->dev, queue_index: 0, napi_id: 0);
188 xdp->rxq.mem.type = MEM_TYPE_PAGE_POOL;
189 xdp->rxq.mem.id = pp->xdp_mem_id;
190 xdp->dev = orig_ctx->rxq->dev;
191 xdp->orig_ctx = orig_ctx;
192
193 return 0;
194
195err_mmodel:
196 page_pool_destroy(pool: pp);
197err_pp:
198 kvfree(addr: xdp->skbs);
199err_skbs:
200 kvfree(addr: xdp->frames);
201 return err;
202}
203
204static void xdp_test_run_teardown(struct xdp_test_data *xdp)
205{
206 xdp_unreg_mem_model(mem: &xdp->mem);
207 page_pool_destroy(pool: xdp->pp);
208 kfree(objp: xdp->frames);
209 kfree(objp: xdp->skbs);
210}
211
212static bool frame_was_changed(const struct xdp_page_head *head)
213{
214 /* xdp_scrub_frame() zeroes the data pointer, flags is the last field,
215 * i.e. has the highest chances to be overwritten. If those two are
216 * untouched, it's most likely safe to skip the context reset.
217 */
218 return head->frame->data != head->orig_ctx.data ||
219 head->frame->flags != head->orig_ctx.flags;
220}
221
222static bool ctx_was_changed(struct xdp_page_head *head)
223{
224 return head->orig_ctx.data != head->ctx.data ||
225 head->orig_ctx.data_meta != head->ctx.data_meta ||
226 head->orig_ctx.data_end != head->ctx.data_end;
227}
228
229static void reset_ctx(struct xdp_page_head *head)
230{
231 if (likely(!frame_was_changed(head) && !ctx_was_changed(head)))
232 return;
233
234 head->ctx.data = head->orig_ctx.data;
235 head->ctx.data_meta = head->orig_ctx.data_meta;
236 head->ctx.data_end = head->orig_ctx.data_end;
237 xdp_update_frame_from_buff(xdp: &head->ctx, xdp_frame: head->frame);
238 head->frame->mem_type = head->orig_ctx.rxq->mem.type;
239}
240
241static int xdp_recv_frames(struct xdp_frame **frames, int nframes,
242 struct sk_buff **skbs,
243 struct net_device *dev)
244{
245 gfp_t gfp = __GFP_ZERO | GFP_ATOMIC;
246 int i, n;
247 LIST_HEAD(list);
248
249 n = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, gfp, nframes,
250 (void **)skbs);
251 if (unlikely(n == 0)) {
252 for (i = 0; i < nframes; i++)
253 xdp_return_frame(xdpf: frames[i]);
254 return -ENOMEM;
255 }
256
257 for (i = 0; i < nframes; i++) {
258 struct xdp_frame *xdpf = frames[i];
259 struct sk_buff *skb = skbs[i];
260
261 skb = __xdp_build_skb_from_frame(xdpf, skb, dev);
262 if (!skb) {
263 xdp_return_frame(xdpf);
264 continue;
265 }
266
267 list_add_tail(new: &skb->list, head: &list);
268 }
269 netif_receive_skb_list(head: &list);
270
271 return 0;
272}
273
274static int xdp_test_run_batch(struct xdp_test_data *xdp, struct bpf_prog *prog,
275 u32 repeat)
276{
277 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
278 int err = 0, act, ret, i, nframes = 0, batch_sz;
279 struct xdp_frame **frames = xdp->frames;
280 struct bpf_redirect_info *ri;
281 struct xdp_page_head *head;
282 struct xdp_frame *frm;
283 bool redirect = false;
284 struct xdp_buff *ctx;
285 struct page *page;
286
287 batch_sz = min_t(u32, repeat, xdp->batch_size);
288
289 local_bh_disable();
290 bpf_net_ctx = bpf_net_ctx_set(bpf_net_ctx: &__bpf_net_ctx);
291 ri = bpf_net_ctx_get_ri();
292 xdp_set_return_frame_no_direct();
293
294 for (i = 0; i < batch_sz; i++) {
295 page = page_pool_dev_alloc_pages(pool: xdp->pp);
296 if (!page) {
297 err = -ENOMEM;
298 goto out;
299 }
300
301 head = phys_to_virt(page_to_phys(page));
302 reset_ctx(head);
303 ctx = &head->ctx;
304 frm = head->frame;
305 xdp->frame_cnt++;
306
307 act = bpf_prog_run_xdp(prog, xdp: ctx);
308
309 /* if program changed pkt bounds we need to update the xdp_frame */
310 if (unlikely(ctx_was_changed(head))) {
311 ret = xdp_update_frame_from_buff(xdp: ctx, xdp_frame: frm);
312 if (ret) {
313 xdp_return_buff(xdp: ctx);
314 continue;
315 }
316 }
317
318 switch (act) {
319 case XDP_TX:
320 /* we can't do a real XDP_TX since we're not in the
321 * driver, so turn it into a REDIRECT back to the same
322 * index
323 */
324 ri->tgt_index = xdp->dev->ifindex;
325 ri->map_id = INT_MAX;
326 ri->map_type = BPF_MAP_TYPE_UNSPEC;
327 fallthrough;
328 case XDP_REDIRECT:
329 redirect = true;
330 ret = xdp_do_redirect_frame(dev: xdp->dev, xdp: ctx, xdpf: frm, prog);
331 if (ret)
332 xdp_return_buff(xdp: ctx);
333 break;
334 case XDP_PASS:
335 frames[nframes++] = frm;
336 break;
337 default:
338 bpf_warn_invalid_xdp_action(NULL, prog, act);
339 fallthrough;
340 case XDP_DROP:
341 xdp_return_buff(xdp: ctx);
342 break;
343 }
344 }
345
346out:
347 if (redirect)
348 xdp_do_flush();
349 if (nframes) {
350 ret = xdp_recv_frames(frames, nframes, skbs: xdp->skbs, dev: xdp->dev);
351 if (ret)
352 err = ret;
353 }
354
355 xdp_clear_return_frame_no_direct();
356 bpf_net_ctx_clear(bpf_net_ctx);
357 local_bh_enable();
358 return err;
359}
360
361static int bpf_test_run_xdp_live(struct bpf_prog *prog, struct xdp_buff *ctx,
362 u32 repeat, u32 batch_size, u32 *time)
363
364{
365 struct xdp_test_data xdp = { .batch_size = batch_size };
366 struct bpf_test_timer t = {};
367 int ret;
368
369 if (!repeat)
370 repeat = 1;
371
372 ret = xdp_test_run_setup(xdp: &xdp, orig_ctx: ctx);
373 if (ret)
374 return ret;
375
376 bpf_test_timer_enter(t: &t);
377 do {
378 xdp.frame_cnt = 0;
379 ret = xdp_test_run_batch(xdp: &xdp, prog, repeat: repeat - t.i);
380 if (unlikely(ret < 0))
381 break;
382 } while (bpf_test_timer_continue(t: &t, iterations: xdp.frame_cnt, repeat, err: &ret, duration: time));
383 bpf_test_timer_leave(t: &t);
384
385 xdp_test_run_teardown(xdp: &xdp);
386 return ret;
387}
388
389static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat,
390 u32 *retval, u32 *time, bool xdp)
391{
392 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
393 struct bpf_prog_array_item item = {.prog = prog};
394 struct bpf_run_ctx *old_ctx;
395 struct bpf_cg_run_ctx run_ctx;
396 struct bpf_test_timer t = {};
397 enum bpf_cgroup_storage_type stype;
398 int ret;
399
400 for_each_cgroup_storage_type(stype) {
401 item.cgroup_storage[stype] = bpf_cgroup_storage_alloc(prog, stype);
402 if (IS_ERR(ptr: item.cgroup_storage[stype])) {
403 item.cgroup_storage[stype] = NULL;
404 for_each_cgroup_storage_type(stype)
405 bpf_cgroup_storage_free(storage: item.cgroup_storage[stype]);
406 return -ENOMEM;
407 }
408 }
409
410 if (!repeat)
411 repeat = 1;
412
413 bpf_test_timer_enter(t: &t);
414 old_ctx = bpf_set_run_ctx(new_ctx: &run_ctx.run_ctx);
415 do {
416 run_ctx.prog_item = &item;
417 local_bh_disable();
418 bpf_net_ctx = bpf_net_ctx_set(bpf_net_ctx: &__bpf_net_ctx);
419
420 if (xdp)
421 *retval = bpf_prog_run_xdp(prog, xdp: ctx);
422 else
423 *retval = bpf_prog_run(prog, ctx);
424
425 bpf_net_ctx_clear(bpf_net_ctx);
426 local_bh_enable();
427 } while (bpf_test_timer_continue(t: &t, iterations: 1, repeat, err: &ret, duration: time));
428 bpf_reset_run_ctx(old_ctx);
429 bpf_test_timer_leave(t: &t);
430
431 for_each_cgroup_storage_type(stype)
432 bpf_cgroup_storage_free(storage: item.cgroup_storage[stype]);
433
434 return ret;
435}
436
437static int bpf_test_finish(const union bpf_attr *kattr,
438 union bpf_attr __user *uattr, const void *data,
439 struct skb_shared_info *sinfo, u32 size, u32 frag_size,
440 u32 retval, u32 duration)
441{
442 void __user *data_out = u64_to_user_ptr(kattr->test.data_out);
443 int err = -EFAULT;
444 u32 copy_size = size;
445
446 /* Clamp copy if the user has provided a size hint, but copy the full
447 * buffer if not to retain old behaviour.
448 */
449 if (kattr->test.data_size_out &&
450 copy_size > kattr->test.data_size_out) {
451 copy_size = kattr->test.data_size_out;
452 err = -ENOSPC;
453 }
454
455 if (data_out) {
456 int len = sinfo ? copy_size - frag_size : copy_size;
457
458 if (len < 0) {
459 err = -ENOSPC;
460 goto out;
461 }
462
463 if (copy_to_user(to: data_out, from: data, n: len))
464 goto out;
465
466 if (sinfo) {
467 int i, offset = len;
468 u32 data_len;
469
470 for (i = 0; i < sinfo->nr_frags; i++) {
471 skb_frag_t *frag = &sinfo->frags[i];
472
473 if (offset >= copy_size) {
474 err = -ENOSPC;
475 break;
476 }
477
478 data_len = min_t(u32, copy_size - offset,
479 skb_frag_size(frag));
480
481 if (copy_to_user(to: data_out + offset,
482 from: skb_frag_address(frag),
483 n: data_len))
484 goto out;
485
486 offset += data_len;
487 }
488 }
489 }
490
491 if (copy_to_user(to: &uattr->test.data_size_out, from: &size, n: sizeof(size)))
492 goto out;
493 if (copy_to_user(to: &uattr->test.retval, from: &retval, n: sizeof(retval)))
494 goto out;
495 if (copy_to_user(to: &uattr->test.duration, from: &duration, n: sizeof(duration)))
496 goto out;
497 if (err != -ENOSPC)
498 err = 0;
499out:
500 trace_bpf_test_finish(err: &err);
501 return err;
502}
503
504/* Integer types of various sizes and pointer combinations cover variety of
505 * architecture dependent calling conventions. 7+ can be supported in the
506 * future.
507 */
508__bpf_kfunc_start_defs();
509
510__bpf_kfunc int bpf_fentry_test1(int a)
511{
512 return a + 1;
513}
514EXPORT_SYMBOL_GPL(bpf_fentry_test1);
515
516noinline int bpf_fentry_test2(int a, u64 b)
517{
518 return a + b;
519}
520
521noinline int bpf_fentry_test3(char a, int b, u64 c)
522{
523 return a + b + c;
524}
525
526noinline int bpf_fentry_test4(void *a, char b, int c, u64 d)
527{
528 return (long)a + b + c + d;
529}
530
531noinline int bpf_fentry_test5(u64 a, void *b, short c, int d, u64 e)
532{
533 return a + (long)b + c + d + e;
534}
535
536noinline int bpf_fentry_test6(u64 a, void *b, short c, int d, void *e, u64 f)
537{
538 return a + (long)b + c + d + (long)e + f;
539}
540
541struct bpf_fentry_test_t {
542 struct bpf_fentry_test_t *a;
543};
544
545noinline int bpf_fentry_test7(struct bpf_fentry_test_t *arg)
546{
547 asm volatile ("" : "+r"(arg));
548 return (long)arg;
549}
550
551noinline int bpf_fentry_test8(struct bpf_fentry_test_t *arg)
552{
553 return (long)arg->a;
554}
555
556__bpf_kfunc u32 bpf_fentry_test9(u32 *a)
557{
558 return *a;
559}
560
561noinline int bpf_fentry_test10(const void *a)
562{
563 return (long)a;
564}
565
566noinline void bpf_fentry_test_sinfo(struct skb_shared_info *sinfo)
567{
568}
569
570__bpf_kfunc int bpf_modify_return_test(int a, int *b)
571{
572 *b += 1;
573 return a + *b;
574}
575
576__bpf_kfunc int bpf_modify_return_test2(int a, int *b, short c, int d,
577 void *e, char f, int g)
578{
579 *b += 1;
580 return a + *b + c + d + (long)e + f + g;
581}
582
583__bpf_kfunc int bpf_modify_return_test_tp(int nonce)
584{
585 trace_bpf_trigger_tp(nonce);
586
587 return nonce;
588}
589
590noinline int bpf_fentry_shadow_test(int a)
591{
592 return a + 1;
593}
594
595struct prog_test_member1 {
596 int a;
597};
598
599struct prog_test_member {
600 struct prog_test_member1 m;
601 int c;
602};
603
604struct prog_test_ref_kfunc {
605 int a;
606 int b;
607 struct prog_test_member memb;
608 struct prog_test_ref_kfunc *next;
609 refcount_t cnt;
610};
611
612__bpf_kfunc void bpf_kfunc_call_test_release(struct prog_test_ref_kfunc *p)
613{
614 refcount_dec(r: &p->cnt);
615}
616
617__bpf_kfunc void bpf_kfunc_call_test_release_dtor(void *p)
618{
619 bpf_kfunc_call_test_release(p);
620}
621CFI_NOSEAL(bpf_kfunc_call_test_release_dtor);
622
623__bpf_kfunc void bpf_kfunc_call_memb_release(struct prog_test_member *p)
624{
625}
626
627__bpf_kfunc void bpf_kfunc_call_memb_release_dtor(void *p)
628{
629}
630CFI_NOSEAL(bpf_kfunc_call_memb_release_dtor);
631
632__bpf_kfunc_end_defs();
633
634BTF_KFUNCS_START(bpf_test_modify_return_ids)
635BTF_ID_FLAGS(func, bpf_modify_return_test)
636BTF_ID_FLAGS(func, bpf_modify_return_test2)
637BTF_ID_FLAGS(func, bpf_modify_return_test_tp)
638BTF_ID_FLAGS(func, bpf_fentry_test1, KF_SLEEPABLE)
639BTF_KFUNCS_END(bpf_test_modify_return_ids)
640
641static const struct btf_kfunc_id_set bpf_test_modify_return_set = {
642 .owner = THIS_MODULE,
643 .set = &bpf_test_modify_return_ids,
644};
645
646BTF_KFUNCS_START(test_sk_check_kfunc_ids)
647BTF_ID_FLAGS(func, bpf_kfunc_call_test_release, KF_RELEASE)
648BTF_ID_FLAGS(func, bpf_kfunc_call_memb_release, KF_RELEASE)
649BTF_KFUNCS_END(test_sk_check_kfunc_ids)
650
651static void *bpf_test_init(const union bpf_attr *kattr, u32 user_size,
652 u32 size, u32 headroom, u32 tailroom)
653{
654 void __user *data_in = u64_to_user_ptr(kattr->test.data_in);
655 void *data;
656
657 if (user_size > PAGE_SIZE - headroom - tailroom)
658 return ERR_PTR(error: -EINVAL);
659
660 size = SKB_DATA_ALIGN(size);
661 data = kzalloc(size + headroom + tailroom, GFP_USER);
662 if (!data)
663 return ERR_PTR(error: -ENOMEM);
664
665 if (copy_from_user(to: data + headroom, from: data_in, n: user_size)) {
666 kfree(objp: data);
667 return ERR_PTR(error: -EFAULT);
668 }
669
670 return data;
671}
672
673int bpf_prog_test_run_tracing(struct bpf_prog *prog,
674 const union bpf_attr *kattr,
675 union bpf_attr __user *uattr)
676{
677 struct bpf_fentry_test_t arg = {};
678 u16 side_effect = 0, ret = 0;
679 int b = 2, err = -EFAULT;
680 u32 retval = 0;
681
682 if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size)
683 return -EINVAL;
684
685 switch (prog->expected_attach_type) {
686 case BPF_TRACE_FENTRY:
687 case BPF_TRACE_FEXIT:
688 if (bpf_fentry_test1(1) != 2 ||
689 bpf_fentry_test2(a: 2, b: 3) != 5 ||
690 bpf_fentry_test3(a: 4, b: 5, c: 6) != 15 ||
691 bpf_fentry_test4(a: (void *)7, b: 8, c: 9, d: 10) != 34 ||
692 bpf_fentry_test5(a: 11, b: (void *)12, c: 13, d: 14, e: 15) != 65 ||
693 bpf_fentry_test6(a: 16, b: (void *)17, c: 18, d: 19, e: (void *)20, f: 21) != 111 ||
694 bpf_fentry_test7(arg: (struct bpf_fentry_test_t *)0) != 0 ||
695 bpf_fentry_test8(arg: &arg) != 0 ||
696 bpf_fentry_test9(a: &retval) != 0 ||
697 bpf_fentry_test10(a: (void *)0) != 0)
698 goto out;
699 break;
700 case BPF_MODIFY_RETURN:
701 ret = bpf_modify_return_test(a: 1, b: &b);
702 if (b != 2)
703 side_effect++;
704 b = 2;
705 ret += bpf_modify_return_test2(a: 1, b: &b, c: 3, d: 4, e: (void *)5, f: 6, g: 7);
706 if (b != 2)
707 side_effect++;
708 break;
709 default:
710 goto out;
711 }
712
713 retval = ((u32)side_effect << 16) | ret;
714 if (copy_to_user(to: &uattr->test.retval, from: &retval, n: sizeof(retval)))
715 goto out;
716
717 err = 0;
718out:
719 trace_bpf_test_finish(err: &err);
720 return err;
721}
722
723struct bpf_raw_tp_test_run_info {
724 struct bpf_prog *prog;
725 void *ctx;
726 u32 retval;
727};
728
729static void
730__bpf_prog_test_run_raw_tp(void *data)
731{
732 struct bpf_raw_tp_test_run_info *info = data;
733 struct bpf_trace_run_ctx run_ctx = {};
734 struct bpf_run_ctx *old_run_ctx;
735
736 old_run_ctx = bpf_set_run_ctx(new_ctx: &run_ctx.run_ctx);
737
738 rcu_read_lock();
739 info->retval = bpf_prog_run(prog: info->prog, ctx: info->ctx);
740 rcu_read_unlock();
741
742 bpf_reset_run_ctx(old_ctx: old_run_ctx);
743}
744
745int bpf_prog_test_run_raw_tp(struct bpf_prog *prog,
746 const union bpf_attr *kattr,
747 union bpf_attr __user *uattr)
748{
749 void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in);
750 __u32 ctx_size_in = kattr->test.ctx_size_in;
751 struct bpf_raw_tp_test_run_info info;
752 int cpu = kattr->test.cpu, err = 0;
753 int current_cpu;
754
755 /* doesn't support data_in/out, ctx_out, duration, or repeat */
756 if (kattr->test.data_in || kattr->test.data_out ||
757 kattr->test.ctx_out || kattr->test.duration ||
758 kattr->test.repeat || kattr->test.batch_size)
759 return -EINVAL;
760
761 if (ctx_size_in < prog->aux->max_ctx_offset ||
762 ctx_size_in > MAX_BPF_FUNC_ARGS * sizeof(u64))
763 return -EINVAL;
764
765 if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 && cpu != 0)
766 return -EINVAL;
767
768 if (ctx_size_in) {
769 info.ctx = memdup_user(ctx_in, ctx_size_in);
770 if (IS_ERR(ptr: info.ctx))
771 return PTR_ERR(ptr: info.ctx);
772 } else {
773 info.ctx = NULL;
774 }
775
776 info.prog = prog;
777
778 current_cpu = get_cpu();
779 if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 ||
780 cpu == current_cpu) {
781 __bpf_prog_test_run_raw_tp(data: &info);
782 } else if (cpu >= nr_cpu_ids || !cpu_online(cpu)) {
783 /* smp_call_function_single() also checks cpu_online()
784 * after csd_lock(). However, since cpu is from user
785 * space, let's do an extra quick check to filter out
786 * invalid value before smp_call_function_single().
787 */
788 err = -ENXIO;
789 } else {
790 err = smp_call_function_single(cpuid: cpu, func: __bpf_prog_test_run_raw_tp,
791 info: &info, wait: 1);
792 }
793 put_cpu();
794
795 if (!err &&
796 copy_to_user(to: &uattr->test.retval, from: &info.retval, n: sizeof(u32)))
797 err = -EFAULT;
798
799 kfree(objp: info.ctx);
800 return err;
801}
802
803static void *bpf_ctx_init(const union bpf_attr *kattr, u32 max_size)
804{
805 void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in);
806 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out);
807 u32 size = kattr->test.ctx_size_in;
808 void *data;
809 int err;
810
811 if (!data_in && !data_out)
812 return NULL;
813
814 data = kzalloc(max_size, GFP_USER);
815 if (!data)
816 return ERR_PTR(error: -ENOMEM);
817
818 if (data_in) {
819 err = bpf_check_uarg_tail_zero(uaddr: USER_BPFPTR(p: data_in), expected_size: max_size, actual_size: size);
820 if (err) {
821 kfree(objp: data);
822 return ERR_PTR(error: err);
823 }
824
825 size = min_t(u32, max_size, size);
826 if (copy_from_user(to: data, from: data_in, n: size)) {
827 kfree(objp: data);
828 return ERR_PTR(error: -EFAULT);
829 }
830 }
831 return data;
832}
833
834static int bpf_ctx_finish(const union bpf_attr *kattr,
835 union bpf_attr __user *uattr, const void *data,
836 u32 size)
837{
838 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out);
839 int err = -EFAULT;
840 u32 copy_size = size;
841
842 if (!data || !data_out)
843 return 0;
844
845 if (copy_size > kattr->test.ctx_size_out) {
846 copy_size = kattr->test.ctx_size_out;
847 err = -ENOSPC;
848 }
849
850 if (copy_to_user(to: data_out, from: data, n: copy_size))
851 goto out;
852 if (copy_to_user(to: &uattr->test.ctx_size_out, from: &size, n: sizeof(size)))
853 goto out;
854 if (err != -ENOSPC)
855 err = 0;
856out:
857 return err;
858}
859
860/**
861 * range_is_zero - test whether buffer is initialized
862 * @buf: buffer to check
863 * @from: check from this position
864 * @to: check up until (excluding) this position
865 *
866 * This function returns true if the there is a non-zero byte
867 * in the buf in the range [from,to).
868 */
869static inline bool range_is_zero(void *buf, size_t from, size_t to)
870{
871 return !memchr_inv(p: (u8 *)buf + from, c: 0, size: to - from);
872}
873
874static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb)
875{
876 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb;
877
878 if (!__skb)
879 return 0;
880
881 /* make sure the fields we don't use are zeroed */
882 if (!range_is_zero(buf: __skb, from: 0, offsetof(struct __sk_buff, mark)))
883 return -EINVAL;
884
885 /* mark is allowed */
886
887 if (!range_is_zero(buf: __skb, offsetofend(struct __sk_buff, mark),
888 offsetof(struct __sk_buff, priority)))
889 return -EINVAL;
890
891 /* priority is allowed */
892 /* ingress_ifindex is allowed */
893 /* ifindex is allowed */
894
895 if (!range_is_zero(buf: __skb, offsetofend(struct __sk_buff, ifindex),
896 offsetof(struct __sk_buff, cb)))
897 return -EINVAL;
898
899 /* cb is allowed */
900
901 if (!range_is_zero(buf: __skb, offsetofend(struct __sk_buff, cb),
902 offsetof(struct __sk_buff, data_end)))
903 return -EINVAL;
904
905 /* data_end is allowed, but not copied to skb */
906
907 if (!range_is_zero(buf: __skb, offsetofend(struct __sk_buff, data_end),
908 offsetof(struct __sk_buff, tstamp)))
909 return -EINVAL;
910
911 /* tstamp is allowed */
912 /* wire_len is allowed */
913 /* gso_segs is allowed */
914
915 if (!range_is_zero(buf: __skb, offsetofend(struct __sk_buff, gso_segs),
916 offsetof(struct __sk_buff, gso_size)))
917 return -EINVAL;
918
919 /* gso_size is allowed */
920
921 if (!range_is_zero(buf: __skb, offsetofend(struct __sk_buff, gso_size),
922 offsetof(struct __sk_buff, hwtstamp)))
923 return -EINVAL;
924
925 /* hwtstamp is allowed */
926
927 if (!range_is_zero(buf: __skb, offsetofend(struct __sk_buff, hwtstamp),
928 to: sizeof(struct __sk_buff)))
929 return -EINVAL;
930
931 skb->mark = __skb->mark;
932 skb->priority = __skb->priority;
933 skb->skb_iif = __skb->ingress_ifindex;
934 skb->tstamp = __skb->tstamp;
935 memcpy(&cb->data, __skb->cb, QDISC_CB_PRIV_LEN);
936
937 if (__skb->wire_len == 0) {
938 cb->pkt_len = skb->len;
939 } else {
940 if (__skb->wire_len < skb->len ||
941 __skb->wire_len > GSO_LEGACY_MAX_SIZE)
942 return -EINVAL;
943 cb->pkt_len = __skb->wire_len;
944 }
945
946 if (__skb->gso_segs > GSO_MAX_SEGS)
947 return -EINVAL;
948
949 /* Currently GSO type is zero/unset. If this gets extended with
950 * a small list of accepted GSO types in future, the filter for
951 * an unset GSO type in bpf_clone_redirect() can be lifted.
952 */
953 skb_shinfo(skb)->gso_segs = __skb->gso_segs;
954 skb_shinfo(skb)->gso_size = __skb->gso_size;
955 skb_shinfo(skb)->hwtstamps.hwtstamp = __skb->hwtstamp;
956
957 return 0;
958}
959
960static void convert_skb_to___skb(struct sk_buff *skb, struct __sk_buff *__skb)
961{
962 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb;
963
964 if (!__skb)
965 return;
966
967 __skb->mark = skb->mark;
968 __skb->priority = skb->priority;
969 __skb->ingress_ifindex = skb->skb_iif;
970 __skb->ifindex = skb->dev->ifindex;
971 __skb->tstamp = skb->tstamp;
972 memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN);
973 __skb->wire_len = cb->pkt_len;
974 __skb->gso_segs = skb_shinfo(skb)->gso_segs;
975 __skb->hwtstamp = skb_shinfo(skb)->hwtstamps.hwtstamp;
976}
977
978static struct proto bpf_dummy_proto = {
979 .name = "bpf_dummy",
980 .owner = THIS_MODULE,
981 .obj_size = sizeof(struct sock),
982};
983
984int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
985 union bpf_attr __user *uattr)
986{
987 bool is_l2 = false, is_direct_pkt_access = false, is_lwt = false;
988 u32 tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
989 struct net *net = current->nsproxy->net_ns;
990 struct net_device *dev = net->loopback_dev;
991 u32 headroom = NET_SKB_PAD + NET_IP_ALIGN;
992 u32 linear_sz = kattr->test.data_size_in;
993 u32 repeat = kattr->test.repeat;
994 struct __sk_buff *ctx = NULL;
995 struct sk_buff *skb = NULL;
996 struct sock *sk = NULL;
997 u32 retval, duration;
998 int hh_len = ETH_HLEN;
999 void *data = NULL;
1000 int ret;
1001
1002 if ((kattr->test.flags & ~BPF_F_TEST_SKB_CHECKSUM_COMPLETE) ||
1003 kattr->test.cpu || kattr->test.batch_size)
1004 return -EINVAL;
1005
1006 if (kattr->test.data_size_in < ETH_HLEN)
1007 return -EINVAL;
1008
1009 switch (prog->type) {
1010 case BPF_PROG_TYPE_SCHED_CLS:
1011 case BPF_PROG_TYPE_SCHED_ACT:
1012 is_direct_pkt_access = true;
1013 is_l2 = true;
1014 break;
1015 case BPF_PROG_TYPE_LWT_IN:
1016 case BPF_PROG_TYPE_LWT_OUT:
1017 case BPF_PROG_TYPE_LWT_XMIT:
1018 is_lwt = true;
1019 fallthrough;
1020 case BPF_PROG_TYPE_CGROUP_SKB:
1021 is_direct_pkt_access = true;
1022 break;
1023 default:
1024 break;
1025 }
1026
1027 ctx = bpf_ctx_init(kattr, max_size: sizeof(struct __sk_buff));
1028 if (IS_ERR(ptr: ctx))
1029 return PTR_ERR(ptr: ctx);
1030
1031 if (ctx) {
1032 if (ctx->data_end > kattr->test.data_size_in || ctx->data || ctx->data_meta) {
1033 ret = -EINVAL;
1034 goto out;
1035 }
1036 if (ctx->data_end) {
1037 /* Non-linear LWT test_run is unsupported for now. */
1038 if (is_lwt) {
1039 ret = -EINVAL;
1040 goto out;
1041 }
1042 linear_sz = max(ETH_HLEN, ctx->data_end);
1043 }
1044 }
1045
1046 linear_sz = min_t(u32, linear_sz, PAGE_SIZE - headroom - tailroom);
1047
1048 data = bpf_test_init(kattr, user_size: linear_sz, size: linear_sz, headroom, tailroom);
1049 if (IS_ERR(ptr: data)) {
1050 ret = PTR_ERR(ptr: data);
1051 data = NULL;
1052 goto out;
1053 }
1054
1055 sk = sk_alloc(net, AF_UNSPEC, GFP_USER, prot: &bpf_dummy_proto, kern: 1);
1056 if (!sk) {
1057 ret = -ENOMEM;
1058 goto out;
1059 }
1060 sock_init_data(NULL, sk);
1061
1062 skb = slab_build_skb(data);
1063 if (!skb) {
1064 ret = -ENOMEM;
1065 goto out;
1066 }
1067 skb->sk = sk;
1068
1069 data = NULL; /* data released via kfree_skb */
1070
1071 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
1072 __skb_put(skb, len: linear_sz);
1073
1074 if (unlikely(kattr->test.data_size_in > linear_sz)) {
1075 void __user *data_in = u64_to_user_ptr(kattr->test.data_in);
1076 struct skb_shared_info *sinfo = skb_shinfo(skb);
1077 u32 copied = linear_sz;
1078
1079 while (copied < kattr->test.data_size_in) {
1080 struct page *page;
1081 u32 data_len;
1082
1083 if (sinfo->nr_frags == MAX_SKB_FRAGS) {
1084 ret = -ENOMEM;
1085 goto out;
1086 }
1087
1088 page = alloc_page(GFP_KERNEL);
1089 if (!page) {
1090 ret = -ENOMEM;
1091 goto out;
1092 }
1093
1094 data_len = min_t(u32, kattr->test.data_size_in - copied,
1095 PAGE_SIZE);
1096 skb_fill_page_desc(skb, i: sinfo->nr_frags, page, off: 0, size: data_len);
1097
1098 if (copy_from_user(page_address(page), from: data_in + copied,
1099 n: data_len)) {
1100 ret = -EFAULT;
1101 goto out;
1102 }
1103 skb->data_len += data_len;
1104 skb->truesize += PAGE_SIZE;
1105 skb->len += data_len;
1106 copied += data_len;
1107 }
1108 }
1109
1110 if (ctx && ctx->ifindex > 1) {
1111 dev = dev_get_by_index(net, ifindex: ctx->ifindex);
1112 if (!dev) {
1113 ret = -ENODEV;
1114 goto out;
1115 }
1116 }
1117 skb->protocol = eth_type_trans(skb, dev);
1118 skb_reset_network_header(skb);
1119
1120 switch (skb->protocol) {
1121 case htons(ETH_P_IP):
1122 sk->sk_family = AF_INET;
1123 if (sizeof(struct iphdr) <= skb_headlen(skb)) {
1124 sk->sk_rcv_saddr = ip_hdr(skb)->saddr;
1125 sk->sk_daddr = ip_hdr(skb)->daddr;
1126 }
1127 break;
1128#if IS_ENABLED(CONFIG_IPV6)
1129 case htons(ETH_P_IPV6):
1130 sk->sk_family = AF_INET6;
1131 if (sizeof(struct ipv6hdr) <= skb_headlen(skb)) {
1132 sk->sk_v6_rcv_saddr = ipv6_hdr(skb)->saddr;
1133 sk->sk_v6_daddr = ipv6_hdr(skb)->daddr;
1134 }
1135 break;
1136#endif
1137 default:
1138 break;
1139 }
1140
1141 if (is_l2)
1142 __skb_push(skb, len: hh_len);
1143 if (is_direct_pkt_access)
1144 bpf_compute_data_pointers(skb);
1145
1146 ret = convert___skb_to_skb(skb, skb: ctx);
1147 if (ret)
1148 goto out;
1149
1150 if (kattr->test.flags & BPF_F_TEST_SKB_CHECKSUM_COMPLETE) {
1151 const int off = skb_network_offset(skb);
1152 int len = skb->len - off;
1153
1154 skb->csum = skb_checksum(skb, offset: off, len, csum: 0);
1155 skb->ip_summed = CHECKSUM_COMPLETE;
1156 }
1157
1158 ret = bpf_test_run(prog, ctx: skb, repeat, retval: &retval, time: &duration, xdp: false);
1159 if (ret)
1160 goto out;
1161 if (!is_l2) {
1162 if (skb_headroom(skb) < hh_len) {
1163 int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb));
1164
1165 if (pskb_expand_head(skb, nhead, ntail: 0, GFP_USER)) {
1166 ret = -ENOMEM;
1167 goto out;
1168 }
1169 }
1170 memset(__skb_push(skb, hh_len), 0, hh_len);
1171 }
1172
1173 if (kattr->test.flags & BPF_F_TEST_SKB_CHECKSUM_COMPLETE) {
1174 const int off = skb_network_offset(skb);
1175 int len = skb->len - off;
1176 __wsum csum;
1177
1178 csum = skb_checksum(skb, offset: off, len, csum: 0);
1179
1180 if (csum_fold(csum: skb->csum) != csum_fold(csum)) {
1181 ret = -EBADMSG;
1182 goto out;
1183 }
1184 }
1185
1186 convert_skb_to___skb(skb, skb: ctx);
1187
1188 if (skb_is_nonlinear(skb))
1189 /* bpf program can never convert linear skb to non-linear */
1190 WARN_ON_ONCE(linear_sz == kattr->test.data_size_in);
1191 ret = bpf_test_finish(kattr, uattr, data: skb->data, skb_shinfo(skb), size: skb->len,
1192 frag_size: skb->data_len, retval, duration);
1193 if (!ret)
1194 ret = bpf_ctx_finish(kattr, uattr, data: ctx,
1195 size: sizeof(struct __sk_buff));
1196out:
1197 if (dev && dev != net->loopback_dev)
1198 dev_put(dev);
1199 kfree_skb(skb);
1200 kfree(objp: data);
1201 if (sk)
1202 sk_free(sk);
1203 kfree(objp: ctx);
1204 return ret;
1205}
1206
1207static int xdp_convert_md_to_buff(struct xdp_md *xdp_md, struct xdp_buff *xdp)
1208{
1209 unsigned int ingress_ifindex, rx_queue_index;
1210 struct netdev_rx_queue *rxqueue;
1211 struct net_device *device;
1212
1213 if (!xdp_md)
1214 return 0;
1215
1216 if (xdp_md->egress_ifindex != 0)
1217 return -EINVAL;
1218
1219 ingress_ifindex = xdp_md->ingress_ifindex;
1220 rx_queue_index = xdp_md->rx_queue_index;
1221
1222 if (!ingress_ifindex && rx_queue_index)
1223 return -EINVAL;
1224
1225 if (ingress_ifindex) {
1226 device = dev_get_by_index(current->nsproxy->net_ns,
1227 ifindex: ingress_ifindex);
1228 if (!device)
1229 return -ENODEV;
1230
1231 if (rx_queue_index >= device->real_num_rx_queues)
1232 goto free_dev;
1233
1234 rxqueue = __netif_get_rx_queue(dev: device, rxq: rx_queue_index);
1235
1236 if (!xdp_rxq_info_is_reg(xdp_rxq: &rxqueue->xdp_rxq))
1237 goto free_dev;
1238
1239 xdp->rxq = &rxqueue->xdp_rxq;
1240 /* The device is now tracked in the xdp->rxq for later
1241 * dev_put()
1242 */
1243 }
1244
1245 xdp->data = xdp->data_meta + xdp_md->data;
1246 return 0;
1247
1248free_dev:
1249 dev_put(dev: device);
1250 return -EINVAL;
1251}
1252
1253static void xdp_convert_buff_to_md(struct xdp_buff *xdp, struct xdp_md *xdp_md)
1254{
1255 if (!xdp_md)
1256 return;
1257
1258 xdp_md->data = xdp->data - xdp->data_meta;
1259 xdp_md->data_end = xdp->data_end - xdp->data_meta;
1260
1261 if (xdp_md->ingress_ifindex)
1262 dev_put(dev: xdp->rxq->dev);
1263}
1264
1265int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,
1266 union bpf_attr __user *uattr)
1267{
1268 bool do_live = (kattr->test.flags & BPF_F_TEST_XDP_LIVE_FRAMES);
1269 u32 tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1270 u32 retval = 0, meta_sz = 0, duration, max_linear_sz, size;
1271 u32 linear_sz = kattr->test.data_size_in;
1272 u32 batch_size = kattr->test.batch_size;
1273 u32 headroom = XDP_PACKET_HEADROOM;
1274 u32 repeat = kattr->test.repeat;
1275 struct netdev_rx_queue *rxqueue;
1276 struct skb_shared_info *sinfo;
1277 struct xdp_buff xdp = {};
1278 int i, ret = -EINVAL;
1279 struct xdp_md *ctx;
1280 void *data;
1281
1282 if (prog->expected_attach_type == BPF_XDP_DEVMAP ||
1283 prog->expected_attach_type == BPF_XDP_CPUMAP)
1284 return -EINVAL;
1285
1286 if (kattr->test.flags & ~BPF_F_TEST_XDP_LIVE_FRAMES)
1287 return -EINVAL;
1288
1289 if (bpf_prog_is_dev_bound(aux: prog->aux))
1290 return -EINVAL;
1291
1292 if (do_live) {
1293 if (!batch_size)
1294 batch_size = NAPI_POLL_WEIGHT;
1295 else if (batch_size > TEST_XDP_MAX_BATCH)
1296 return -E2BIG;
1297 } else if (batch_size) {
1298 return -EINVAL;
1299 }
1300
1301 ctx = bpf_ctx_init(kattr, max_size: sizeof(struct xdp_md));
1302 if (IS_ERR(ptr: ctx))
1303 return PTR_ERR(ptr: ctx);
1304
1305 if (ctx) {
1306 /* There can't be user provided data before the meta data */
1307 if (ctx->data_meta || ctx->data_end > kattr->test.data_size_in ||
1308 ctx->data > ctx->data_end ||
1309 (do_live && (kattr->test.data_out || kattr->test.ctx_out)))
1310 goto free_ctx;
1311
1312 meta_sz = ctx->data;
1313 if (xdp_metalen_invalid(metalen: meta_sz) || meta_sz > headroom - sizeof(struct xdp_frame))
1314 goto free_ctx;
1315
1316 /* Meta data is allocated from the headroom */
1317 headroom -= meta_sz;
1318 linear_sz = ctx->data_end;
1319 }
1320
1321 /* The xdp_page_head structure takes up space in each page, limiting the
1322 * size of the packet data; add the extra size to headroom here to make
1323 * sure it's accounted in the length checks below, but not in the
1324 * metadata size check above.
1325 */
1326 if (do_live)
1327 headroom += sizeof(struct xdp_page_head);
1328
1329 max_linear_sz = PAGE_SIZE - headroom - tailroom;
1330 linear_sz = min_t(u32, linear_sz, max_linear_sz);
1331
1332 /* disallow live data mode for jumbo frames */
1333 if (do_live && kattr->test.data_size_in > linear_sz)
1334 goto free_ctx;
1335
1336 if (kattr->test.data_size_in - meta_sz < ETH_HLEN)
1337 goto free_ctx;
1338
1339 data = bpf_test_init(kattr, user_size: linear_sz, size: max_linear_sz, headroom, tailroom);
1340 if (IS_ERR(ptr: data)) {
1341 ret = PTR_ERR(ptr: data);
1342 goto free_ctx;
1343 }
1344
1345 rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, rxq: 0);
1346 rxqueue->xdp_rxq.frag_size = PAGE_SIZE;
1347 xdp_init_buff(xdp: &xdp, frame_sz: rxqueue->xdp_rxq.frag_size, rxq: &rxqueue->xdp_rxq);
1348 xdp_prepare_buff(xdp: &xdp, hard_start: data, headroom, data_len: linear_sz, meta_valid: true);
1349 sinfo = xdp_get_shared_info_from_buff(xdp: &xdp);
1350
1351 ret = xdp_convert_md_to_buff(xdp_md: ctx, xdp: &xdp);
1352 if (ret)
1353 goto free_data;
1354
1355 size = linear_sz;
1356 if (unlikely(kattr->test.data_size_in > size)) {
1357 void __user *data_in = u64_to_user_ptr(kattr->test.data_in);
1358
1359 while (size < kattr->test.data_size_in) {
1360 struct page *page;
1361 skb_frag_t *frag;
1362 u32 data_len;
1363
1364 if (sinfo->nr_frags == MAX_SKB_FRAGS) {
1365 ret = -ENOMEM;
1366 goto out_put_dev;
1367 }
1368
1369 page = alloc_page(GFP_KERNEL);
1370 if (!page) {
1371 ret = -ENOMEM;
1372 goto out_put_dev;
1373 }
1374
1375 frag = &sinfo->frags[sinfo->nr_frags++];
1376
1377 data_len = min_t(u32, kattr->test.data_size_in - size,
1378 PAGE_SIZE);
1379 skb_frag_fill_page_desc(frag, page, off: 0, size: data_len);
1380
1381 if (copy_from_user(page_address(page), from: data_in + size,
1382 n: data_len)) {
1383 ret = -EFAULT;
1384 goto out_put_dev;
1385 }
1386 sinfo->xdp_frags_size += data_len;
1387 size += data_len;
1388 }
1389 xdp_buff_set_frags_flag(xdp: &xdp);
1390 }
1391
1392 if (repeat > 1)
1393 bpf_prog_change_xdp(NULL, prog);
1394
1395 if (do_live)
1396 ret = bpf_test_run_xdp_live(prog, ctx: &xdp, repeat, batch_size, time: &duration);
1397 else
1398 ret = bpf_test_run(prog, ctx: &xdp, repeat, retval: &retval, time: &duration, xdp: true);
1399out_put_dev:
1400 /* We convert the xdp_buff back to an xdp_md before checking the return
1401 * code so the reference count of any held netdevice will be decremented
1402 * even if the test run failed.
1403 */
1404 xdp_convert_buff_to_md(xdp: &xdp, xdp_md: ctx);
1405 if (ret)
1406 goto out;
1407
1408 size = xdp.data_end - xdp.data_meta + sinfo->xdp_frags_size;
1409 ret = bpf_test_finish(kattr, uattr, data: xdp.data_meta, sinfo, size, frag_size: sinfo->xdp_frags_size,
1410 retval, duration);
1411 if (!ret)
1412 ret = bpf_ctx_finish(kattr, uattr, data: ctx,
1413 size: sizeof(struct xdp_md));
1414
1415out:
1416 if (repeat > 1)
1417 bpf_prog_change_xdp(prev_prog: prog, NULL);
1418free_data:
1419 for (i = 0; i < sinfo->nr_frags; i++)
1420 __free_page(skb_frag_page(&sinfo->frags[i]));
1421 kfree(objp: data);
1422free_ctx:
1423 kfree(objp: ctx);
1424 return ret;
1425}
1426
1427static int verify_user_bpf_flow_keys(struct bpf_flow_keys *ctx)
1428{
1429 /* make sure the fields we don't use are zeroed */
1430 if (!range_is_zero(buf: ctx, from: 0, offsetof(struct bpf_flow_keys, flags)))
1431 return -EINVAL;
1432
1433 /* flags is allowed */
1434
1435 if (!range_is_zero(buf: ctx, offsetofend(struct bpf_flow_keys, flags),
1436 to: sizeof(struct bpf_flow_keys)))
1437 return -EINVAL;
1438
1439 return 0;
1440}
1441
1442int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
1443 const union bpf_attr *kattr,
1444 union bpf_attr __user *uattr)
1445{
1446 struct bpf_test_timer t = {};
1447 u32 size = kattr->test.data_size_in;
1448 struct bpf_flow_dissector ctx = {};
1449 u32 repeat = kattr->test.repeat;
1450 struct bpf_flow_keys *user_ctx;
1451 struct bpf_flow_keys flow_keys;
1452 const struct ethhdr *eth;
1453 unsigned int flags = 0;
1454 u32 retval, duration;
1455 void *data;
1456 int ret;
1457
1458 if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size)
1459 return -EINVAL;
1460
1461 if (size < ETH_HLEN)
1462 return -EINVAL;
1463
1464 data = bpf_test_init(kattr, user_size: kattr->test.data_size_in, size, headroom: 0, tailroom: 0);
1465 if (IS_ERR(ptr: data))
1466 return PTR_ERR(ptr: data);
1467
1468 eth = (struct ethhdr *)data;
1469
1470 if (!repeat)
1471 repeat = 1;
1472
1473 user_ctx = bpf_ctx_init(kattr, max_size: sizeof(struct bpf_flow_keys));
1474 if (IS_ERR(ptr: user_ctx)) {
1475 kfree(objp: data);
1476 return PTR_ERR(ptr: user_ctx);
1477 }
1478 if (user_ctx) {
1479 ret = verify_user_bpf_flow_keys(ctx: user_ctx);
1480 if (ret)
1481 goto out;
1482 flags = user_ctx->flags;
1483 }
1484
1485 ctx.flow_keys = &flow_keys;
1486 ctx.data = data;
1487 ctx.data_end = (__u8 *)data + size;
1488
1489 bpf_test_timer_enter(t: &t);
1490 do {
1491 retval = bpf_flow_dissect(prog, ctx: &ctx, proto: eth->h_proto, ETH_HLEN,
1492 hlen: size, flags);
1493 } while (bpf_test_timer_continue(t: &t, iterations: 1, repeat, err: &ret, duration: &duration));
1494 bpf_test_timer_leave(t: &t);
1495
1496 if (ret < 0)
1497 goto out;
1498
1499 ret = bpf_test_finish(kattr, uattr, data: &flow_keys, NULL,
1500 size: sizeof(flow_keys), frag_size: 0, retval, duration);
1501 if (!ret)
1502 ret = bpf_ctx_finish(kattr, uattr, data: user_ctx,
1503 size: sizeof(struct bpf_flow_keys));
1504
1505out:
1506 kfree(objp: user_ctx);
1507 kfree(objp: data);
1508 return ret;
1509}
1510
1511int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, const union bpf_attr *kattr,
1512 union bpf_attr __user *uattr)
1513{
1514 struct bpf_test_timer t = {};
1515 struct bpf_prog_array *progs = NULL;
1516 struct bpf_sk_lookup_kern ctx = {};
1517 u32 repeat = kattr->test.repeat;
1518 struct bpf_sk_lookup *user_ctx;
1519 u32 retval, duration;
1520 int ret = -EINVAL;
1521
1522 if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size)
1523 return -EINVAL;
1524
1525 if (kattr->test.data_in || kattr->test.data_size_in || kattr->test.data_out ||
1526 kattr->test.data_size_out)
1527 return -EINVAL;
1528
1529 if (!repeat)
1530 repeat = 1;
1531
1532 user_ctx = bpf_ctx_init(kattr, max_size: sizeof(*user_ctx));
1533 if (IS_ERR(ptr: user_ctx))
1534 return PTR_ERR(ptr: user_ctx);
1535
1536 if (!user_ctx)
1537 return -EINVAL;
1538
1539 if (user_ctx->sk)
1540 goto out;
1541
1542 if (!range_is_zero(buf: user_ctx, offsetofend(typeof(*user_ctx), local_port), to: sizeof(*user_ctx)))
1543 goto out;
1544
1545 if (user_ctx->local_port > U16_MAX) {
1546 ret = -ERANGE;
1547 goto out;
1548 }
1549
1550 ctx.family = (u16)user_ctx->family;
1551 ctx.protocol = (u16)user_ctx->protocol;
1552 ctx.dport = (u16)user_ctx->local_port;
1553 ctx.sport = user_ctx->remote_port;
1554
1555 switch (ctx.family) {
1556 case AF_INET:
1557 ctx.v4.daddr = (__force __be32)user_ctx->local_ip4;
1558 ctx.v4.saddr = (__force __be32)user_ctx->remote_ip4;
1559 break;
1560
1561#if IS_ENABLED(CONFIG_IPV6)
1562 case AF_INET6:
1563 ctx.v6.daddr = (struct in6_addr *)user_ctx->local_ip6;
1564 ctx.v6.saddr = (struct in6_addr *)user_ctx->remote_ip6;
1565 break;
1566#endif
1567
1568 default:
1569 ret = -EAFNOSUPPORT;
1570 goto out;
1571 }
1572
1573 progs = bpf_prog_array_alloc(prog_cnt: 1, GFP_KERNEL);
1574 if (!progs) {
1575 ret = -ENOMEM;
1576 goto out;
1577 }
1578
1579 progs->items[0].prog = prog;
1580
1581 bpf_test_timer_enter(t: &t);
1582 do {
1583 ctx.selected_sk = NULL;
1584 retval = BPF_PROG_SK_LOOKUP_RUN_ARRAY(progs, ctx, bpf_prog_run);
1585 } while (bpf_test_timer_continue(t: &t, iterations: 1, repeat, err: &ret, duration: &duration));
1586 bpf_test_timer_leave(t: &t);
1587
1588 if (ret < 0)
1589 goto out;
1590
1591 user_ctx->cookie = 0;
1592 if (ctx.selected_sk) {
1593 if (ctx.selected_sk->sk_reuseport && !ctx.no_reuseport) {
1594 ret = -EOPNOTSUPP;
1595 goto out;
1596 }
1597
1598 user_ctx->cookie = sock_gen_cookie(sk: ctx.selected_sk);
1599 }
1600
1601 ret = bpf_test_finish(kattr, uattr, NULL, NULL, size: 0, frag_size: 0, retval, duration);
1602 if (!ret)
1603 ret = bpf_ctx_finish(kattr, uattr, data: user_ctx, size: sizeof(*user_ctx));
1604
1605out:
1606 bpf_prog_array_free(progs);
1607 kfree(objp: user_ctx);
1608 return ret;
1609}
1610
1611int bpf_prog_test_run_syscall(struct bpf_prog *prog,
1612 const union bpf_attr *kattr,
1613 union bpf_attr __user *uattr)
1614{
1615 void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in);
1616 __u32 ctx_size_in = kattr->test.ctx_size_in;
1617 void *ctx = NULL;
1618 u32 retval;
1619 int err = 0;
1620
1621 /* doesn't support data_in/out, ctx_out, duration, or repeat or flags */
1622 if (kattr->test.data_in || kattr->test.data_out ||
1623 kattr->test.ctx_out || kattr->test.duration ||
1624 kattr->test.repeat || kattr->test.flags ||
1625 kattr->test.batch_size)
1626 return -EINVAL;
1627
1628 if (ctx_size_in < prog->aux->max_ctx_offset ||
1629 ctx_size_in > U16_MAX)
1630 return -EINVAL;
1631
1632 if (ctx_size_in) {
1633 ctx = memdup_user(ctx_in, ctx_size_in);
1634 if (IS_ERR(ptr: ctx))
1635 return PTR_ERR(ptr: ctx);
1636 }
1637
1638 rcu_read_lock_trace();
1639 retval = bpf_prog_run_pin_on_cpu(prog, ctx);
1640 rcu_read_unlock_trace();
1641
1642 if (copy_to_user(to: &uattr->test.retval, from: &retval, n: sizeof(u32))) {
1643 err = -EFAULT;
1644 goto out;
1645 }
1646 if (ctx_size_in)
1647 if (copy_to_user(to: ctx_in, from: ctx, n: ctx_size_in))
1648 err = -EFAULT;
1649out:
1650 kfree(objp: ctx);
1651 return err;
1652}
1653
1654static int verify_and_copy_hook_state(struct nf_hook_state *state,
1655 const struct nf_hook_state *user,
1656 struct net_device *dev)
1657{
1658 if (user->in || user->out)
1659 return -EINVAL;
1660
1661 if (user->net || user->sk || user->okfn)
1662 return -EINVAL;
1663
1664 switch (user->pf) {
1665 case NFPROTO_IPV4:
1666 case NFPROTO_IPV6:
1667 switch (state->hook) {
1668 case NF_INET_PRE_ROUTING:
1669 state->in = dev;
1670 break;
1671 case NF_INET_LOCAL_IN:
1672 state->in = dev;
1673 break;
1674 case NF_INET_FORWARD:
1675 state->in = dev;
1676 state->out = dev;
1677 break;
1678 case NF_INET_LOCAL_OUT:
1679 state->out = dev;
1680 break;
1681 case NF_INET_POST_ROUTING:
1682 state->out = dev;
1683 break;
1684 }
1685
1686 break;
1687 default:
1688 return -EINVAL;
1689 }
1690
1691 state->pf = user->pf;
1692 state->hook = user->hook;
1693
1694 return 0;
1695}
1696
1697static __be16 nfproto_eth(int nfproto)
1698{
1699 switch (nfproto) {
1700 case NFPROTO_IPV4:
1701 return htons(ETH_P_IP);
1702 case NFPROTO_IPV6:
1703 break;
1704 }
1705
1706 return htons(ETH_P_IPV6);
1707}
1708
1709int bpf_prog_test_run_nf(struct bpf_prog *prog,
1710 const union bpf_attr *kattr,
1711 union bpf_attr __user *uattr)
1712{
1713 struct net *net = current->nsproxy->net_ns;
1714 struct net_device *dev = net->loopback_dev;
1715 struct nf_hook_state *user_ctx, hook_state = {
1716 .pf = NFPROTO_IPV4,
1717 .hook = NF_INET_LOCAL_OUT,
1718 };
1719 u32 size = kattr->test.data_size_in;
1720 u32 repeat = kattr->test.repeat;
1721 struct bpf_nf_ctx ctx = {
1722 .state = &hook_state,
1723 };
1724 struct sk_buff *skb = NULL;
1725 u32 retval, duration;
1726 void *data;
1727 int ret;
1728
1729 if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size)
1730 return -EINVAL;
1731
1732 if (size < sizeof(struct iphdr))
1733 return -EINVAL;
1734
1735 data = bpf_test_init(kattr, user_size: kattr->test.data_size_in, size,
1736 NET_SKB_PAD + NET_IP_ALIGN,
1737 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));
1738 if (IS_ERR(ptr: data))
1739 return PTR_ERR(ptr: data);
1740
1741 if (!repeat)
1742 repeat = 1;
1743
1744 user_ctx = bpf_ctx_init(kattr, max_size: sizeof(struct nf_hook_state));
1745 if (IS_ERR(ptr: user_ctx)) {
1746 kfree(objp: data);
1747 return PTR_ERR(ptr: user_ctx);
1748 }
1749
1750 if (user_ctx) {
1751 ret = verify_and_copy_hook_state(state: &hook_state, user: user_ctx, dev);
1752 if (ret)
1753 goto out;
1754 }
1755
1756 skb = slab_build_skb(data);
1757 if (!skb) {
1758 ret = -ENOMEM;
1759 goto out;
1760 }
1761
1762 data = NULL; /* data released via kfree_skb */
1763
1764 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
1765 __skb_put(skb, len: size);
1766
1767 ret = -EINVAL;
1768
1769 if (hook_state.hook != NF_INET_LOCAL_OUT) {
1770 if (size < ETH_HLEN + sizeof(struct iphdr))
1771 goto out;
1772
1773 skb->protocol = eth_type_trans(skb, dev);
1774 switch (skb->protocol) {
1775 case htons(ETH_P_IP):
1776 if (hook_state.pf == NFPROTO_IPV4)
1777 break;
1778 goto out;
1779 case htons(ETH_P_IPV6):
1780 if (size < ETH_HLEN + sizeof(struct ipv6hdr))
1781 goto out;
1782 if (hook_state.pf == NFPROTO_IPV6)
1783 break;
1784 goto out;
1785 default:
1786 ret = -EPROTO;
1787 goto out;
1788 }
1789
1790 skb_reset_network_header(skb);
1791 } else {
1792 skb->protocol = nfproto_eth(nfproto: hook_state.pf);
1793 }
1794
1795 ctx.skb = skb;
1796
1797 ret = bpf_test_run(prog, ctx: &ctx, repeat, retval: &retval, time: &duration, xdp: false);
1798 if (ret)
1799 goto out;
1800
1801 ret = bpf_test_finish(kattr, uattr, NULL, NULL, size: 0, frag_size: 0, retval, duration);
1802
1803out:
1804 kfree(objp: user_ctx);
1805 kfree_skb(skb);
1806 kfree(objp: data);
1807 return ret;
1808}
1809
1810static const struct btf_kfunc_id_set bpf_prog_test_kfunc_set = {
1811 .owner = THIS_MODULE,
1812 .set = &test_sk_check_kfunc_ids,
1813};
1814
1815BTF_ID_LIST(bpf_prog_test_dtor_kfunc_ids)
1816BTF_ID(struct, prog_test_ref_kfunc)
1817BTF_ID(func, bpf_kfunc_call_test_release_dtor)
1818BTF_ID(struct, prog_test_member)
1819BTF_ID(func, bpf_kfunc_call_memb_release_dtor)
1820
1821static int __init bpf_prog_test_run_init(void)
1822{
1823 const struct btf_id_dtor_kfunc bpf_prog_test_dtor_kfunc[] = {
1824 {
1825 .btf_id = bpf_prog_test_dtor_kfunc_ids[0],
1826 .kfunc_btf_id = bpf_prog_test_dtor_kfunc_ids[1]
1827 },
1828 {
1829 .btf_id = bpf_prog_test_dtor_kfunc_ids[2],
1830 .kfunc_btf_id = bpf_prog_test_dtor_kfunc_ids[3],
1831 },
1832 };
1833 int ret;
1834
1835 ret = register_btf_fmodret_id_set(kset: &bpf_test_modify_return_set);
1836 ret = ret ?: register_btf_kfunc_id_set(prog_type: BPF_PROG_TYPE_SCHED_CLS, s: &bpf_prog_test_kfunc_set);
1837 ret = ret ?: register_btf_kfunc_id_set(prog_type: BPF_PROG_TYPE_TRACING, s: &bpf_prog_test_kfunc_set);
1838 ret = ret ?: register_btf_kfunc_id_set(prog_type: BPF_PROG_TYPE_SYSCALL, s: &bpf_prog_test_kfunc_set);
1839 return ret ?: register_btf_id_dtor_kfuncs(dtors: bpf_prog_test_dtor_kfunc,
1840 ARRAY_SIZE(bpf_prog_test_dtor_kfunc),
1841 THIS_MODULE);
1842}
1843late_initcall(bpf_prog_test_run_init);
1844

source code of linux/net/bpf/test_run.c