1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (c) 2023 Isovalent */
3
4#include <linux/netdevice.h>
5#include <linux/ethtool.h>
6#include <linux/etherdevice.h>
7#include <linux/filter.h>
8#include <linux/netfilter_netdev.h>
9#include <linux/bpf_mprog.h>
10#include <linux/indirect_call_wrapper.h>
11
12#include <net/netkit.h>
13#include <net/dst.h>
14#include <net/tcx.h>
15
16#define DRV_NAME "netkit"
17
18struct netkit {
19 __cacheline_group_begin(netkit_fastpath);
20 struct net_device __rcu *peer;
21 struct bpf_mprog_entry __rcu *active;
22 enum netkit_action policy;
23 enum netkit_scrub scrub;
24 struct bpf_mprog_bundle bundle;
25 __cacheline_group_end(netkit_fastpath);
26
27 __cacheline_group_begin(netkit_slowpath);
28 enum netkit_mode mode;
29 bool primary;
30 u32 headroom;
31 __cacheline_group_end(netkit_slowpath);
32};
33
34struct netkit_link {
35 struct bpf_link link;
36 struct net_device *dev;
37};
38
39static __always_inline int
40netkit_run(const struct bpf_mprog_entry *entry, struct sk_buff *skb,
41 enum netkit_action ret)
42{
43 const struct bpf_mprog_fp *fp;
44 const struct bpf_prog *prog;
45
46 bpf_mprog_foreach_prog(entry, fp, prog) {
47 bpf_compute_data_pointers(skb);
48 ret = bpf_prog_run(prog, ctx: skb);
49 if (ret != NETKIT_NEXT)
50 break;
51 }
52 return ret;
53}
54
55static void netkit_xnet(struct sk_buff *skb)
56{
57 skb->priority = 0;
58 skb->mark = 0;
59}
60
61static void netkit_prep_forward(struct sk_buff *skb,
62 bool xnet, bool xnet_scrub)
63{
64 skb_scrub_packet(skb, xnet: false);
65 nf_skip_egress(skb, skip: true);
66 skb_reset_mac_header(skb);
67 if (!xnet)
68 return;
69 skb_clear_tstamp(skb);
70 if (xnet_scrub)
71 netkit_xnet(skb);
72}
73
74static struct netkit *netkit_priv(const struct net_device *dev)
75{
76 return netdev_priv(dev);
77}
78
79static netdev_tx_t netkit_xmit(struct sk_buff *skb, struct net_device *dev)
80{
81 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
82 struct netkit *nk = netkit_priv(dev);
83 enum netkit_action ret = READ_ONCE(nk->policy);
84 netdev_tx_t ret_dev = NET_XMIT_SUCCESS;
85 const struct bpf_mprog_entry *entry;
86 struct net_device *peer;
87 int len = skb->len;
88
89 bpf_net_ctx = bpf_net_ctx_set(bpf_net_ctx: &__bpf_net_ctx);
90 rcu_read_lock();
91 peer = rcu_dereference(nk->peer);
92 if (unlikely(!peer || !(peer->flags & IFF_UP) ||
93 !pskb_may_pull(skb, ETH_HLEN) ||
94 skb_orphan_frags(skb, GFP_ATOMIC)))
95 goto drop;
96 netkit_prep_forward(skb, xnet: !net_eq(net1: dev_net(dev), net2: dev_net(dev: peer)),
97 xnet_scrub: nk->scrub);
98 eth_skb_pkt_type(skb, dev: peer);
99 skb->dev = peer;
100 entry = rcu_dereference(nk->active);
101 if (entry)
102 ret = netkit_run(entry, skb, ret);
103 switch (ret) {
104 case NETKIT_NEXT:
105 case NETKIT_PASS:
106 eth_skb_pull_mac(skb);
107 skb_postpull_rcsum(skb, start: eth_hdr(skb), ETH_HLEN);
108 if (likely(__netif_rx(skb) == NET_RX_SUCCESS)) {
109 dev_sw_netstats_tx_add(dev, packets: 1, len);
110 dev_sw_netstats_rx_add(dev: peer, len);
111 } else {
112 goto drop_stats;
113 }
114 break;
115 case NETKIT_REDIRECT:
116 dev_sw_netstats_tx_add(dev, packets: 1, len);
117 skb_do_redirect(skb);
118 break;
119 case NETKIT_DROP:
120 default:
121drop:
122 kfree_skb(skb);
123drop_stats:
124 dev_core_stats_tx_dropped_inc(dev);
125 ret_dev = NET_XMIT_DROP;
126 break;
127 }
128 rcu_read_unlock();
129 bpf_net_ctx_clear(bpf_net_ctx);
130 return ret_dev;
131}
132
133static int netkit_open(struct net_device *dev)
134{
135 struct netkit *nk = netkit_priv(dev);
136 struct net_device *peer = rtnl_dereference(nk->peer);
137
138 if (!peer)
139 return -ENOTCONN;
140 if (peer->flags & IFF_UP) {
141 netif_carrier_on(dev);
142 netif_carrier_on(dev: peer);
143 }
144 return 0;
145}
146
147static int netkit_close(struct net_device *dev)
148{
149 struct netkit *nk = netkit_priv(dev);
150 struct net_device *peer = rtnl_dereference(nk->peer);
151
152 netif_carrier_off(dev);
153 if (peer)
154 netif_carrier_off(dev: peer);
155 return 0;
156}
157
158static int netkit_get_iflink(const struct net_device *dev)
159{
160 struct netkit *nk = netkit_priv(dev);
161 struct net_device *peer;
162 int iflink = 0;
163
164 rcu_read_lock();
165 peer = rcu_dereference(nk->peer);
166 if (peer)
167 iflink = READ_ONCE(peer->ifindex);
168 rcu_read_unlock();
169 return iflink;
170}
171
172static void netkit_set_multicast(struct net_device *dev)
173{
174 /* Nothing to do, we receive whatever gets pushed to us! */
175}
176
177static int netkit_set_macaddr(struct net_device *dev, void *sa)
178{
179 struct netkit *nk = netkit_priv(dev);
180
181 if (nk->mode != NETKIT_L2)
182 return -EOPNOTSUPP;
183
184 return eth_mac_addr(dev, p: sa);
185}
186
187static void netkit_set_headroom(struct net_device *dev, int headroom)
188{
189 struct netkit *nk = netkit_priv(dev), *nk2;
190 struct net_device *peer;
191
192 if (headroom < 0)
193 headroom = NET_SKB_PAD;
194
195 rcu_read_lock();
196 peer = rcu_dereference(nk->peer);
197 if (unlikely(!peer))
198 goto out;
199
200 nk2 = netkit_priv(dev: peer);
201 nk->headroom = headroom;
202 headroom = max(nk->headroom, nk2->headroom);
203
204 peer->needed_headroom = headroom;
205 dev->needed_headroom = headroom;
206out:
207 rcu_read_unlock();
208}
209
210INDIRECT_CALLABLE_SCOPE struct net_device *netkit_peer_dev(struct net_device *dev)
211{
212 return rcu_dereference(netkit_priv(dev)->peer);
213}
214
215static void netkit_get_stats(struct net_device *dev,
216 struct rtnl_link_stats64 *stats)
217{
218 dev_fetch_sw_netstats(s: stats, netstats: dev->tstats);
219 stats->tx_dropped = DEV_STATS_READ(dev, tx_dropped);
220}
221
222static void netkit_uninit(struct net_device *dev);
223
224static const struct net_device_ops netkit_netdev_ops = {
225 .ndo_open = netkit_open,
226 .ndo_stop = netkit_close,
227 .ndo_start_xmit = netkit_xmit,
228 .ndo_set_rx_mode = netkit_set_multicast,
229 .ndo_set_rx_headroom = netkit_set_headroom,
230 .ndo_set_mac_address = netkit_set_macaddr,
231 .ndo_get_iflink = netkit_get_iflink,
232 .ndo_get_peer_dev = netkit_peer_dev,
233 .ndo_get_stats64 = netkit_get_stats,
234 .ndo_uninit = netkit_uninit,
235 .ndo_features_check = passthru_features_check,
236};
237
238static void netkit_get_drvinfo(struct net_device *dev,
239 struct ethtool_drvinfo *info)
240{
241 strscpy(info->driver, DRV_NAME, sizeof(info->driver));
242}
243
244static const struct ethtool_ops netkit_ethtool_ops = {
245 .get_drvinfo = netkit_get_drvinfo,
246};
247
248static void netkit_setup(struct net_device *dev)
249{
250 static const netdev_features_t netkit_features_hw_vlan =
251 NETIF_F_HW_VLAN_CTAG_TX |
252 NETIF_F_HW_VLAN_CTAG_RX |
253 NETIF_F_HW_VLAN_STAG_TX |
254 NETIF_F_HW_VLAN_STAG_RX;
255 static const netdev_features_t netkit_features =
256 netkit_features_hw_vlan |
257 NETIF_F_SG |
258 NETIF_F_FRAGLIST |
259 NETIF_F_HW_CSUM |
260 NETIF_F_RXCSUM |
261 NETIF_F_SCTP_CRC |
262 NETIF_F_HIGHDMA |
263 NETIF_F_GSO_SOFTWARE |
264 NETIF_F_GSO_ENCAP_ALL;
265
266 ether_setup(dev);
267 dev->max_mtu = ETH_MAX_MTU;
268 dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS;
269
270 dev->flags |= IFF_NOARP;
271 dev->priv_flags &= ~IFF_TX_SKB_SHARING;
272 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
273 dev->priv_flags |= IFF_PHONY_HEADROOM;
274 dev->priv_flags |= IFF_NO_QUEUE;
275 dev->priv_flags |= IFF_DISABLE_NETPOLL;
276 dev->lltx = true;
277
278 dev->ethtool_ops = &netkit_ethtool_ops;
279 dev->netdev_ops = &netkit_netdev_ops;
280
281 dev->features |= netkit_features;
282 dev->hw_features = netkit_features;
283 dev->hw_enc_features = netkit_features;
284 dev->mpls_features = NETIF_F_HW_CSUM | NETIF_F_GSO_SOFTWARE;
285 dev->vlan_features = dev->features & ~netkit_features_hw_vlan;
286
287 dev->needs_free_netdev = true;
288
289 netif_set_tso_max_size(dev, GSO_MAX_SIZE);
290}
291
292static struct net *netkit_get_link_net(const struct net_device *dev)
293{
294 struct netkit *nk = netkit_priv(dev);
295 struct net_device *peer = rtnl_dereference(nk->peer);
296
297 return peer ? dev_net(dev: peer) : dev_net(dev);
298}
299
300static int netkit_check_policy(int policy, struct nlattr *tb,
301 struct netlink_ext_ack *extack)
302{
303 switch (policy) {
304 case NETKIT_PASS:
305 case NETKIT_DROP:
306 return 0;
307 default:
308 NL_SET_ERR_MSG_ATTR(extack, tb,
309 "Provided default xmit policy not supported");
310 return -EINVAL;
311 }
312}
313
314static int netkit_validate(struct nlattr *tb[], struct nlattr *data[],
315 struct netlink_ext_ack *extack)
316{
317 struct nlattr *attr = tb[IFLA_ADDRESS];
318
319 if (!attr)
320 return 0;
321 if (nla_len(nla: attr) != ETH_ALEN)
322 return -EINVAL;
323 if (!is_valid_ether_addr(addr: nla_data(nla: attr)))
324 return -EADDRNOTAVAIL;
325 return 0;
326}
327
328static struct rtnl_link_ops netkit_link_ops;
329
330static int netkit_new_link(struct net_device *dev,
331 struct rtnl_newlink_params *params,
332 struct netlink_ext_ack *extack)
333{
334 struct net *peer_net = rtnl_newlink_peer_net(p: params);
335 enum netkit_scrub scrub_prim = NETKIT_SCRUB_DEFAULT;
336 enum netkit_scrub scrub_peer = NETKIT_SCRUB_DEFAULT;
337 struct nlattr *peer_tb[IFLA_MAX + 1], **tbp, *attr;
338 enum netkit_action policy_prim = NETKIT_PASS;
339 enum netkit_action policy_peer = NETKIT_PASS;
340 struct nlattr **data = params->data;
341 enum netkit_mode mode = NETKIT_L3;
342 unsigned char ifname_assign_type;
343 struct nlattr **tb = params->tb;
344 u16 headroom = 0, tailroom = 0;
345 struct ifinfomsg *ifmp = NULL;
346 struct net_device *peer;
347 char ifname[IFNAMSIZ];
348 struct netkit *nk;
349 int err;
350
351 tbp = tb;
352 if (data) {
353 if (data[IFLA_NETKIT_MODE])
354 mode = nla_get_u32(nla: data[IFLA_NETKIT_MODE]);
355 if (data[IFLA_NETKIT_PEER_INFO]) {
356 attr = data[IFLA_NETKIT_PEER_INFO];
357 ifmp = nla_data(nla: attr);
358 rtnl_nla_parse_ifinfomsg(tb: peer_tb, nla_peer: attr, exterr: extack);
359 tbp = peer_tb;
360 }
361 if (data[IFLA_NETKIT_SCRUB])
362 scrub_prim = nla_get_u32(nla: data[IFLA_NETKIT_SCRUB]);
363 if (data[IFLA_NETKIT_PEER_SCRUB])
364 scrub_peer = nla_get_u32(nla: data[IFLA_NETKIT_PEER_SCRUB]);
365 if (data[IFLA_NETKIT_POLICY]) {
366 attr = data[IFLA_NETKIT_POLICY];
367 policy_prim = nla_get_u32(nla: attr);
368 err = netkit_check_policy(policy: policy_prim, tb: attr, extack);
369 if (err < 0)
370 return err;
371 }
372 if (data[IFLA_NETKIT_PEER_POLICY]) {
373 attr = data[IFLA_NETKIT_PEER_POLICY];
374 policy_peer = nla_get_u32(nla: attr);
375 err = netkit_check_policy(policy: policy_peer, tb: attr, extack);
376 if (err < 0)
377 return err;
378 }
379 if (data[IFLA_NETKIT_HEADROOM])
380 headroom = nla_get_u16(nla: data[IFLA_NETKIT_HEADROOM]);
381 if (data[IFLA_NETKIT_TAILROOM])
382 tailroom = nla_get_u16(nla: data[IFLA_NETKIT_TAILROOM]);
383 }
384
385 if (ifmp && tbp[IFLA_IFNAME]) {
386 nla_strscpy(dst: ifname, nla: tbp[IFLA_IFNAME], IFNAMSIZ);
387 ifname_assign_type = NET_NAME_USER;
388 } else {
389 strscpy(ifname, "nk%d", IFNAMSIZ);
390 ifname_assign_type = NET_NAME_ENUM;
391 }
392 if (mode != NETKIT_L2 &&
393 (tb[IFLA_ADDRESS] || tbp[IFLA_ADDRESS]))
394 return -EOPNOTSUPP;
395
396 peer = rtnl_create_link(net: peer_net, ifname, name_assign_type: ifname_assign_type,
397 ops: &netkit_link_ops, tb: tbp, extack);
398 if (IS_ERR(ptr: peer))
399 return PTR_ERR(ptr: peer);
400
401 netif_inherit_tso_max(to: peer, from: dev);
402 if (headroom) {
403 peer->needed_headroom = headroom;
404 dev->needed_headroom = headroom;
405 }
406 if (tailroom) {
407 peer->needed_tailroom = tailroom;
408 dev->needed_tailroom = tailroom;
409 }
410
411 if (mode == NETKIT_L2 && !(ifmp && tbp[IFLA_ADDRESS]))
412 eth_hw_addr_random(dev: peer);
413 if (ifmp && dev->ifindex)
414 peer->ifindex = ifmp->ifi_index;
415
416 nk = netkit_priv(dev: peer);
417 nk->primary = false;
418 nk->policy = policy_peer;
419 nk->scrub = scrub_peer;
420 nk->mode = mode;
421 nk->headroom = headroom;
422 bpf_mprog_bundle_init(bundle: &nk->bundle);
423
424 err = register_netdevice(dev: peer);
425 if (err < 0)
426 goto err_register_peer;
427 netif_carrier_off(dev: peer);
428 if (mode == NETKIT_L2)
429 dev_change_flags(dev: peer, flags: peer->flags & ~IFF_NOARP, NULL);
430
431 err = rtnl_configure_link(dev: peer, NULL, portid: 0, NULL);
432 if (err < 0)
433 goto err_configure_peer;
434
435 if (mode == NETKIT_L2 && !tb[IFLA_ADDRESS])
436 eth_hw_addr_random(dev);
437 if (tb[IFLA_IFNAME])
438 nla_strscpy(dst: dev->name, nla: tb[IFLA_IFNAME], IFNAMSIZ);
439 else
440 strscpy(dev->name, "nk%d", IFNAMSIZ);
441
442 nk = netkit_priv(dev);
443 nk->primary = true;
444 nk->policy = policy_prim;
445 nk->scrub = scrub_prim;
446 nk->mode = mode;
447 nk->headroom = headroom;
448 bpf_mprog_bundle_init(bundle: &nk->bundle);
449
450 err = register_netdevice(dev);
451 if (err < 0)
452 goto err_configure_peer;
453 netif_carrier_off(dev);
454 if (mode == NETKIT_L2)
455 dev_change_flags(dev, flags: dev->flags & ~IFF_NOARP, NULL);
456
457 rcu_assign_pointer(netkit_priv(dev)->peer, peer);
458 rcu_assign_pointer(netkit_priv(peer)->peer, dev);
459 return 0;
460err_configure_peer:
461 unregister_netdevice(dev: peer);
462 return err;
463err_register_peer:
464 free_netdev(dev: peer);
465 return err;
466}
467
468static struct bpf_mprog_entry *netkit_entry_fetch(struct net_device *dev,
469 bool bundle_fallback)
470{
471 struct netkit *nk = netkit_priv(dev);
472 struct bpf_mprog_entry *entry;
473
474 ASSERT_RTNL();
475 entry = rcu_dereference_rtnl(nk->active);
476 if (entry)
477 return entry;
478 if (bundle_fallback)
479 return &nk->bundle.a;
480 return NULL;
481}
482
483static void netkit_entry_update(struct net_device *dev,
484 struct bpf_mprog_entry *entry)
485{
486 struct netkit *nk = netkit_priv(dev);
487
488 ASSERT_RTNL();
489 rcu_assign_pointer(nk->active, entry);
490}
491
492static void netkit_entry_sync(void)
493{
494 synchronize_rcu();
495}
496
497static struct net_device *netkit_dev_fetch(struct net *net, u32 ifindex, u32 which)
498{
499 struct net_device *dev;
500 struct netkit *nk;
501
502 ASSERT_RTNL();
503
504 switch (which) {
505 case BPF_NETKIT_PRIMARY:
506 case BPF_NETKIT_PEER:
507 break;
508 default:
509 return ERR_PTR(error: -EINVAL);
510 }
511
512 dev = __dev_get_by_index(net, ifindex);
513 if (!dev)
514 return ERR_PTR(error: -ENODEV);
515 if (dev->netdev_ops != &netkit_netdev_ops)
516 return ERR_PTR(error: -ENXIO);
517
518 nk = netkit_priv(dev);
519 if (!nk->primary)
520 return ERR_PTR(error: -EACCES);
521 if (which == BPF_NETKIT_PEER) {
522 dev = rcu_dereference_rtnl(nk->peer);
523 if (!dev)
524 return ERR_PTR(error: -ENODEV);
525 }
526 return dev;
527}
528
529int netkit_prog_attach(const union bpf_attr *attr, struct bpf_prog *prog)
530{
531 struct bpf_mprog_entry *entry, *entry_new;
532 struct bpf_prog *replace_prog = NULL;
533 struct net_device *dev;
534 int ret;
535
536 rtnl_lock();
537 dev = netkit_dev_fetch(current->nsproxy->net_ns, ifindex: attr->target_ifindex,
538 which: attr->attach_type);
539 if (IS_ERR(ptr: dev)) {
540 ret = PTR_ERR(ptr: dev);
541 goto out;
542 }
543 entry = netkit_entry_fetch(dev, bundle_fallback: true);
544 if (attr->attach_flags & BPF_F_REPLACE) {
545 replace_prog = bpf_prog_get_type(ufd: attr->replace_bpf_fd,
546 type: prog->type);
547 if (IS_ERR(ptr: replace_prog)) {
548 ret = PTR_ERR(ptr: replace_prog);
549 replace_prog = NULL;
550 goto out;
551 }
552 }
553 ret = bpf_mprog_attach(entry, entry_new: &entry_new, prog_new: prog, NULL, prog_old: replace_prog,
554 flags: attr->attach_flags, id_or_fd: attr->relative_fd,
555 revision: attr->expected_revision);
556 if (!ret) {
557 if (entry != entry_new) {
558 netkit_entry_update(dev, entry: entry_new);
559 netkit_entry_sync();
560 }
561 bpf_mprog_commit(entry);
562 }
563out:
564 if (replace_prog)
565 bpf_prog_put(prog: replace_prog);
566 rtnl_unlock();
567 return ret;
568}
569
570int netkit_prog_detach(const union bpf_attr *attr, struct bpf_prog *prog)
571{
572 struct bpf_mprog_entry *entry, *entry_new;
573 struct net_device *dev;
574 int ret;
575
576 rtnl_lock();
577 dev = netkit_dev_fetch(current->nsproxy->net_ns, ifindex: attr->target_ifindex,
578 which: attr->attach_type);
579 if (IS_ERR(ptr: dev)) {
580 ret = PTR_ERR(ptr: dev);
581 goto out;
582 }
583 entry = netkit_entry_fetch(dev, bundle_fallback: false);
584 if (!entry) {
585 ret = -ENOENT;
586 goto out;
587 }
588 ret = bpf_mprog_detach(entry, entry_new: &entry_new, prog, NULL, flags: attr->attach_flags,
589 id_or_fd: attr->relative_fd, revision: attr->expected_revision);
590 if (!ret) {
591 if (!bpf_mprog_total(entry: entry_new))
592 entry_new = NULL;
593 netkit_entry_update(dev, entry: entry_new);
594 netkit_entry_sync();
595 bpf_mprog_commit(entry);
596 }
597out:
598 rtnl_unlock();
599 return ret;
600}
601
602int netkit_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr)
603{
604 struct net_device *dev;
605 int ret;
606
607 rtnl_lock();
608 dev = netkit_dev_fetch(current->nsproxy->net_ns,
609 ifindex: attr->query.target_ifindex,
610 which: attr->query.attach_type);
611 if (IS_ERR(ptr: dev)) {
612 ret = PTR_ERR(ptr: dev);
613 goto out;
614 }
615 ret = bpf_mprog_query(attr, uattr, entry: netkit_entry_fetch(dev, bundle_fallback: false));
616out:
617 rtnl_unlock();
618 return ret;
619}
620
621static struct netkit_link *netkit_link(const struct bpf_link *link)
622{
623 return container_of(link, struct netkit_link, link);
624}
625
626static int netkit_link_prog_attach(struct bpf_link *link, u32 flags,
627 u32 id_or_fd, u64 revision)
628{
629 struct netkit_link *nkl = netkit_link(link);
630 struct bpf_mprog_entry *entry, *entry_new;
631 struct net_device *dev = nkl->dev;
632 int ret;
633
634 ASSERT_RTNL();
635 entry = netkit_entry_fetch(dev, bundle_fallback: true);
636 ret = bpf_mprog_attach(entry, entry_new: &entry_new, prog_new: link->prog, link, NULL, flags,
637 id_or_fd, revision);
638 if (!ret) {
639 if (entry != entry_new) {
640 netkit_entry_update(dev, entry: entry_new);
641 netkit_entry_sync();
642 }
643 bpf_mprog_commit(entry);
644 }
645 return ret;
646}
647
648static void netkit_link_release(struct bpf_link *link)
649{
650 struct netkit_link *nkl = netkit_link(link);
651 struct bpf_mprog_entry *entry, *entry_new;
652 struct net_device *dev;
653 int ret = 0;
654
655 rtnl_lock();
656 dev = nkl->dev;
657 if (!dev)
658 goto out;
659 entry = netkit_entry_fetch(dev, bundle_fallback: false);
660 if (!entry) {
661 ret = -ENOENT;
662 goto out;
663 }
664 ret = bpf_mprog_detach(entry, entry_new: &entry_new, prog: link->prog, link, flags: 0, id_or_fd: 0, revision: 0);
665 if (!ret) {
666 if (!bpf_mprog_total(entry: entry_new))
667 entry_new = NULL;
668 netkit_entry_update(dev, entry: entry_new);
669 netkit_entry_sync();
670 bpf_mprog_commit(entry);
671 nkl->dev = NULL;
672 }
673out:
674 WARN_ON_ONCE(ret);
675 rtnl_unlock();
676}
677
678static int netkit_link_update(struct bpf_link *link, struct bpf_prog *nprog,
679 struct bpf_prog *oprog)
680{
681 struct netkit_link *nkl = netkit_link(link);
682 struct bpf_mprog_entry *entry, *entry_new;
683 struct net_device *dev;
684 int ret = 0;
685
686 rtnl_lock();
687 dev = nkl->dev;
688 if (!dev) {
689 ret = -ENOLINK;
690 goto out;
691 }
692 if (oprog && link->prog != oprog) {
693 ret = -EPERM;
694 goto out;
695 }
696 oprog = link->prog;
697 if (oprog == nprog) {
698 bpf_prog_put(prog: nprog);
699 goto out;
700 }
701 entry = netkit_entry_fetch(dev, bundle_fallback: false);
702 if (!entry) {
703 ret = -ENOENT;
704 goto out;
705 }
706 ret = bpf_mprog_attach(entry, entry_new: &entry_new, prog_new: nprog, link, prog_old: oprog,
707 BPF_F_REPLACE | BPF_F_ID,
708 id_or_fd: link->prog->aux->id, revision: 0);
709 if (!ret) {
710 WARN_ON_ONCE(entry != entry_new);
711 oprog = xchg(&link->prog, nprog);
712 bpf_prog_put(prog: oprog);
713 bpf_mprog_commit(entry);
714 }
715out:
716 rtnl_unlock();
717 return ret;
718}
719
720static void netkit_link_dealloc(struct bpf_link *link)
721{
722 kfree(objp: netkit_link(link));
723}
724
725static void netkit_link_fdinfo(const struct bpf_link *link, struct seq_file *seq)
726{
727 const struct netkit_link *nkl = netkit_link(link);
728 u32 ifindex = 0;
729
730 rtnl_lock();
731 if (nkl->dev)
732 ifindex = nkl->dev->ifindex;
733 rtnl_unlock();
734
735 seq_printf(m: seq, fmt: "ifindex:\t%u\n", ifindex);
736 seq_printf(m: seq, fmt: "attach_type:\t%u (%s)\n",
737 link->attach_type,
738 link->attach_type == BPF_NETKIT_PRIMARY ? "primary" : "peer");
739}
740
741static int netkit_link_fill_info(const struct bpf_link *link,
742 struct bpf_link_info *info)
743{
744 const struct netkit_link *nkl = netkit_link(link);
745 u32 ifindex = 0;
746
747 rtnl_lock();
748 if (nkl->dev)
749 ifindex = nkl->dev->ifindex;
750 rtnl_unlock();
751
752 info->netkit.ifindex = ifindex;
753 info->netkit.attach_type = link->attach_type;
754 return 0;
755}
756
757static int netkit_link_detach(struct bpf_link *link)
758{
759 netkit_link_release(link);
760 return 0;
761}
762
763static const struct bpf_link_ops netkit_link_lops = {
764 .release = netkit_link_release,
765 .detach = netkit_link_detach,
766 .dealloc = netkit_link_dealloc,
767 .update_prog = netkit_link_update,
768 .show_fdinfo = netkit_link_fdinfo,
769 .fill_link_info = netkit_link_fill_info,
770};
771
772static int netkit_link_init(struct netkit_link *nkl,
773 struct bpf_link_primer *link_primer,
774 const union bpf_attr *attr,
775 struct net_device *dev,
776 struct bpf_prog *prog)
777{
778 bpf_link_init(link: &nkl->link, type: BPF_LINK_TYPE_NETKIT,
779 ops: &netkit_link_lops, prog, attach_type: attr->link_create.attach_type);
780 nkl->dev = dev;
781 return bpf_link_prime(link: &nkl->link, primer: link_primer);
782}
783
784int netkit_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
785{
786 struct bpf_link_primer link_primer;
787 struct netkit_link *nkl;
788 struct net_device *dev;
789 int ret;
790
791 rtnl_lock();
792 dev = netkit_dev_fetch(current->nsproxy->net_ns,
793 ifindex: attr->link_create.target_ifindex,
794 which: attr->link_create.attach_type);
795 if (IS_ERR(ptr: dev)) {
796 ret = PTR_ERR(ptr: dev);
797 goto out;
798 }
799 nkl = kzalloc(sizeof(*nkl), GFP_KERNEL_ACCOUNT);
800 if (!nkl) {
801 ret = -ENOMEM;
802 goto out;
803 }
804 ret = netkit_link_init(nkl, link_primer: &link_primer, attr, dev, prog);
805 if (ret) {
806 kfree(objp: nkl);
807 goto out;
808 }
809 ret = netkit_link_prog_attach(link: &nkl->link,
810 flags: attr->link_create.flags,
811 id_or_fd: attr->link_create.netkit.relative_fd,
812 revision: attr->link_create.netkit.expected_revision);
813 if (ret) {
814 nkl->dev = NULL;
815 bpf_link_cleanup(primer: &link_primer);
816 goto out;
817 }
818 ret = bpf_link_settle(primer: &link_primer);
819out:
820 rtnl_unlock();
821 return ret;
822}
823
824static void netkit_release_all(struct net_device *dev)
825{
826 struct bpf_mprog_entry *entry;
827 struct bpf_tuple tuple = {};
828 struct bpf_mprog_fp *fp;
829 struct bpf_mprog_cp *cp;
830
831 entry = netkit_entry_fetch(dev, bundle_fallback: false);
832 if (!entry)
833 return;
834 netkit_entry_update(dev, NULL);
835 netkit_entry_sync();
836 bpf_mprog_foreach_tuple(entry, fp, cp, tuple) {
837 if (tuple.link)
838 netkit_link(link: tuple.link)->dev = NULL;
839 else
840 bpf_prog_put(prog: tuple.prog);
841 }
842}
843
844static void netkit_uninit(struct net_device *dev)
845{
846 netkit_release_all(dev);
847}
848
849static void netkit_del_link(struct net_device *dev, struct list_head *head)
850{
851 struct netkit *nk = netkit_priv(dev);
852 struct net_device *peer = rtnl_dereference(nk->peer);
853
854 RCU_INIT_POINTER(nk->peer, NULL);
855 unregister_netdevice_queue(dev, head);
856 if (peer) {
857 nk = netkit_priv(dev: peer);
858 RCU_INIT_POINTER(nk->peer, NULL);
859 unregister_netdevice_queue(dev: peer, head);
860 }
861}
862
863static int netkit_change_link(struct net_device *dev, struct nlattr *tb[],
864 struct nlattr *data[],
865 struct netlink_ext_ack *extack)
866{
867 struct netkit *nk = netkit_priv(dev);
868 struct net_device *peer = rtnl_dereference(nk->peer);
869 enum netkit_action policy;
870 struct nlattr *attr;
871 int err, i;
872 static const struct {
873 u32 attr;
874 char *name;
875 } fixed_params[] = {
876 { IFLA_NETKIT_MODE, "operating mode" },
877 { IFLA_NETKIT_SCRUB, "scrubbing" },
878 { IFLA_NETKIT_PEER_SCRUB, "peer scrubbing" },
879 { IFLA_NETKIT_PEER_INFO, "peer info" },
880 { IFLA_NETKIT_HEADROOM, "headroom" },
881 { IFLA_NETKIT_TAILROOM, "tailroom" },
882 };
883
884 if (!nk->primary) {
885 NL_SET_ERR_MSG(extack,
886 "netkit link settings can be changed only through the primary device");
887 return -EACCES;
888 }
889
890 for (i = 0; i < ARRAY_SIZE(fixed_params); i++) {
891 attr = data[fixed_params[i].attr];
892 if (attr) {
893 NL_SET_ERR_MSG_ATTR_FMT(extack, attr,
894 "netkit link %s cannot be changed after device creation",
895 fixed_params[i].name);
896 return -EACCES;
897 }
898 }
899
900 if (data[IFLA_NETKIT_POLICY]) {
901 attr = data[IFLA_NETKIT_POLICY];
902 policy = nla_get_u32(nla: attr);
903 err = netkit_check_policy(policy, tb: attr, extack);
904 if (err)
905 return err;
906 WRITE_ONCE(nk->policy, policy);
907 }
908
909 if (data[IFLA_NETKIT_PEER_POLICY]) {
910 err = -EOPNOTSUPP;
911 attr = data[IFLA_NETKIT_PEER_POLICY];
912 policy = nla_get_u32(nla: attr);
913 if (peer)
914 err = netkit_check_policy(policy, tb: attr, extack);
915 if (err)
916 return err;
917 nk = netkit_priv(dev: peer);
918 WRITE_ONCE(nk->policy, policy);
919 }
920
921 return 0;
922}
923
924static size_t netkit_get_size(const struct net_device *dev)
925{
926 return nla_total_size(payload: sizeof(u32)) + /* IFLA_NETKIT_POLICY */
927 nla_total_size(payload: sizeof(u32)) + /* IFLA_NETKIT_PEER_POLICY */
928 nla_total_size(payload: sizeof(u32)) + /* IFLA_NETKIT_SCRUB */
929 nla_total_size(payload: sizeof(u32)) + /* IFLA_NETKIT_PEER_SCRUB */
930 nla_total_size(payload: sizeof(u32)) + /* IFLA_NETKIT_MODE */
931 nla_total_size(payload: sizeof(u8)) + /* IFLA_NETKIT_PRIMARY */
932 nla_total_size(payload: sizeof(u16)) + /* IFLA_NETKIT_HEADROOM */
933 nla_total_size(payload: sizeof(u16)) + /* IFLA_NETKIT_TAILROOM */
934 0;
935}
936
937static int netkit_fill_info(struct sk_buff *skb, const struct net_device *dev)
938{
939 struct netkit *nk = netkit_priv(dev);
940 struct net_device *peer = rtnl_dereference(nk->peer);
941
942 if (nla_put_u8(skb, attrtype: IFLA_NETKIT_PRIMARY, value: nk->primary))
943 return -EMSGSIZE;
944 if (nla_put_u32(skb, attrtype: IFLA_NETKIT_POLICY, value: nk->policy))
945 return -EMSGSIZE;
946 if (nla_put_u32(skb, attrtype: IFLA_NETKIT_MODE, value: nk->mode))
947 return -EMSGSIZE;
948 if (nla_put_u32(skb, attrtype: IFLA_NETKIT_SCRUB, value: nk->scrub))
949 return -EMSGSIZE;
950 if (nla_put_u16(skb, attrtype: IFLA_NETKIT_HEADROOM, value: dev->needed_headroom))
951 return -EMSGSIZE;
952 if (nla_put_u16(skb, attrtype: IFLA_NETKIT_TAILROOM, value: dev->needed_tailroom))
953 return -EMSGSIZE;
954
955 if (peer) {
956 nk = netkit_priv(dev: peer);
957 if (nla_put_u32(skb, attrtype: IFLA_NETKIT_PEER_POLICY, value: nk->policy))
958 return -EMSGSIZE;
959 if (nla_put_u32(skb, attrtype: IFLA_NETKIT_PEER_SCRUB, value: nk->scrub))
960 return -EMSGSIZE;
961 }
962
963 return 0;
964}
965
966static const struct nla_policy netkit_policy[IFLA_NETKIT_MAX + 1] = {
967 [IFLA_NETKIT_PEER_INFO] = { .len = sizeof(struct ifinfomsg) },
968 [IFLA_NETKIT_MODE] = NLA_POLICY_MAX(NLA_U32, NETKIT_L3),
969 [IFLA_NETKIT_POLICY] = { .type = NLA_U32 },
970 [IFLA_NETKIT_PEER_POLICY] = { .type = NLA_U32 },
971 [IFLA_NETKIT_HEADROOM] = { .type = NLA_U16 },
972 [IFLA_NETKIT_TAILROOM] = { .type = NLA_U16 },
973 [IFLA_NETKIT_SCRUB] = NLA_POLICY_MAX(NLA_U32, NETKIT_SCRUB_DEFAULT),
974 [IFLA_NETKIT_PEER_SCRUB] = NLA_POLICY_MAX(NLA_U32, NETKIT_SCRUB_DEFAULT),
975 [IFLA_NETKIT_PRIMARY] = { .type = NLA_REJECT,
976 .reject_message = "Primary attribute is read-only" },
977};
978
979static struct rtnl_link_ops netkit_link_ops = {
980 .kind = DRV_NAME,
981 .priv_size = sizeof(struct netkit),
982 .setup = netkit_setup,
983 .newlink = netkit_new_link,
984 .dellink = netkit_del_link,
985 .changelink = netkit_change_link,
986 .get_link_net = netkit_get_link_net,
987 .get_size = netkit_get_size,
988 .fill_info = netkit_fill_info,
989 .policy = netkit_policy,
990 .validate = netkit_validate,
991 .peer_type = IFLA_NETKIT_PEER_INFO,
992 .maxtype = IFLA_NETKIT_MAX,
993};
994
995static __init int netkit_init(void)
996{
997 BUILD_BUG_ON((int)NETKIT_NEXT != (int)TCX_NEXT ||
998 (int)NETKIT_PASS != (int)TCX_PASS ||
999 (int)NETKIT_DROP != (int)TCX_DROP ||
1000 (int)NETKIT_REDIRECT != (int)TCX_REDIRECT);
1001
1002 return rtnl_link_register(ops: &netkit_link_ops);
1003}
1004
1005static __exit void netkit_exit(void)
1006{
1007 rtnl_link_unregister(ops: &netkit_link_ops);
1008}
1009
1010module_init(netkit_init);
1011module_exit(netkit_exit);
1012
1013MODULE_DESCRIPTION("BPF-programmable network device");
1014MODULE_AUTHOR("Daniel Borkmann <daniel@iogearbox.net>");
1015MODULE_AUTHOR("Nikolay Aleksandrov <razor@blackwall.org>");
1016MODULE_LICENSE("GPL");
1017MODULE_ALIAS_RTNL_LINK(DRV_NAME);
1018

source code of linux/drivers/net/netkit.c