1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2/* Copyright (c) 2021, Microsoft Corporation. */
3
4#include <linux/inetdevice.h>
5#include <linux/etherdevice.h>
6#include <linux/mm.h>
7#include <linux/bpf.h>
8#include <linux/bpf_trace.h>
9#include <net/xdp.h>
10
11#include <net/mana/mana.h>
12
13void mana_xdp_tx(struct sk_buff *skb, struct net_device *ndev)
14{
15 u16 txq_idx = skb_get_queue_mapping(skb);
16 struct netdev_queue *ndevtxq;
17 int rc;
18
19 __skb_push(skb, ETH_HLEN);
20
21 ndevtxq = netdev_get_tx_queue(dev: ndev, index: txq_idx);
22 __netif_tx_lock(txq: ndevtxq, smp_processor_id());
23
24 rc = mana_start_xmit(skb, ndev);
25
26 __netif_tx_unlock(txq: ndevtxq);
27
28 if (dev_xmit_complete(rc))
29 return;
30
31 dev_kfree_skb_any(skb);
32 ndev->stats.tx_dropped++;
33}
34
35static int mana_xdp_xmit_fm(struct net_device *ndev, struct xdp_frame *frame,
36 u16 q_idx)
37{
38 struct sk_buff *skb;
39
40 skb = xdp_build_skb_from_frame(xdpf: frame, dev: ndev);
41 if (unlikely(!skb))
42 return -ENOMEM;
43
44 skb_set_queue_mapping(skb, queue_mapping: q_idx);
45
46 mana_xdp_tx(skb, ndev);
47
48 return 0;
49}
50
51int mana_xdp_xmit(struct net_device *ndev, int n, struct xdp_frame **frames,
52 u32 flags)
53{
54 struct mana_port_context *apc = netdev_priv(dev: ndev);
55 struct mana_stats_tx *tx_stats;
56 int i, count = 0;
57 u16 q_idx;
58
59 if (unlikely(!apc->port_is_up))
60 return 0;
61
62 q_idx = smp_processor_id() % ndev->real_num_tx_queues;
63
64 for (i = 0; i < n; i++) {
65 if (mana_xdp_xmit_fm(ndev, frame: frames[i], q_idx))
66 break;
67
68 count++;
69 }
70
71 tx_stats = &apc->tx_qp[q_idx].txq.stats;
72
73 u64_stats_update_begin(syncp: &tx_stats->syncp);
74 tx_stats->xdp_xmit += count;
75 u64_stats_update_end(syncp: &tx_stats->syncp);
76
77 return count;
78}
79
80u32 mana_run_xdp(struct net_device *ndev, struct mana_rxq *rxq,
81 struct xdp_buff *xdp, void *buf_va, uint pkt_len)
82{
83 struct mana_stats_rx *rx_stats;
84 struct bpf_prog *prog;
85 u32 act = XDP_PASS;
86
87 rcu_read_lock();
88 prog = rcu_dereference(rxq->bpf_prog);
89
90 if (!prog)
91 goto out;
92
93 xdp_init_buff(xdp, PAGE_SIZE, rxq: &rxq->xdp_rxq);
94 xdp_prepare_buff(xdp, hard_start: buf_va, XDP_PACKET_HEADROOM, data_len: pkt_len, meta_valid: false);
95
96 act = bpf_prog_run_xdp(prog, xdp);
97
98 rx_stats = &rxq->stats;
99
100 switch (act) {
101 case XDP_PASS:
102 case XDP_TX:
103 case XDP_DROP:
104 break;
105
106 case XDP_REDIRECT:
107 rxq->xdp_rc = xdp_do_redirect(dev: ndev, xdp, prog);
108 if (!rxq->xdp_rc) {
109 rxq->xdp_flush = true;
110
111 u64_stats_update_begin(syncp: &rx_stats->syncp);
112 rx_stats->packets++;
113 rx_stats->bytes += pkt_len;
114 rx_stats->xdp_redirect++;
115 u64_stats_update_end(syncp: &rx_stats->syncp);
116
117 break;
118 }
119
120 fallthrough;
121
122 case XDP_ABORTED:
123 trace_xdp_exception(dev: ndev, xdp: prog, act);
124 break;
125
126 default:
127 bpf_warn_invalid_xdp_action(dev: ndev, prog, act);
128 }
129
130out:
131 rcu_read_unlock();
132
133 return act;
134}
135
136struct bpf_prog *mana_xdp_get(struct mana_port_context *apc)
137{
138 ASSERT_RTNL();
139
140 return apc->bpf_prog;
141}
142
143static struct bpf_prog *mana_chn_xdp_get(struct mana_port_context *apc)
144{
145 return rtnl_dereference(apc->rxqs[0]->bpf_prog);
146}
147
148/* Set xdp program on channels */
149void mana_chn_setxdp(struct mana_port_context *apc, struct bpf_prog *prog)
150{
151 struct bpf_prog *old_prog = mana_chn_xdp_get(apc);
152 unsigned int num_queues = apc->num_queues;
153 int i;
154
155 ASSERT_RTNL();
156
157 if (old_prog == prog)
158 return;
159
160 if (prog)
161 bpf_prog_add(prog, i: num_queues);
162
163 for (i = 0; i < num_queues; i++)
164 rcu_assign_pointer(apc->rxqs[i]->bpf_prog, prog);
165
166 if (old_prog)
167 for (i = 0; i < num_queues; i++)
168 bpf_prog_put(prog: old_prog);
169}
170
171static int mana_xdp_set(struct net_device *ndev, struct bpf_prog *prog,
172 struct netlink_ext_ack *extack)
173{
174 struct mana_port_context *apc = netdev_priv(dev: ndev);
175 struct bpf_prog *old_prog;
176 struct gdma_context *gc;
177
178 gc = apc->ac->gdma_dev->gdma_context;
179
180 old_prog = mana_xdp_get(apc);
181
182 if (!old_prog && !prog)
183 return 0;
184
185 if (prog && ndev->mtu > MANA_XDP_MTU_MAX) {
186 netdev_err(dev: ndev, format: "XDP: mtu:%u too large, mtu_max:%lu\n",
187 ndev->mtu, MANA_XDP_MTU_MAX);
188 NL_SET_ERR_MSG_MOD(extack, "XDP: mtu too large");
189
190 return -EOPNOTSUPP;
191 }
192
193 /* One refcnt of the prog is hold by the caller already, so
194 * don't increase refcnt for this one.
195 */
196 apc->bpf_prog = prog;
197
198 if (old_prog)
199 bpf_prog_put(prog: old_prog);
200
201 if (apc->port_is_up)
202 mana_chn_setxdp(apc, prog);
203
204 if (prog)
205 ndev->max_mtu = MANA_XDP_MTU_MAX;
206 else
207 ndev->max_mtu = gc->adapter_mtu - ETH_HLEN;
208
209 return 0;
210}
211
212int mana_bpf(struct net_device *ndev, struct netdev_bpf *bpf)
213{
214 struct netlink_ext_ack *extack = bpf->extack;
215 int ret;
216
217 switch (bpf->command) {
218 case XDP_SETUP_PROG:
219 return mana_xdp_set(ndev, prog: bpf->prog, extack);
220
221 default:
222 return -EOPNOTSUPP;
223 }
224
225 return ret;
226}
227

source code of linux/drivers/net/ethernet/microsoft/mana/mana_bpf.c