1 | // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause |
2 | /* Copyright (c) 2021, Microsoft Corporation. */ |
3 | |
4 | #include <linux/inetdevice.h> |
5 | #include <linux/etherdevice.h> |
6 | #include <linux/mm.h> |
7 | #include <linux/bpf.h> |
8 | #include <linux/bpf_trace.h> |
9 | #include <net/xdp.h> |
10 | |
11 | #include <net/mana/mana.h> |
12 | |
13 | void mana_xdp_tx(struct sk_buff *skb, struct net_device *ndev) |
14 | { |
15 | u16 txq_idx = skb_get_queue_mapping(skb); |
16 | struct netdev_queue *ndevtxq; |
17 | int rc; |
18 | |
19 | __skb_push(skb, ETH_HLEN); |
20 | |
21 | ndevtxq = netdev_get_tx_queue(dev: ndev, index: txq_idx); |
22 | __netif_tx_lock(txq: ndevtxq, smp_processor_id()); |
23 | |
24 | rc = mana_start_xmit(skb, ndev); |
25 | |
26 | __netif_tx_unlock(txq: ndevtxq); |
27 | |
28 | if (dev_xmit_complete(rc)) |
29 | return; |
30 | |
31 | dev_kfree_skb_any(skb); |
32 | ndev->stats.tx_dropped++; |
33 | } |
34 | |
35 | static int mana_xdp_xmit_fm(struct net_device *ndev, struct xdp_frame *frame, |
36 | u16 q_idx) |
37 | { |
38 | struct sk_buff *skb; |
39 | |
40 | skb = xdp_build_skb_from_frame(xdpf: frame, dev: ndev); |
41 | if (unlikely(!skb)) |
42 | return -ENOMEM; |
43 | |
44 | skb_set_queue_mapping(skb, queue_mapping: q_idx); |
45 | |
46 | mana_xdp_tx(skb, ndev); |
47 | |
48 | return 0; |
49 | } |
50 | |
51 | int mana_xdp_xmit(struct net_device *ndev, int n, struct xdp_frame **frames, |
52 | u32 flags) |
53 | { |
54 | struct mana_port_context *apc = netdev_priv(dev: ndev); |
55 | struct mana_stats_tx *tx_stats; |
56 | int i, count = 0; |
57 | u16 q_idx; |
58 | |
59 | if (unlikely(!apc->port_is_up)) |
60 | return 0; |
61 | |
62 | q_idx = smp_processor_id() % ndev->real_num_tx_queues; |
63 | |
64 | for (i = 0; i < n; i++) { |
65 | if (mana_xdp_xmit_fm(ndev, frame: frames[i], q_idx)) |
66 | break; |
67 | |
68 | count++; |
69 | } |
70 | |
71 | tx_stats = &apc->tx_qp[q_idx].txq.stats; |
72 | |
73 | u64_stats_update_begin(syncp: &tx_stats->syncp); |
74 | tx_stats->xdp_xmit += count; |
75 | u64_stats_update_end(syncp: &tx_stats->syncp); |
76 | |
77 | return count; |
78 | } |
79 | |
80 | u32 mana_run_xdp(struct net_device *ndev, struct mana_rxq *rxq, |
81 | struct xdp_buff *xdp, void *buf_va, uint pkt_len) |
82 | { |
83 | struct mana_stats_rx *rx_stats; |
84 | struct bpf_prog *prog; |
85 | u32 act = XDP_PASS; |
86 | |
87 | rcu_read_lock(); |
88 | prog = rcu_dereference(rxq->bpf_prog); |
89 | |
90 | if (!prog) |
91 | goto out; |
92 | |
93 | xdp_init_buff(xdp, PAGE_SIZE, rxq: &rxq->xdp_rxq); |
94 | xdp_prepare_buff(xdp, hard_start: buf_va, XDP_PACKET_HEADROOM, data_len: pkt_len, meta_valid: false); |
95 | |
96 | act = bpf_prog_run_xdp(prog, xdp); |
97 | |
98 | rx_stats = &rxq->stats; |
99 | |
100 | switch (act) { |
101 | case XDP_PASS: |
102 | case XDP_TX: |
103 | case XDP_DROP: |
104 | break; |
105 | |
106 | case XDP_REDIRECT: |
107 | rxq->xdp_rc = xdp_do_redirect(dev: ndev, xdp, prog); |
108 | if (!rxq->xdp_rc) { |
109 | rxq->xdp_flush = true; |
110 | |
111 | u64_stats_update_begin(syncp: &rx_stats->syncp); |
112 | rx_stats->packets++; |
113 | rx_stats->bytes += pkt_len; |
114 | rx_stats->xdp_redirect++; |
115 | u64_stats_update_end(syncp: &rx_stats->syncp); |
116 | |
117 | break; |
118 | } |
119 | |
120 | fallthrough; |
121 | |
122 | case XDP_ABORTED: |
123 | trace_xdp_exception(dev: ndev, xdp: prog, act); |
124 | break; |
125 | |
126 | default: |
127 | bpf_warn_invalid_xdp_action(dev: ndev, prog, act); |
128 | } |
129 | |
130 | out: |
131 | rcu_read_unlock(); |
132 | |
133 | return act; |
134 | } |
135 | |
136 | struct bpf_prog *mana_xdp_get(struct mana_port_context *apc) |
137 | { |
138 | ASSERT_RTNL(); |
139 | |
140 | return apc->bpf_prog; |
141 | } |
142 | |
143 | static struct bpf_prog *mana_chn_xdp_get(struct mana_port_context *apc) |
144 | { |
145 | return rtnl_dereference(apc->rxqs[0]->bpf_prog); |
146 | } |
147 | |
148 | /* Set xdp program on channels */ |
149 | void mana_chn_setxdp(struct mana_port_context *apc, struct bpf_prog *prog) |
150 | { |
151 | struct bpf_prog *old_prog = mana_chn_xdp_get(apc); |
152 | unsigned int num_queues = apc->num_queues; |
153 | int i; |
154 | |
155 | ASSERT_RTNL(); |
156 | |
157 | if (old_prog == prog) |
158 | return; |
159 | |
160 | if (prog) |
161 | bpf_prog_add(prog, i: num_queues); |
162 | |
163 | for (i = 0; i < num_queues; i++) |
164 | rcu_assign_pointer(apc->rxqs[i]->bpf_prog, prog); |
165 | |
166 | if (old_prog) |
167 | for (i = 0; i < num_queues; i++) |
168 | bpf_prog_put(prog: old_prog); |
169 | } |
170 | |
171 | static int mana_xdp_set(struct net_device *ndev, struct bpf_prog *prog, |
172 | struct netlink_ext_ack *extack) |
173 | { |
174 | struct mana_port_context *apc = netdev_priv(dev: ndev); |
175 | struct bpf_prog *old_prog; |
176 | struct gdma_context *gc; |
177 | |
178 | gc = apc->ac->gdma_dev->gdma_context; |
179 | |
180 | old_prog = mana_xdp_get(apc); |
181 | |
182 | if (!old_prog && !prog) |
183 | return 0; |
184 | |
185 | if (prog && ndev->mtu > MANA_XDP_MTU_MAX) { |
186 | netdev_err(dev: ndev, format: "XDP: mtu:%u too large, mtu_max:%lu\n" , |
187 | ndev->mtu, MANA_XDP_MTU_MAX); |
188 | NL_SET_ERR_MSG_MOD(extack, "XDP: mtu too large" ); |
189 | |
190 | return -EOPNOTSUPP; |
191 | } |
192 | |
193 | /* One refcnt of the prog is hold by the caller already, so |
194 | * don't increase refcnt for this one. |
195 | */ |
196 | apc->bpf_prog = prog; |
197 | |
198 | if (old_prog) |
199 | bpf_prog_put(prog: old_prog); |
200 | |
201 | if (apc->port_is_up) |
202 | mana_chn_setxdp(apc, prog); |
203 | |
204 | if (prog) |
205 | ndev->max_mtu = MANA_XDP_MTU_MAX; |
206 | else |
207 | ndev->max_mtu = gc->adapter_mtu - ETH_HLEN; |
208 | |
209 | return 0; |
210 | } |
211 | |
212 | int mana_bpf(struct net_device *ndev, struct netdev_bpf *bpf) |
213 | { |
214 | struct netlink_ext_ack *extack = bpf->extack; |
215 | int ret; |
216 | |
217 | switch (bpf->command) { |
218 | case XDP_SETUP_PROG: |
219 | return mana_xdp_set(ndev, prog: bpf->prog, extack); |
220 | |
221 | default: |
222 | return -EOPNOTSUPP; |
223 | } |
224 | |
225 | return ret; |
226 | } |
227 | |