1 | /* Broadcom NetXtreme-C/E network driver. |
2 | * |
3 | * Copyright (c) 2016-2017 Broadcom Limited |
4 | * |
5 | * This program is free software; you can redistribute it and/or modify |
6 | * it under the terms of the GNU General Public License as published by |
7 | * the Free Software Foundation. |
8 | */ |
9 | #include <linux/kernel.h> |
10 | #include <linux/errno.h> |
11 | #include <linux/pci.h> |
12 | #include <linux/netdevice.h> |
13 | #include <linux/etherdevice.h> |
14 | #include <linux/if_vlan.h> |
15 | #include <linux/bpf.h> |
16 | #include <linux/bpf_trace.h> |
17 | #include <linux/filter.h> |
18 | #include <net/netdev_lock.h> |
19 | #include <net/page_pool/helpers.h> |
20 | #include "bnxt_hsi.h" |
21 | #include "bnxt.h" |
22 | #include "bnxt_xdp.h" |
23 | |
24 | DEFINE_STATIC_KEY_FALSE(bnxt_xdp_locking_key); |
25 | |
26 | struct bnxt_sw_tx_bd *bnxt_xmit_bd(struct bnxt *bp, |
27 | struct bnxt_tx_ring_info *txr, |
28 | dma_addr_t mapping, u32 len, |
29 | struct xdp_buff *xdp) |
30 | { |
31 | struct skb_shared_info *sinfo; |
32 | struct bnxt_sw_tx_bd *tx_buf; |
33 | struct tx_bd *txbd; |
34 | int num_frags = 0; |
35 | u32 flags; |
36 | u16 prod; |
37 | int i; |
38 | |
39 | if (xdp && xdp_buff_has_frags(xdp)) { |
40 | sinfo = xdp_get_shared_info_from_buff(xdp); |
41 | num_frags = sinfo->nr_frags; |
42 | } |
43 | |
44 | /* fill up the first buffer */ |
45 | prod = txr->tx_prod; |
46 | tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)]; |
47 | tx_buf->nr_frags = num_frags; |
48 | if (xdp) |
49 | tx_buf->page = virt_to_head_page(x: xdp->data); |
50 | |
51 | txbd = &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)]; |
52 | flags = (len << TX_BD_LEN_SHIFT) | TX_BD_CNT(num_frags + 1) | |
53 | bnxt_lhint_arr[len >> 9]; |
54 | txbd->tx_bd_len_flags_type = cpu_to_le32(flags); |
55 | txbd->tx_bd_opaque = SET_TX_OPAQUE(bp, txr, prod, 1 + num_frags); |
56 | txbd->tx_bd_haddr = cpu_to_le64(mapping); |
57 | |
58 | /* now let us fill up the frags into the next buffers */ |
59 | for (i = 0; i < num_frags ; i++) { |
60 | skb_frag_t *frag = &sinfo->frags[i]; |
61 | struct bnxt_sw_tx_bd *frag_tx_buf; |
62 | dma_addr_t frag_mapping; |
63 | int frag_len; |
64 | |
65 | prod = NEXT_TX(prod); |
66 | WRITE_ONCE(txr->tx_prod, prod); |
67 | |
68 | /* first fill up the first buffer */ |
69 | frag_tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)]; |
70 | frag_tx_buf->page = skb_frag_page(frag); |
71 | |
72 | txbd = &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)]; |
73 | |
74 | frag_len = skb_frag_size(frag); |
75 | flags = frag_len << TX_BD_LEN_SHIFT; |
76 | txbd->tx_bd_len_flags_type = cpu_to_le32(flags); |
77 | frag_mapping = page_pool_get_dma_addr(page: skb_frag_page(frag)) + |
78 | skb_frag_off(frag); |
79 | txbd->tx_bd_haddr = cpu_to_le64(frag_mapping); |
80 | |
81 | len = frag_len; |
82 | } |
83 | |
84 | flags &= ~TX_BD_LEN; |
85 | txbd->tx_bd_len_flags_type = cpu_to_le32(((len) << TX_BD_LEN_SHIFT) | flags | |
86 | TX_BD_FLAGS_PACKET_END); |
87 | /* Sync TX BD */ |
88 | wmb(); |
89 | prod = NEXT_TX(prod); |
90 | WRITE_ONCE(txr->tx_prod, prod); |
91 | |
92 | return tx_buf; |
93 | } |
94 | |
95 | static void __bnxt_xmit_xdp(struct bnxt *bp, struct bnxt_tx_ring_info *txr, |
96 | dma_addr_t mapping, u32 len, u16 rx_prod, |
97 | struct xdp_buff *xdp) |
98 | { |
99 | struct bnxt_sw_tx_bd *tx_buf; |
100 | |
101 | tx_buf = bnxt_xmit_bd(bp, txr, mapping, len, xdp); |
102 | tx_buf->rx_prod = rx_prod; |
103 | tx_buf->action = XDP_TX; |
104 | |
105 | } |
106 | |
107 | static void __bnxt_xmit_xdp_redirect(struct bnxt *bp, |
108 | struct bnxt_tx_ring_info *txr, |
109 | dma_addr_t mapping, u32 len, |
110 | struct xdp_frame *xdpf) |
111 | { |
112 | struct bnxt_sw_tx_bd *tx_buf; |
113 | |
114 | tx_buf = bnxt_xmit_bd(bp, txr, mapping, len, NULL); |
115 | tx_buf->action = XDP_REDIRECT; |
116 | tx_buf->xdpf = xdpf; |
117 | dma_unmap_addr_set(tx_buf, mapping, mapping); |
118 | dma_unmap_len_set(tx_buf, len, 0); |
119 | } |
120 | |
121 | void bnxt_tx_int_xdp(struct bnxt *bp, struct bnxt_napi *bnapi, int budget) |
122 | { |
123 | struct bnxt_tx_ring_info *txr = bnapi->tx_ring[0]; |
124 | struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; |
125 | u16 tx_hw_cons = txr->tx_hw_cons; |
126 | bool rx_doorbell_needed = false; |
127 | struct bnxt_sw_tx_bd *tx_buf; |
128 | u16 tx_cons = txr->tx_cons; |
129 | u16 last_tx_cons = tx_cons; |
130 | int j, frags; |
131 | |
132 | if (!budget) |
133 | return; |
134 | |
135 | while (RING_TX(bp, tx_cons) != tx_hw_cons) { |
136 | tx_buf = &txr->tx_buf_ring[RING_TX(bp, tx_cons)]; |
137 | |
138 | if (tx_buf->action == XDP_REDIRECT) { |
139 | struct pci_dev *pdev = bp->pdev; |
140 | |
141 | dma_unmap_single(&pdev->dev, |
142 | dma_unmap_addr(tx_buf, mapping), |
143 | dma_unmap_len(tx_buf, len), |
144 | DMA_TO_DEVICE); |
145 | xdp_return_frame(xdpf: tx_buf->xdpf); |
146 | tx_buf->action = 0; |
147 | tx_buf->xdpf = NULL; |
148 | } else if (tx_buf->action == XDP_TX) { |
149 | tx_buf->action = 0; |
150 | rx_doorbell_needed = true; |
151 | last_tx_cons = tx_cons; |
152 | |
153 | frags = tx_buf->nr_frags; |
154 | for (j = 0; j < frags; j++) { |
155 | tx_cons = NEXT_TX(tx_cons); |
156 | tx_buf = &txr->tx_buf_ring[RING_TX(bp, tx_cons)]; |
157 | page_pool_recycle_direct(pool: rxr->page_pool, page: tx_buf->page); |
158 | } |
159 | } else { |
160 | bnxt_sched_reset_txr(bp, txr, curr: tx_cons); |
161 | return; |
162 | } |
163 | tx_cons = NEXT_TX(tx_cons); |
164 | } |
165 | |
166 | bnapi->events &= ~BNXT_TX_CMP_EVENT; |
167 | WRITE_ONCE(txr->tx_cons, tx_cons); |
168 | if (rx_doorbell_needed) { |
169 | tx_buf = &txr->tx_buf_ring[RING_TX(bp, last_tx_cons)]; |
170 | bnxt_db_write(bp, db: &rxr->rx_db, idx: tx_buf->rx_prod); |
171 | |
172 | } |
173 | } |
174 | |
175 | bool bnxt_xdp_attached(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) |
176 | { |
177 | struct bpf_prog *xdp_prog = READ_ONCE(rxr->xdp_prog); |
178 | |
179 | return !!xdp_prog; |
180 | } |
181 | |
182 | void bnxt_xdp_buff_init(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, |
183 | u16 cons, u8 *data_ptr, unsigned int len, |
184 | struct xdp_buff *xdp) |
185 | { |
186 | u32 buflen = BNXT_RX_PAGE_SIZE; |
187 | struct bnxt_sw_rx_bd *rx_buf; |
188 | struct pci_dev *pdev; |
189 | dma_addr_t mapping; |
190 | u32 offset; |
191 | |
192 | pdev = bp->pdev; |
193 | rx_buf = &rxr->rx_buf_ring[cons]; |
194 | offset = bp->rx_offset; |
195 | |
196 | mapping = rx_buf->mapping - bp->rx_dma_offset; |
197 | dma_sync_single_for_cpu(dev: &pdev->dev, addr: mapping + offset, size: len, dir: bp->rx_dir); |
198 | |
199 | xdp_init_buff(xdp, frame_sz: buflen, rxq: &rxr->xdp_rxq); |
200 | xdp_prepare_buff(xdp, hard_start: data_ptr - offset, headroom: offset, data_len: len, meta_valid: true); |
201 | } |
202 | |
203 | void bnxt_xdp_buff_frags_free(struct bnxt_rx_ring_info *rxr, |
204 | struct xdp_buff *xdp) |
205 | { |
206 | struct skb_shared_info *shinfo; |
207 | int i; |
208 | |
209 | if (!xdp || !xdp_buff_has_frags(xdp)) |
210 | return; |
211 | shinfo = xdp_get_shared_info_from_buff(xdp); |
212 | for (i = 0; i < shinfo->nr_frags; i++) { |
213 | struct page *page = skb_frag_page(frag: &shinfo->frags[i]); |
214 | |
215 | page_pool_recycle_direct(pool: rxr->page_pool, page); |
216 | } |
217 | shinfo->nr_frags = 0; |
218 | } |
219 | |
220 | /* returns the following: |
221 | * true - packet consumed by XDP and new buffer is allocated. |
222 | * false - packet should be passed to the stack. |
223 | */ |
224 | bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons, |
225 | struct xdp_buff *xdp, struct page *page, u8 **data_ptr, |
226 | unsigned int *len, u8 *event) |
227 | { |
228 | struct bpf_prog *xdp_prog = READ_ONCE(rxr->xdp_prog); |
229 | struct bnxt_tx_ring_info *txr; |
230 | struct bnxt_sw_rx_bd *rx_buf; |
231 | struct pci_dev *pdev; |
232 | dma_addr_t mapping; |
233 | u32 tx_needed = 1; |
234 | void *orig_data; |
235 | u32 tx_avail; |
236 | u32 offset; |
237 | u32 act; |
238 | |
239 | if (!xdp_prog) |
240 | return false; |
241 | |
242 | pdev = bp->pdev; |
243 | offset = bp->rx_offset; |
244 | |
245 | txr = rxr->bnapi->tx_ring[0]; |
246 | /* BNXT_RX_PAGE_MODE(bp) when XDP enabled */ |
247 | orig_data = xdp->data; |
248 | |
249 | act = bpf_prog_run_xdp(prog: xdp_prog, xdp); |
250 | |
251 | tx_avail = bnxt_tx_avail(bp, txr); |
252 | /* If the tx ring is not full, we must not update the rx producer yet |
253 | * because we may still be transmitting on some BDs. |
254 | */ |
255 | if (tx_avail != bp->tx_ring_size) |
256 | *event &= ~BNXT_RX_EVENT; |
257 | |
258 | *len = xdp->data_end - xdp->data; |
259 | if (orig_data != xdp->data) { |
260 | offset = xdp->data - xdp->data_hard_start; |
261 | *data_ptr = xdp->data_hard_start + offset; |
262 | } |
263 | |
264 | switch (act) { |
265 | case XDP_PASS: |
266 | return false; |
267 | |
268 | case XDP_TX: |
269 | rx_buf = &rxr->rx_buf_ring[cons]; |
270 | mapping = rx_buf->mapping - bp->rx_dma_offset; |
271 | *event &= BNXT_TX_CMP_EVENT; |
272 | |
273 | if (unlikely(xdp_buff_has_frags(xdp))) { |
274 | struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); |
275 | |
276 | tx_needed += sinfo->nr_frags; |
277 | *event = BNXT_AGG_EVENT; |
278 | } |
279 | |
280 | if (tx_avail < tx_needed) { |
281 | trace_xdp_exception(dev: bp->dev, xdp: xdp_prog, act); |
282 | bnxt_xdp_buff_frags_free(rxr, xdp); |
283 | bnxt_reuse_rx_data(rxr, cons, data: page); |
284 | return true; |
285 | } |
286 | |
287 | dma_sync_single_for_device(dev: &pdev->dev, addr: mapping + offset, size: *len, |
288 | dir: bp->rx_dir); |
289 | |
290 | *event |= BNXT_TX_EVENT; |
291 | __bnxt_xmit_xdp(bp, txr, mapping: mapping + offset, len: *len, |
292 | NEXT_RX(rxr->rx_prod), xdp); |
293 | bnxt_reuse_rx_data(rxr, cons, data: page); |
294 | return true; |
295 | case XDP_REDIRECT: |
296 | /* if we are calling this here then we know that the |
297 | * redirect is coming from a frame received by the |
298 | * bnxt_en driver. |
299 | */ |
300 | |
301 | /* if we are unable to allocate a new buffer, abort and reuse */ |
302 | if (bnxt_alloc_rx_data(bp, rxr, prod: rxr->rx_prod, GFP_ATOMIC)) { |
303 | trace_xdp_exception(dev: bp->dev, xdp: xdp_prog, act); |
304 | bnxt_xdp_buff_frags_free(rxr, xdp); |
305 | bnxt_reuse_rx_data(rxr, cons, data: page); |
306 | return true; |
307 | } |
308 | |
309 | if (xdp_do_redirect(dev: bp->dev, xdp, prog: xdp_prog)) { |
310 | trace_xdp_exception(dev: bp->dev, xdp: xdp_prog, act); |
311 | page_pool_recycle_direct(pool: rxr->page_pool, page); |
312 | return true; |
313 | } |
314 | |
315 | *event |= BNXT_REDIRECT_EVENT; |
316 | break; |
317 | default: |
318 | bpf_warn_invalid_xdp_action(dev: bp->dev, prog: xdp_prog, act); |
319 | fallthrough; |
320 | case XDP_ABORTED: |
321 | trace_xdp_exception(dev: bp->dev, xdp: xdp_prog, act); |
322 | fallthrough; |
323 | case XDP_DROP: |
324 | bnxt_xdp_buff_frags_free(rxr, xdp); |
325 | bnxt_reuse_rx_data(rxr, cons, data: page); |
326 | break; |
327 | } |
328 | return true; |
329 | } |
330 | |
331 | int bnxt_xdp_xmit(struct net_device *dev, int num_frames, |
332 | struct xdp_frame **frames, u32 flags) |
333 | { |
334 | struct bnxt *bp = netdev_priv(dev); |
335 | struct bpf_prog *xdp_prog = READ_ONCE(bp->xdp_prog); |
336 | struct pci_dev *pdev = bp->pdev; |
337 | struct bnxt_tx_ring_info *txr; |
338 | dma_addr_t mapping; |
339 | int nxmit = 0; |
340 | int ring; |
341 | int i; |
342 | |
343 | if (!test_bit(BNXT_STATE_OPEN, &bp->state) || |
344 | !bp->tx_nr_rings_xdp || |
345 | !xdp_prog) |
346 | return -EINVAL; |
347 | |
348 | ring = smp_processor_id() % bp->tx_nr_rings_xdp; |
349 | txr = &bp->tx_ring[ring]; |
350 | |
351 | if (READ_ONCE(txr->dev_state) == BNXT_DEV_STATE_CLOSING) |
352 | return -EINVAL; |
353 | |
354 | if (static_branch_unlikely(&bnxt_xdp_locking_key)) |
355 | spin_lock(lock: &txr->xdp_tx_lock); |
356 | |
357 | for (i = 0; i < num_frames; i++) { |
358 | struct xdp_frame *xdp = frames[i]; |
359 | |
360 | if (!bnxt_tx_avail(bp, txr)) |
361 | break; |
362 | |
363 | mapping = dma_map_single(&pdev->dev, xdp->data, xdp->len, |
364 | DMA_TO_DEVICE); |
365 | |
366 | if (dma_mapping_error(dev: &pdev->dev, dma_addr: mapping)) |
367 | break; |
368 | |
369 | __bnxt_xmit_xdp_redirect(bp, txr, mapping, len: xdp->len, xdpf: xdp); |
370 | nxmit++; |
371 | } |
372 | |
373 | if (flags & XDP_XMIT_FLUSH) { |
374 | /* Sync BD data before updating doorbell */ |
375 | wmb(); |
376 | bnxt_db_write(bp, db: &txr->tx_db, idx: txr->tx_prod); |
377 | } |
378 | |
379 | if (static_branch_unlikely(&bnxt_xdp_locking_key)) |
380 | spin_unlock(lock: &txr->xdp_tx_lock); |
381 | |
382 | return nxmit; |
383 | } |
384 | |
385 | static int bnxt_xdp_set(struct bnxt *bp, struct bpf_prog *prog) |
386 | { |
387 | struct net_device *dev = bp->dev; |
388 | int tx_xdp = 0, tx_cp, rc, tc; |
389 | struct bpf_prog *old; |
390 | |
391 | netdev_assert_locked(dev); |
392 | |
393 | if (prog && !prog->aux->xdp_has_frags && |
394 | bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU) { |
395 | netdev_warn(dev, format: "MTU %d larger than %d without XDP frag support.\n" , |
396 | bp->dev->mtu, BNXT_MAX_PAGE_MODE_MTU); |
397 | return -EOPNOTSUPP; |
398 | } |
399 | if (prog && bp->flags & BNXT_FLAG_HDS) { |
400 | netdev_warn(dev, format: "XDP is disallowed when HDS is enabled.\n" ); |
401 | return -EOPNOTSUPP; |
402 | } |
403 | if (!(bp->flags & BNXT_FLAG_SHARED_RINGS)) { |
404 | netdev_warn(dev, format: "ethtool rx/tx channels must be combined to support XDP.\n" ); |
405 | return -EOPNOTSUPP; |
406 | } |
407 | if (prog) |
408 | tx_xdp = bp->rx_nr_rings; |
409 | |
410 | tc = bp->num_tc; |
411 | if (!tc) |
412 | tc = 1; |
413 | rc = bnxt_check_rings(bp, tx: bp->tx_nr_rings_per_tc, rx: bp->rx_nr_rings, |
414 | sh: true, tcs: tc, tx_xdp); |
415 | if (rc) { |
416 | netdev_warn(dev, format: "Unable to reserve enough TX rings to support XDP.\n" ); |
417 | return rc; |
418 | } |
419 | if (netif_running(dev)) |
420 | bnxt_close_nic(bp, true, false); |
421 | |
422 | old = xchg(&bp->xdp_prog, prog); |
423 | if (old) |
424 | bpf_prog_put(prog: old); |
425 | |
426 | if (prog) { |
427 | bnxt_set_rx_skb_mode(bp, page_mode: true); |
428 | xdp_features_set_redirect_target_locked(dev, support_sg: true); |
429 | } else { |
430 | xdp_features_clear_redirect_target_locked(dev); |
431 | bnxt_set_rx_skb_mode(bp, page_mode: false); |
432 | } |
433 | bp->tx_nr_rings_xdp = tx_xdp; |
434 | bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc + tx_xdp; |
435 | tx_cp = bnxt_num_tx_to_cp(bp, tx: bp->tx_nr_rings); |
436 | bp->cp_nr_rings = max_t(int, tx_cp, bp->rx_nr_rings); |
437 | bnxt_set_tpa_flags(bp); |
438 | bnxt_set_ring_params(bp); |
439 | |
440 | if (netif_running(dev)) |
441 | return bnxt_open_nic(bp, true, false); |
442 | |
443 | return 0; |
444 | } |
445 | |
446 | int bnxt_xdp(struct net_device *dev, struct netdev_bpf *xdp) |
447 | { |
448 | struct bnxt *bp = netdev_priv(dev); |
449 | int rc; |
450 | |
451 | switch (xdp->command) { |
452 | case XDP_SETUP_PROG: |
453 | rc = bnxt_xdp_set(bp, prog: xdp->prog); |
454 | break; |
455 | default: |
456 | rc = -EINVAL; |
457 | break; |
458 | } |
459 | return rc; |
460 | } |
461 | |
462 | struct sk_buff * |
463 | bnxt_xdp_build_skb(struct bnxt *bp, struct sk_buff *skb, u8 num_frags, |
464 | struct page_pool *pool, struct xdp_buff *xdp) |
465 | { |
466 | struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); |
467 | |
468 | if (!skb) |
469 | return NULL; |
470 | |
471 | xdp_update_skb_shared_info(skb, nr_frags: num_frags, |
472 | size: sinfo->xdp_frags_size, |
473 | BNXT_RX_PAGE_SIZE * num_frags, |
474 | pfmemalloc: xdp_buff_is_frag_pfmemalloc(xdp)); |
475 | return skb; |
476 | } |
477 | |