1	// SPDX-License-Identifier: GPL-2.0
2	/* XDP sockets
3	*
4	* AF_XDP sockets allows a channel between XDP programs and userspace
5	* applications.
6	* Copyright(c) 2018 Intel Corporation.
7	*
8	* Author(s): Björn Töpel <bjorn.topel@intel.com>
9	* Magnus Karlsson <magnus.karlsson@intel.com>
10	*/
11
12	#define pr_fmt(fmt) "AF_XDP: %s: " fmt, __func__
13
14	#include <linux/if_xdp.h>
15	#include <linux/init.h>
16	#include <linux/sched/mm.h>
17	#include <linux/sched/signal.h>
18	#include <linux/sched/task.h>
19	#include <linux/socket.h>
20	#include <linux/file.h>
21	#include <linux/uaccess.h>
22	#include <linux/net.h>
23	#include <linux/netdevice.h>
24	#include <linux/rculist.h>
25	#include <linux/vmalloc.h>
26	#include <net/xdp_sock_drv.h>
27	#include <net/busy_poll.h>
28	#include <net/netdev_lock.h>
29	#include <net/netdev_rx_queue.h>
30	#include <net/xdp.h>
31
32	#include "xsk_queue.h"
33	#include "xdp_umem.h"
34	#include "xsk.h"
35
36	#define TX_BATCH_SIZE 32
37	#define MAX_PER_SOCKET_BUDGET 32
38
39	struct xsk_addrs {
40	u32 num_descs;
41	u64 addrs[MAX_SKB_FRAGS + 1];
42	};
43
44	static struct kmem_cache *xsk_tx_generic_cache;
45
46	void xsk_set_rx_need_wakeup(struct xsk_buff_pool *pool)
47	{
48	if (pool->cached_need_wakeup & XDP_WAKEUP_RX)
49	return;
50
51	pool->fq->ring->flags |= XDP_RING_NEED_WAKEUP;
52	pool->cached_need_wakeup |= XDP_WAKEUP_RX;
53	}
54	EXPORT_SYMBOL(xsk_set_rx_need_wakeup);
55
56	void xsk_set_tx_need_wakeup(struct xsk_buff_pool *pool)
57	{
58	struct xdp_sock *xs;
59
60	if (pool->cached_need_wakeup & XDP_WAKEUP_TX)
61	return;
62
63	rcu_read_lock();
64	list_for_each_entry_rcu(xs, &pool->xsk_tx_list, tx_list) {
65	xs->tx->ring->flags |= XDP_RING_NEED_WAKEUP;
66	}
67	rcu_read_unlock();
68
69	pool->cached_need_wakeup |= XDP_WAKEUP_TX;
70	}
71	EXPORT_SYMBOL(xsk_set_tx_need_wakeup);
72
73	void xsk_clear_rx_need_wakeup(struct xsk_buff_pool *pool)
74	{
75	if (!(pool->cached_need_wakeup & XDP_WAKEUP_RX))
76	return;
77
78	pool->fq->ring->flags &= ~XDP_RING_NEED_WAKEUP;
79	pool->cached_need_wakeup &= ~XDP_WAKEUP_RX;
80	}
81	EXPORT_SYMBOL(xsk_clear_rx_need_wakeup);
82
83	void xsk_clear_tx_need_wakeup(struct xsk_buff_pool *pool)
84	{
85	struct xdp_sock *xs;
86
87	if (!(pool->cached_need_wakeup & XDP_WAKEUP_TX))
88	return;
89
90	rcu_read_lock();
91	list_for_each_entry_rcu(xs, &pool->xsk_tx_list, tx_list) {
92	xs->tx->ring->flags &= ~XDP_RING_NEED_WAKEUP;
93	}
94	rcu_read_unlock();
95
96	pool->cached_need_wakeup &= ~XDP_WAKEUP_TX;
97	}
98	EXPORT_SYMBOL(xsk_clear_tx_need_wakeup);
99
100	bool xsk_uses_need_wakeup(struct xsk_buff_pool *pool)
101	{
102	return pool->uses_need_wakeup;
103	}
104	EXPORT_SYMBOL(xsk_uses_need_wakeup);
105
106	struct xsk_buff_pool *xsk_get_pool_from_qid(struct net_device *dev,
107	u16 queue_id)
108	{
109	if (queue_id < dev->real_num_rx_queues)
110	return dev->_rx[queue_id].pool;
111	if (queue_id < dev->real_num_tx_queues)
112	return dev->_tx[queue_id].pool;
113
114	return NULL;
115	}
116	EXPORT_SYMBOL(xsk_get_pool_from_qid);
117
118	void xsk_clear_pool_at_qid(struct net_device *dev, u16 queue_id)
119	{
120	if (queue_id < dev->num_rx_queues)
121	dev->_rx[queue_id].pool = NULL;
122	if (queue_id < dev->num_tx_queues)
123	dev->_tx[queue_id].pool = NULL;
124	}
125
126	/* The buffer pool is stored both in the _rx struct and the _tx struct as we do
127	* not know if the device has more tx queues than rx, or the opposite.
128	* This might also change during run time.
129	*/
130	int xsk_reg_pool_at_qid(struct net_device *dev, struct xsk_buff_pool *pool,
131	u16 queue_id)
132	{
133	if (queue_id >= max_t(unsigned int,
134	dev->real_num_rx_queues,
135	dev->real_num_tx_queues))
136	return -EINVAL;
137
138	if (queue_id < dev->real_num_rx_queues)
139	dev->_rx[queue_id].pool = pool;
140	if (queue_id < dev->real_num_tx_queues)
141	dev->_tx[queue_id].pool = pool;
142
143	return 0;
144	}
145
146	static int __xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff_xsk *xskb, u32 len,
147	u32 flags)
148	{
149	u64 addr;
150	int err;
151
152	addr = xp_get_handle(xskb, pool: xskb->pool);
153	err = xskq_prod_reserve_desc(q: xs->rx, addr, len, flags);
154	if (err) {
155	xs->rx_queue_full++;
156	return err;
157	}
158
159	xp_release(xskb);
160	return 0;
161	}
162
163	static int xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
164	{
165	struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
166	u32 frags = xdp_buff_has_frags(xdp);
167	struct xdp_buff_xsk *pos, *tmp;
168	struct list_head *xskb_list;
169	u32 contd = 0;
170	int err;
171
172	if (frags)
173	contd = XDP_PKT_CONTD;
174
175	err = __xsk_rcv_zc(xs, xskb, len, flags: contd);
176	if (err)
177	goto err;
178	if (likely(!frags))
179	return 0;
180
181	xskb_list = &xskb->pool->xskb_list;
182	list_for_each_entry_safe(pos, tmp, xskb_list, list_node) {
183	if (list_is_singular(head: xskb_list))
184	contd = 0;
185	len = pos->xdp.data_end - pos->xdp.data;
186	err = __xsk_rcv_zc(xs, xskb: pos, len, flags: contd);
187	if (err)
188	goto err;
189	list_del(entry: &pos->list_node);
190	}
191
192	return 0;
193	err:
194	xsk_buff_free(xdp);
195	return err;
196	}
197
198	static void *xsk_copy_xdp_start(struct xdp_buff *from)
199	{
200	if (unlikely(xdp_data_meta_unsupported(from)))
201	return from->data;
202	else
203	return from->data_meta;
204	}
205
206	static u32 xsk_copy_xdp(void *to, void **from, u32 to_len,
207	u32 *from_len, skb_frag_t **frag, u32 rem)
208	{
209	u32 copied = 0;
210
211	while (1) {
212	u32 copy_len = min_t(u32, *from_len, to_len);
213
214	memcpy(to, *from, copy_len);
215	copied += copy_len;
216	if (rem == copied)
217	return copied;
218
219	if (*from_len == copy_len) {
220	*from = skb_frag_address(frag: *frag);
221	*from_len = skb_frag_size(frag: (*frag)++);
222	} else {
223	*from += copy_len;
224	*from_len -= copy_len;
225	}
226	if (to_len == copy_len)
227	return copied;
228
229	to_len -= copy_len;
230	to += copy_len;
231	}
232	}
233
234	static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
235	{
236	u32 frame_size = xsk_pool_get_rx_frame_size(pool: xs->pool);
237	void *copy_from = xsk_copy_xdp_start(from: xdp), *copy_to;
238	u32 from_len, meta_len, rem, num_desc;
239	struct xdp_buff_xsk *xskb;
240	struct xdp_buff *xsk_xdp;
241	skb_frag_t *frag;
242
243	from_len = xdp->data_end - copy_from;
244	meta_len = xdp->data - copy_from;
245	rem = len + meta_len;
246
247	if (len <= frame_size && !xdp_buff_has_frags(xdp)) {
248	int err;
249
250	xsk_xdp = xsk_buff_alloc(pool: xs->pool);
251	if (!xsk_xdp) {
252	xs->rx_dropped++;
253	return -ENOMEM;
254	}
255	memcpy(xsk_xdp->data - meta_len, copy_from, rem);
256	xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp);
257	err = __xsk_rcv_zc(xs, xskb, len, flags: 0);
258	if (err) {
259	xsk_buff_free(xdp: xsk_xdp);
260	return err;
261	}
262
263	return 0;
264	}
265
266	num_desc = (len - 1) / frame_size + 1;
267
268	if (!xsk_buff_can_alloc(pool: xs->pool, count: num_desc)) {
269	xs->rx_dropped++;
270	return -ENOMEM;
271	}
272	if (xskq_prod_nb_free(q: xs->rx, max: num_desc) < num_desc) {
273	xs->rx_queue_full++;
274	return -ENOBUFS;
275	}
276
277	if (xdp_buff_has_frags(xdp)) {
278	struct skb_shared_info *sinfo;
279
280	sinfo = xdp_get_shared_info_from_buff(xdp);
281	frag = &sinfo->frags[0];
282	}
283
284	do {
285	u32 to_len = frame_size + meta_len;
286	u32 copied;
287
288	xsk_xdp = xsk_buff_alloc(pool: xs->pool);
289	copy_to = xsk_xdp->data - meta_len;
290
291	copied = xsk_copy_xdp(to: copy_to, from: &copy_from, to_len, from_len: &from_len, frag: &frag, rem);
292	rem -= copied;
293
294	xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp);
295	__xsk_rcv_zc(xs, xskb, len: copied - meta_len, flags: rem ? XDP_PKT_CONTD : 0);
296	meta_len = 0;
297	} while (rem);
298
299	return 0;
300	}
301
302	static bool xsk_tx_writeable(struct xdp_sock *xs)
303	{
304	if (xskq_cons_present_entries(q: xs->tx) > xs->tx->nentries / 2)
305	return false;
306
307	return true;
308	}
309
310	static void __xsk_tx_release(struct xdp_sock *xs)
311	{
312	__xskq_cons_release(q: xs->tx);
313	if (xsk_tx_writeable(xs))
314	xs->sk.sk_write_space(&xs->sk);
315	}
316
317	static bool xsk_is_bound(struct xdp_sock *xs)
318	{
319	if (READ_ONCE(xs->state) == XSK_BOUND) {
320	/* Matches smp_wmb() in bind(). */
321	smp_rmb();
322	return true;
323	}
324	return false;
325	}
326
327	static int xsk_rcv_check(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
328	{
329	if (!xsk_is_bound(xs))
330	return -ENXIO;
331
332	if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index)
333	return -EINVAL;
334
335	if (len > xsk_pool_get_rx_frame_size(pool: xs->pool) && !xs->sg) {
336	xs->rx_dropped++;
337	return -ENOSPC;
338	}
339
340	return 0;
341	}
342
343	static void xsk_flush(struct xdp_sock *xs)
344	{
345	xskq_prod_submit(q: xs->rx);
346	__xskq_cons_release(q: xs->pool->fq);
347	sock_def_readable(sk: &xs->sk);
348	}
349
350	int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
351	{
352	u32 len = xdp_get_buff_len(xdp);
353	int err;
354
355	err = xsk_rcv_check(xs, xdp, len);
356	if (!err) {
357	spin_lock_bh(lock: &xs->pool->rx_lock);
358	err = __xsk_rcv(xs, xdp, len);
359	xsk_flush(xs);
360	spin_unlock_bh(lock: &xs->pool->rx_lock);
361	}
362
363	return err;
364	}
365
366	static int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
367	{
368	u32 len = xdp_get_buff_len(xdp);
369	int err;
370
371	err = xsk_rcv_check(xs, xdp, len);
372	if (err)
373	return err;
374
375	if (xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL) {
376	len = xdp->data_end - xdp->data;
377	return xsk_rcv_zc(xs, xdp, len);
378	}
379
380	err = __xsk_rcv(xs, xdp, len);
381	if (!err)
382	xdp_return_buff(xdp);
383	return err;
384	}
385
386	int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp)
387	{
388	int err;
389
390	err = xsk_rcv(xs, xdp);
391	if (err)
392	return err;
393
394	if (!xs->flush_node.prev) {
395	struct list_head *flush_list = bpf_net_ctx_get_xskmap_flush_list();
396
397	list_add(new: &xs->flush_node, head: flush_list);
398	}
399
400	return 0;
401	}
402
403	void __xsk_map_flush(struct list_head *flush_list)
404	{
405	struct xdp_sock *xs, *tmp;
406
407	list_for_each_entry_safe(xs, tmp, flush_list, flush_node) {
408	xsk_flush(xs);
409	__list_del_clearprev(entry: &xs->flush_node);
410	}
411	}
412
413	void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries)
414	{
415	xskq_prod_submit_n(q: pool->cq, nb_entries);
416	}
417	EXPORT_SYMBOL(xsk_tx_completed);
418
419	void xsk_tx_release(struct xsk_buff_pool *pool)
420	{
421	struct xdp_sock *xs;
422
423	rcu_read_lock();
424	list_for_each_entry_rcu(xs, &pool->xsk_tx_list, tx_list)
425	__xsk_tx_release(xs);
426	rcu_read_unlock();
427	}
428	EXPORT_SYMBOL(xsk_tx_release);
429
430	bool xsk_tx_peek_desc(struct xsk_buff_pool *pool, struct xdp_desc *desc)
431	{
432	bool budget_exhausted = false;
433	struct xdp_sock *xs;
434
435	rcu_read_lock();
436	again:
437	list_for_each_entry_rcu(xs, &pool->xsk_tx_list, tx_list) {
438	if (xs->tx_budget_spent >= MAX_PER_SOCKET_BUDGET) {
439	budget_exhausted = true;
440	continue;
441	}
442
443	if (!xskq_cons_peek_desc(q: xs->tx, desc, pool)) {
444	if (xskq_has_descs(q: xs->tx))
445	xskq_cons_release(q: xs->tx);
446	continue;
447	}
448
449	xs->tx_budget_spent++;
450
451	/* This is the backpressure mechanism for the Tx path.
452	* Reserve space in the completion queue and only proceed
453	* if there is space in it. This avoids having to implement
454	* any buffering in the Tx path.
455	*/
456	if (xskq_prod_reserve_addr(q: pool->cq, addr: desc->addr))
457	goto out;
458
459	xskq_cons_release(q: xs->tx);
460	rcu_read_unlock();
461	return true;
462	}
463
464	if (budget_exhausted) {
465	list_for_each_entry_rcu(xs, &pool->xsk_tx_list, tx_list)
466	xs->tx_budget_spent = 0;
467
468	budget_exhausted = false;
469	goto again;
470	}
471
472	out:
473	rcu_read_unlock();
474	return false;
475	}
476	EXPORT_SYMBOL(xsk_tx_peek_desc);
477
478	static u32 xsk_tx_peek_release_fallback(struct xsk_buff_pool *pool, u32 max_entries)
479	{
480	struct xdp_desc *descs = pool->tx_descs;
481	u32 nb_pkts = 0;
482
483	while (nb_pkts < max_entries && xsk_tx_peek_desc(pool, &descs[nb_pkts]))
484	nb_pkts++;
485
486	xsk_tx_release(pool);
487	return nb_pkts;
488	}
489
490	u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 nb_pkts)
491	{
492	struct xdp_sock *xs;
493
494	rcu_read_lock();
495	if (!list_is_singular(head: &pool->xsk_tx_list)) {
496	/* Fallback to the non-batched version */
497	rcu_read_unlock();
498	return xsk_tx_peek_release_fallback(pool, max_entries: nb_pkts);
499	}
500
501	xs = list_first_or_null_rcu(&pool->xsk_tx_list, struct xdp_sock, tx_list);
502	if (!xs) {
503	nb_pkts = 0;
504	goto out;
505	}
506
507	nb_pkts = xskq_cons_nb_entries(q: xs->tx, max: nb_pkts);
508
509	/* This is the backpressure mechanism for the Tx path. Try to
510	* reserve space in the completion queue for all packets, but
511	* if there are fewer slots available, just process that many
512	* packets. This avoids having to implement any buffering in
513	* the Tx path.
514	*/
515	nb_pkts = xskq_prod_nb_free(q: pool->cq, max: nb_pkts);
516	if (!nb_pkts)
517	goto out;
518
519	nb_pkts = xskq_cons_read_desc_batch(q: xs->tx, pool, max: nb_pkts);
520	if (!nb_pkts) {
521	xs->tx->queue_empty_descs++;
522	goto out;
523	}
524
525	__xskq_cons_release(q: xs->tx);
526	xskq_prod_write_addr_batch(q: pool->cq, descs: pool->tx_descs, nb_entries: nb_pkts);
527	xs->sk.sk_write_space(&xs->sk);
528
529	out:
530	rcu_read_unlock();
531	return nb_pkts;
532	}
533	EXPORT_SYMBOL(xsk_tx_peek_release_desc_batch);
534
535	static int xsk_wakeup(struct xdp_sock *xs, u8 flags)
536	{
537	struct net_device *dev = xs->dev;
538
539	return dev->netdev_ops->ndo_xsk_wakeup(dev, xs->queue_id, flags);
540	}
541
542	static int xsk_cq_reserve_locked(struct xsk_buff_pool *pool)
543	{
544	int ret;
545
546	spin_lock(lock: &pool->cq_cached_prod_lock);
547	ret = xskq_prod_reserve(q: pool->cq);
548	spin_unlock(lock: &pool->cq_cached_prod_lock);
549
550	return ret;
551	}
552
553	static bool xsk_skb_destructor_is_addr(struct sk_buff *skb)
554	{
555	return (uintptr_t)skb_shinfo(skb)->destructor_arg & 0x1UL;
556	}
557
558	static u64 xsk_skb_destructor_get_addr(struct sk_buff *skb)
559	{
560	return (u64)((uintptr_t)skb_shinfo(skb)->destructor_arg & ~0x1UL);
561	}
562
563	static void xsk_skb_destructor_set_addr(struct sk_buff *skb, u64 addr)
564	{
565	skb_shinfo(skb)->destructor_arg = (void *)((uintptr_t)addr | 0x1UL);
566	}
567
568	static void xsk_inc_num_desc(struct sk_buff *skb)
569	{
570	struct xsk_addrs *xsk_addr;
571
572	if (!xsk_skb_destructor_is_addr(skb)) {
573	xsk_addr = (struct xsk_addrs *)skb_shinfo(skb)->destructor_arg;
574	xsk_addr->num_descs++;
575	}
576	}
577
578	static u32 xsk_get_num_desc(struct sk_buff *skb)
579	{
580	struct xsk_addrs *xsk_addr;
581
582	if (xsk_skb_destructor_is_addr(skb))
583	return 1;
584
585	xsk_addr = (struct xsk_addrs *)skb_shinfo(skb)->destructor_arg;
586
587	return xsk_addr->num_descs;
588	}
589
590	static void xsk_cq_submit_addr_locked(struct xsk_buff_pool *pool,
591	struct sk_buff *skb)
592	{
593	u32 num_descs = xsk_get_num_desc(skb);
594	struct xsk_addrs *xsk_addr;
595	u32 descs_processed = 0;
596	unsigned long flags;
597	u32 idx, i;
598
599	spin_lock_irqsave(&pool->cq_prod_lock, flags);
600	idx = xskq_get_prod(q: pool->cq);
601
602	if (unlikely(num_descs > 1)) {
603	xsk_addr = (struct xsk_addrs *)skb_shinfo(skb)->destructor_arg;
604
605	for (i = 0; i < num_descs; i++) {
606	xskq_prod_write_addr(q: pool->cq, idx: idx + descs_processed,
607	addr: xsk_addr->addrs[i]);
608	descs_processed++;
609	}
610	kmem_cache_free(s: xsk_tx_generic_cache, objp: xsk_addr);
611	} else {
612	xskq_prod_write_addr(q: pool->cq, idx,
613	addr: xsk_skb_destructor_get_addr(skb));
614	descs_processed++;
615	}
616	xskq_prod_submit_n(q: pool->cq, nb_entries: descs_processed);
617	spin_unlock_irqrestore(lock: &pool->cq_prod_lock, flags);
618	}
619
620	static void xsk_cq_cancel_locked(struct xsk_buff_pool *pool, u32 n)
621	{
622	spin_lock(lock: &pool->cq_cached_prod_lock);
623	xskq_prod_cancel_n(q: pool->cq, cnt: n);
624	spin_unlock(lock: &pool->cq_cached_prod_lock);
625	}
626
627	INDIRECT_CALLABLE_SCOPE
628	void xsk_destruct_skb(struct sk_buff *skb)
629	{
630	struct xsk_tx_metadata_compl *compl = &skb_shinfo(skb)->xsk_meta;
631
632	if (compl->tx_timestamp) {
633	/* sw completion timestamp, not a real one */
634	*compl->tx_timestamp = ktime_get_tai_fast_ns();
635	}
636
637	xsk_cq_submit_addr_locked(pool: xdp_sk(sk: skb->sk)->pool, skb);
638	sock_wfree(skb);
639	}
640
641	static void xsk_skb_init_misc(struct sk_buff *skb, struct xdp_sock *xs,
642	u64 addr)
643	{
644	skb->dev = xs->dev;
645	skb->priority = READ_ONCE(xs->sk.sk_priority);
646	skb->mark = READ_ONCE(xs->sk.sk_mark);
647	skb->destructor = xsk_destruct_skb;
648	xsk_skb_destructor_set_addr(skb, addr);
649	}
650
651	static void xsk_consume_skb(struct sk_buff *skb)
652	{
653	struct xdp_sock *xs = xdp_sk(sk: skb->sk);
654	u32 num_descs = xsk_get_num_desc(skb);
655	struct xsk_addrs *xsk_addr;
656
657	if (unlikely(num_descs > 1)) {
658	xsk_addr = (struct xsk_addrs *)skb_shinfo(skb)->destructor_arg;
659	kmem_cache_free(s: xsk_tx_generic_cache, objp: xsk_addr);
660	}
661
662	skb->destructor = sock_wfree;
663	xsk_cq_cancel_locked(pool: xs->pool, n: num_descs);
664	/* Free skb without triggering the perf drop trace */
665	consume_skb(skb);
666	xs->skb = NULL;
667	}
668
669	static void xsk_drop_skb(struct sk_buff *skb)
670	{
671	xdp_sk(sk: skb->sk)->tx->invalid_descs += xsk_get_num_desc(skb);
672	xsk_consume_skb(skb);
673	}
674
675	static int xsk_skb_metadata(struct sk_buff *skb, void *buffer,
676	struct xdp_desc *desc, struct xsk_buff_pool *pool,
677	u32 hr)
678	{
679	struct xsk_tx_metadata *meta = NULL;
680
681	if (unlikely(pool->tx_metadata_len == 0))
682	return -EINVAL;
683
684	meta = buffer - pool->tx_metadata_len;
685	if (unlikely(!xsk_buff_valid_tx_metadata(meta)))
686	return -EINVAL;
687
688	if (meta->flags & XDP_TXMD_FLAGS_CHECKSUM) {
689	if (unlikely(meta->request.csum_start +
690	meta->request.csum_offset +
691	sizeof(__sum16) > desc->len))
692	return -EINVAL;
693
694	skb->csum_start = hr + meta->request.csum_start;
695	skb->csum_offset = meta->request.csum_offset;
696	skb->ip_summed = CHECKSUM_PARTIAL;
697
698	if (unlikely(pool->tx_sw_csum)) {
699	int err;
700
701	err = skb_checksum_help(skb);
702	if (err)
703	return err;
704	}
705	}
706
707	if (meta->flags & XDP_TXMD_FLAGS_LAUNCH_TIME)
708	skb->skb_mstamp_ns = meta->request.launch_time;
709	xsk_tx_metadata_to_compl(meta, compl: &skb_shinfo(skb)->xsk_meta);
710
711	return 0;
712	}
713
714	static struct sk_buff *xsk_build_skb_zerocopy(struct xdp_sock *xs,
715	struct xdp_desc *desc)
716	{
717	struct xsk_buff_pool *pool = xs->pool;
718	u32 hr, len, ts, offset, copy, copied;
719	struct sk_buff *skb = xs->skb;
720	struct page *page;
721	void *buffer;
722	int err, i;
723	u64 addr;
724
725	addr = desc->addr;
726	buffer = xsk_buff_raw_get_data(pool, addr);
727
728	if (!skb) {
729	hr = max(NET_SKB_PAD, L1_CACHE_ALIGN(xs->dev->needed_headroom));
730
731	skb = sock_alloc_send_skb(sk: &xs->sk, size: hr, noblock: 1, errcode: &err);
732	if (unlikely(!skb))
733	return ERR_PTR(error: err);
734
735	skb_reserve(skb, len: hr);
736
737	xsk_skb_init_misc(skb, xs, addr: desc->addr);
738	if (desc->options & XDP_TX_METADATA) {
739	err = xsk_skb_metadata(skb, buffer, desc, pool, hr);
740	if (unlikely(err))
741	return ERR_PTR(error: err);
742	}
743	} else {
744	struct xsk_addrs *xsk_addr;
745
746	if (xsk_skb_destructor_is_addr(skb)) {
747	xsk_addr = kmem_cache_zalloc(xsk_tx_generic_cache,
748	GFP_KERNEL);
749	if (!xsk_addr)
750	return ERR_PTR(error: -ENOMEM);
751
752	xsk_addr->num_descs = 1;
753	xsk_addr->addrs[0] = xsk_skb_destructor_get_addr(skb);
754	skb_shinfo(skb)->destructor_arg = (void *)xsk_addr;
755	} else {
756	xsk_addr = (struct xsk_addrs *)skb_shinfo(skb)->destructor_arg;
757	}
758
759	/* in case of -EOVERFLOW that could happen below,
760	* xsk_consume_skb() will release this node as whole skb
761	* would be dropped, which implies freeing all list elements
762	*/
763	xsk_addr->addrs[xsk_addr->num_descs] = desc->addr;
764	}
765
766	len = desc->len;
767	ts = pool->unaligned ? len : pool->chunk_size;
768
769	offset = offset_in_page(buffer);
770	addr = buffer - pool->addrs;
771
772	for (copied = 0, i = skb_shinfo(skb)->nr_frags; copied < len; i++) {
773	if (unlikely(i >= MAX_SKB_FRAGS))
774	return ERR_PTR(error: -EOVERFLOW);
775
776	page = pool->umem->pgs[addr >> PAGE_SHIFT];
777	get_page(page);
778
779	copy = min_t(u32, PAGE_SIZE - offset, len - copied);
780	skb_fill_page_desc(skb, i, page, off: offset, size: copy);
781
782	copied += copy;
783	addr += copy;
784	offset = 0;
785	}
786
787	skb->len += len;
788	skb->data_len += len;
789	skb->truesize += ts;
790
791	refcount_add(i: ts, r: &xs->sk.sk_wmem_alloc);
792
793	return skb;
794	}
795
796	static struct sk_buff *xsk_build_skb(struct xdp_sock *xs,
797	struct xdp_desc *desc)
798	{
799	struct net_device *dev = xs->dev;
800	struct sk_buff *skb = xs->skb;
801	int err;
802
803	if (dev->priv_flags & IFF_TX_SKB_NO_LINEAR) {
804	skb = xsk_build_skb_zerocopy(xs, desc);
805	if (IS_ERR(ptr: skb)) {
806	err = PTR_ERR(ptr: skb);
807	skb = NULL;
808	goto free_err;
809	}
810	} else {
811	u32 hr, tr, len;
812	void *buffer;
813
814	buffer = xsk_buff_raw_get_data(pool: xs->pool, addr: desc->addr);
815	len = desc->len;
816
817	if (!skb) {
818	hr = max(NET_SKB_PAD, L1_CACHE_ALIGN(dev->needed_headroom));
819	tr = dev->needed_tailroom;
820	skb = sock_alloc_send_skb(sk: &xs->sk, size: hr + len + tr, noblock: 1, errcode: &err);
821	if (unlikely(!skb))
822	goto free_err;
823
824	skb_reserve(skb, len: hr);
825	skb_put(skb, len);
826
827	err = skb_store_bits(skb, offset: 0, from: buffer, len);
828	if (unlikely(err))
829	goto free_err;
830
831	xsk_skb_init_misc(skb, xs, addr: desc->addr);
832	if (desc->options & XDP_TX_METADATA) {
833	err = xsk_skb_metadata(skb, buffer, desc,
834	pool: xs->pool, hr);
835	if (unlikely(err))
836	goto free_err;
837	}
838	} else {
839	int nr_frags = skb_shinfo(skb)->nr_frags;
840	struct xsk_addrs *xsk_addr;
841	struct page *page;
842	u8 *vaddr;
843
844	if (xsk_skb_destructor_is_addr(skb)) {
845	xsk_addr = kmem_cache_zalloc(xsk_tx_generic_cache,
846	GFP_KERNEL);
847	if (!xsk_addr) {
848	err = -ENOMEM;
849	goto free_err;
850	}
851
852	xsk_addr->num_descs = 1;
853	xsk_addr->addrs[0] = xsk_skb_destructor_get_addr(skb);
854	skb_shinfo(skb)->destructor_arg = (void *)xsk_addr;
855	} else {
856	xsk_addr = (struct xsk_addrs *)skb_shinfo(skb)->destructor_arg;
857	}
858
859	if (unlikely(nr_frags == (MAX_SKB_FRAGS - 1) && xp_mb_desc(desc))) {
860	err = -EOVERFLOW;
861	goto free_err;
862	}
863
864	page = alloc_page(xs->sk.sk_allocation);
865	if (unlikely(!page)) {
866	err = -EAGAIN;
867	goto free_err;
868	}
869
870	vaddr = kmap_local_page(page);
871	memcpy(vaddr, buffer, len);
872	kunmap_local(vaddr);
873
874	skb_add_rx_frag(skb, i: nr_frags, page, off: 0, size: len, PAGE_SIZE);
875	refcount_add(PAGE_SIZE, r: &xs->sk.sk_wmem_alloc);
876
877	xsk_addr->addrs[xsk_addr->num_descs] = desc->addr;
878	}
879	}
880
881	xsk_inc_num_desc(skb);
882
883	return skb;
884
885	free_err:
886	if (skb && !skb_shinfo(skb)->nr_frags)
887	kfree_skb(skb);
888
889	if (err == -EOVERFLOW) {
890	/* Drop the packet */
891	xsk_inc_num_desc(skb: xs->skb);
892	xsk_drop_skb(skb: xs->skb);
893	xskq_cons_release(q: xs->tx);
894	} else {
895	/* Let application retry */
896	xsk_cq_cancel_locked(pool: xs->pool, n: 1);
897	}
898
899	return ERR_PTR(error: err);
900	}
901
902	static int __xsk_generic_xmit(struct sock *sk)
903	{
904	struct xdp_sock *xs = xdp_sk(sk);
905	bool sent_frame = false;
906	struct xdp_desc desc;
907	struct sk_buff *skb;
908	u32 max_batch;
909	int err = 0;
910
911	mutex_lock(&xs->mutex);
912
913	/* Since we dropped the RCU read lock, the socket state might have changed. */
914	if (unlikely(!xsk_is_bound(xs))) {
915	err = -ENXIO;
916	goto out;
917	}
918
919	if (xs->queue_id >= xs->dev->real_num_tx_queues)
920	goto out;
921
922	max_batch = READ_ONCE(xs->max_tx_budget);
923	while (xskq_cons_peek_desc(q: xs->tx, desc: &desc, pool: xs->pool)) {
924	if (max_batch-- == 0) {
925	err = -EAGAIN;
926	goto out;
927	}
928
929	/* This is the backpressure mechanism for the Tx path.
930	* Reserve space in the completion queue and only proceed
931	* if there is space in it. This avoids having to implement
932	* any buffering in the Tx path.
933	*/
934	err = xsk_cq_reserve_locked(pool: xs->pool);
935	if (err) {
936	err = -EAGAIN;
937	goto out;
938	}
939
940	skb = xsk_build_skb(xs, desc: &desc);
941	if (IS_ERR(ptr: skb)) {
942	err = PTR_ERR(ptr: skb);
943	if (err != -EOVERFLOW)
944	goto out;
945	err = 0;
946	continue;
947	}
948
949	xskq_cons_release(q: xs->tx);
950
951	if (xp_mb_desc(desc: &desc)) {
952	xs->skb = skb;
953	continue;
954	}
955
956	err = __dev_direct_xmit(skb, queue_id: xs->queue_id);
957	if (err == NETDEV_TX_BUSY) {
958	/* Tell user-space to retry the send */
959	xskq_cons_cancel_n(q: xs->tx, cnt: xsk_get_num_desc(skb));
960	xsk_consume_skb(skb);
961	err = -EAGAIN;
962	goto out;
963	}
964
965	/* Ignore NET_XMIT_CN as packet might have been sent */
966	if (err == NET_XMIT_DROP) {
967	/* SKB completed but not sent */
968	err = -EBUSY;
969	xs->skb = NULL;
970	goto out;
971	}
972
973	sent_frame = true;
974	xs->skb = NULL;
975	}
976
977	if (xskq_has_descs(q: xs->tx)) {
978	if (xs->skb)
979	xsk_drop_skb(skb: xs->skb);
980	xskq_cons_release(q: xs->tx);
981	}
982
983	out:
984	if (sent_frame)
985	__xsk_tx_release(xs);
986
987	mutex_unlock(lock: &xs->mutex);
988	return err;
989	}
990
991	static int xsk_generic_xmit(struct sock *sk)
992	{
993	int ret;
994
995	/* Drop the RCU lock since the SKB path might sleep. */
996	rcu_read_unlock();
997	ret = __xsk_generic_xmit(sk);
998	/* Reaquire RCU lock before going into common code. */
999	rcu_read_lock();
1000
1001	return ret;
1002	}
1003
1004	static bool xsk_no_wakeup(struct sock *sk)
1005	{
1006	#ifdef CONFIG_NET_RX_BUSY_POLL
1007	/* Prefer busy-polling, skip the wakeup. */
1008	return READ_ONCE(sk->sk_prefer_busy_poll) && READ_ONCE(sk->sk_ll_usec) &&
1009	napi_id_valid(READ_ONCE(sk->sk_napi_id));
1010	#else
1011	return false;
1012	#endif
1013	}
1014
1015	static int xsk_check_common(struct xdp_sock *xs)
1016	{
1017	if (unlikely(!xsk_is_bound(xs)))
1018	return -ENXIO;
1019	if (unlikely(!(xs->dev->flags & IFF_UP)))
1020	return -ENETDOWN;
1021
1022	return 0;
1023	}
1024
1025	static int __xsk_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len)
1026	{
1027	bool need_wait = !(m->msg_flags & MSG_DONTWAIT);
1028	struct sock *sk = sock->sk;
1029	struct xdp_sock *xs = xdp_sk(sk);
1030	struct xsk_buff_pool *pool;
1031	int err;
1032
1033	err = xsk_check_common(xs);
1034	if (err)
1035	return err;
1036	if (unlikely(need_wait))
1037	return -EOPNOTSUPP;
1038	if (unlikely(!xs->tx))
1039	return -ENOBUFS;
1040
1041	if (sk_can_busy_loop(sk))
1042	sk_busy_loop(sk, nonblock: 1); /* only support non-blocking sockets */
1043
1044	if (xs->zc && xsk_no_wakeup(sk))
1045	return 0;
1046
1047	pool = xs->pool;
1048	if (pool->cached_need_wakeup & XDP_WAKEUP_TX) {
1049	if (xs->zc)
1050	return xsk_wakeup(xs, XDP_WAKEUP_TX);
1051	return xsk_generic_xmit(sk);
1052	}
1053	return 0;
1054	}
1055
1056	static int xsk_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len)
1057	{
1058	int ret;
1059
1060	rcu_read_lock();
1061	ret = __xsk_sendmsg(sock, m, total_len);
1062	rcu_read_unlock();
1063
1064	return ret;
1065	}
1066
1067	static int __xsk_recvmsg(struct socket *sock, struct msghdr *m, size_t len, int flags)
1068	{
1069	bool need_wait = !(flags & MSG_DONTWAIT);
1070	struct sock *sk = sock->sk;
1071	struct xdp_sock *xs = xdp_sk(sk);
1072	int err;
1073
1074	err = xsk_check_common(xs);
1075	if (err)
1076	return err;
1077	if (unlikely(!xs->rx))
1078	return -ENOBUFS;
1079	if (unlikely(need_wait))
1080	return -EOPNOTSUPP;
1081
1082	if (sk_can_busy_loop(sk))
1083	sk_busy_loop(sk, nonblock: 1); /* only support non-blocking sockets */
1084
1085	if (xsk_no_wakeup(sk))
1086	return 0;
1087
1088	if (xs->pool->cached_need_wakeup & XDP_WAKEUP_RX && xs->zc)
1089	return xsk_wakeup(xs, XDP_WAKEUP_RX);
1090	return 0;
1091	}
1092
1093	static int xsk_recvmsg(struct socket *sock, struct msghdr *m, size_t len, int flags)
1094	{
1095	int ret;
1096
1097	rcu_read_lock();
1098	ret = __xsk_recvmsg(sock, m, len, flags);
1099	rcu_read_unlock();
1100
1101	return ret;
1102	}
1103
1104	static __poll_t xsk_poll(struct file *file, struct socket *sock,
1105	struct poll_table_struct *wait)
1106	{
1107	__poll_t mask = 0;
1108	struct sock *sk = sock->sk;
1109	struct xdp_sock *xs = xdp_sk(sk);
1110	struct xsk_buff_pool *pool;
1111
1112	sock_poll_wait(filp: file, sock, p: wait);
1113
1114	rcu_read_lock();
1115	if (xsk_check_common(xs))
1116	goto out;
1117
1118	pool = xs->pool;
1119
1120	if (pool->cached_need_wakeup) {
1121	if (xs->zc)
1122	xsk_wakeup(xs, flags: pool->cached_need_wakeup);
1123	else if (xs->tx)
1124	/* Poll needs to drive Tx also in copy mode */
1125	xsk_generic_xmit(sk);
1126	}
1127
1128	if (xs->rx && !xskq_prod_is_empty(q: xs->rx))
1129	mask |= EPOLLIN | EPOLLRDNORM;
1130	if (xs->tx && xsk_tx_writeable(xs))
1131	mask |= EPOLLOUT | EPOLLWRNORM;
1132	out:
1133	rcu_read_unlock();
1134	return mask;
1135	}
1136
1137	static int xsk_init_queue(u32 entries, struct xsk_queue **queue,
1138	bool umem_queue)
1139	{
1140	struct xsk_queue *q;
1141
1142	if (entries == 0 || *queue || !is_power_of_2(n: entries))
1143	return -EINVAL;
1144
1145	q = xskq_create(nentries: entries, umem_queue);
1146	if (!q)
1147	return -ENOMEM;
1148
1149	/* Make sure queue is ready before it can be seen by others */
1150	smp_wmb();
1151	WRITE_ONCE(*queue, q);
1152	return 0;
1153	}
1154
1155	static void xsk_unbind_dev(struct xdp_sock *xs)
1156	{
1157	struct net_device *dev = xs->dev;
1158
1159	if (xs->state != XSK_BOUND)
1160	return;
1161	WRITE_ONCE(xs->state, XSK_UNBOUND);
1162
1163	/* Wait for driver to stop using the xdp socket. */
1164	xp_del_xsk(pool: xs->pool, xs);
1165	synchronize_net();
1166	dev_put(dev);
1167	}
1168
1169	static struct xsk_map *xsk_get_map_list_entry(struct xdp_sock *xs,
1170	struct xdp_sock __rcu ***map_entry)
1171	{
1172	struct xsk_map *map = NULL;
1173	struct xsk_map_node *node;
1174
1175	*map_entry = NULL;
1176
1177	spin_lock_bh(lock: &xs->map_list_lock);
1178	node = list_first_entry_or_null(&xs->map_list, struct xsk_map_node,
1179	node);
1180	if (node) {
1181	bpf_map_inc(map: &node->map->map);
1182	map = node->map;
1183	*map_entry = node->map_entry;
1184	}
1185	spin_unlock_bh(lock: &xs->map_list_lock);
1186	return map;
1187	}
1188
1189	static void xsk_delete_from_maps(struct xdp_sock *xs)
1190	{
1191	/* This function removes the current XDP socket from all the
1192	* maps it resides in. We need to take extra care here, due to
1193	* the two locks involved. Each map has a lock synchronizing
1194	* updates to the entries, and each socket has a lock that
1195	* synchronizes access to the list of maps (map_list). For
1196	* deadlock avoidance the locks need to be taken in the order
1197	* "map lock"->"socket map list lock". We start off by
1198	* accessing the socket map list, and take a reference to the
1199	* map to guarantee existence between the
1200	* xsk_get_map_list_entry() and xsk_map_try_sock_delete()
1201	* calls. Then we ask the map to remove the socket, which
1202	* tries to remove the socket from the map. Note that there
1203	* might be updates to the map between
1204	* xsk_get_map_list_entry() and xsk_map_try_sock_delete().
1205	*/
1206	struct xdp_sock __rcu **map_entry = NULL;
1207	struct xsk_map *map;
1208
1209	while ((map = xsk_get_map_list_entry(xs, map_entry: &map_entry))) {
1210	xsk_map_try_sock_delete(map, xs, map_entry);
1211	bpf_map_put(map: &map->map);
1212	}
1213	}
1214
1215	static int xsk_release(struct socket *sock)
1216	{
1217	struct sock *sk = sock->sk;
1218	struct xdp_sock *xs = xdp_sk(sk);
1219	struct net *net;
1220
1221	if (!sk)
1222	return 0;
1223
1224	net = sock_net(sk);
1225
1226	if (xs->skb)
1227	xsk_drop_skb(skb: xs->skb);
1228
1229	mutex_lock(&net->xdp.lock);
1230	sk_del_node_init_rcu(sk);
1231	mutex_unlock(lock: &net->xdp.lock);
1232
1233	sock_prot_inuse_add(net, prot: sk->sk_prot, val: -1);
1234
1235	xsk_delete_from_maps(xs);
1236	mutex_lock(&xs->mutex);
1237	xsk_unbind_dev(xs);
1238	mutex_unlock(lock: &xs->mutex);
1239
1240	xskq_destroy(q_ops: xs->rx);
1241	xskq_destroy(q_ops: xs->tx);
1242	xskq_destroy(q_ops: xs->fq_tmp);
1243	xskq_destroy(q_ops: xs->cq_tmp);
1244
1245	sock_orphan(sk);
1246	sock->sk = NULL;
1247
1248	sock_put(sk);
1249
1250	return 0;
1251	}
1252
1253	static struct socket *xsk_lookup_xsk_from_fd(int fd)
1254	{
1255	struct socket *sock;
1256	int err;
1257
1258	sock = sockfd_lookup(fd, err: &err);
1259	if (!sock)
1260	return ERR_PTR(error: -ENOTSOCK);
1261
1262	if (sock->sk->sk_family != PF_XDP) {
1263	sockfd_put(sock);
1264	return ERR_PTR(error: -ENOPROTOOPT);
1265	}
1266
1267	return sock;
1268	}
1269
1270	static bool xsk_validate_queues(struct xdp_sock *xs)
1271	{
1272	return xs->fq_tmp && xs->cq_tmp;
1273	}
1274
1275	static int xsk_bind(struct socket *sock, struct sockaddr_unsized *addr, int addr_len)
1276	{
1277	struct sockaddr_xdp *sxdp = (struct sockaddr_xdp *)addr;
1278	struct sock *sk = sock->sk;
1279	struct xdp_sock *xs = xdp_sk(sk);
1280	struct net_device *dev;
1281	int bound_dev_if;
1282	u32 flags, qid;
1283	int err = 0;
1284
1285	if (addr_len < sizeof(struct sockaddr_xdp))
1286	return -EINVAL;
1287	if (sxdp->sxdp_family != AF_XDP)
1288	return -EINVAL;
1289
1290	flags = sxdp->sxdp_flags;
1291	if (flags & ~(XDP_SHARED_UMEM | XDP_COPY | XDP_ZEROCOPY |
1292	XDP_USE_NEED_WAKEUP | XDP_USE_SG))
1293	return -EINVAL;
1294
1295	bound_dev_if = READ_ONCE(sk->sk_bound_dev_if);
1296	if (bound_dev_if && bound_dev_if != sxdp->sxdp_ifindex)
1297	return -EINVAL;
1298
1299	rtnl_lock();
1300	mutex_lock(&xs->mutex);
1301	if (xs->state != XSK_READY) {
1302	err = -EBUSY;
1303	goto out_release;
1304	}
1305
1306	dev = dev_get_by_index(net: sock_net(sk), ifindex: sxdp->sxdp_ifindex);
1307	if (!dev) {
1308	err = -ENODEV;
1309	goto out_release;
1310	}
1311
1312	netdev_lock_ops(dev);
1313
1314	if (!xs->rx && !xs->tx) {
1315	err = -EINVAL;
1316	goto out_unlock;
1317	}
1318
1319	qid = sxdp->sxdp_queue_id;
1320
1321	if (flags & XDP_SHARED_UMEM) {
1322	struct xdp_sock *umem_xs;
1323	struct socket *sock;
1324
1325	if ((flags & XDP_COPY) || (flags & XDP_ZEROCOPY) ||
1326	(flags & XDP_USE_NEED_WAKEUP) || (flags & XDP_USE_SG)) {
1327	/* Cannot specify flags for shared sockets. */
1328	err = -EINVAL;
1329	goto out_unlock;
1330	}
1331
1332	if (xs->umem) {
1333	/* We have already our own. */
1334	err = -EINVAL;
1335	goto out_unlock;
1336	}
1337
1338	sock = xsk_lookup_xsk_from_fd(fd: sxdp->sxdp_shared_umem_fd);
1339	if (IS_ERR(ptr: sock)) {
1340	err = PTR_ERR(ptr: sock);
1341	goto out_unlock;
1342	}
1343
1344	umem_xs = xdp_sk(sk: sock->sk);
1345	if (!xsk_is_bound(xs: umem_xs)) {
1346	err = -EBADF;
1347	sockfd_put(sock);
1348	goto out_unlock;
1349	}
1350
1351	if (umem_xs->queue_id != qid || umem_xs->dev != dev) {
1352	/* Share the umem with another socket on another qid
1353	* and/or device.
1354	*/
1355	xs->pool = xp_create_and_assign_umem(xs,
1356	umem: umem_xs->umem);
1357	if (!xs->pool) {
1358	err = -ENOMEM;
1359	sockfd_put(sock);
1360	goto out_unlock;
1361	}
1362
1363	err = xp_assign_dev_shared(pool: xs->pool, umem_xs, dev,
1364	queue_id: qid);
1365	if (err) {
1366	xp_destroy(pool: xs->pool);
1367	xs->pool = NULL;
1368	sockfd_put(sock);
1369	goto out_unlock;
1370	}
1371	} else {
1372	/* Share the buffer pool with the other socket. */
1373	if (xs->fq_tmp || xs->cq_tmp) {
1374	/* Do not allow setting your own fq or cq. */
1375	err = -EINVAL;
1376	sockfd_put(sock);
1377	goto out_unlock;
1378	}
1379
1380	xp_get_pool(pool: umem_xs->pool);
1381	xs->pool = umem_xs->pool;
1382
1383	/* If underlying shared umem was created without Tx
1384	* ring, allocate Tx descs array that Tx batching API
1385	* utilizes
1386	*/
1387	if (xs->tx && !xs->pool->tx_descs) {
1388	err = xp_alloc_tx_descs(pool: xs->pool, xs);
1389	if (err) {
1390	xp_put_pool(pool: xs->pool);
1391	xs->pool = NULL;
1392	sockfd_put(sock);
1393	goto out_unlock;
1394	}
1395	}
1396	}
1397
1398	xdp_get_umem(umem: umem_xs->umem);
1399	WRITE_ONCE(xs->umem, umem_xs->umem);
1400	sockfd_put(sock);
1401	} else if (!xs->umem || !xsk_validate_queues(xs)) {
1402	err = -EINVAL;
1403	goto out_unlock;
1404	} else {
1405	/* This xsk has its own umem. */
1406	xs->pool = xp_create_and_assign_umem(xs, umem: xs->umem);
1407	if (!xs->pool) {
1408	err = -ENOMEM;
1409	goto out_unlock;
1410	}
1411
1412	err = xp_assign_dev(pool: xs->pool, dev, queue_id: qid, flags);
1413	if (err) {
1414	xp_destroy(pool: xs->pool);
1415	xs->pool = NULL;
1416	goto out_unlock;
1417	}
1418	}
1419
1420	/* FQ and CQ are now owned by the buffer pool and cleaned up with it. */
1421	xs->fq_tmp = NULL;
1422	xs->cq_tmp = NULL;
1423
1424	xs->dev = dev;
1425	xs->zc = xs->umem->zc;
1426	xs->sg = !!(xs->umem->flags & XDP_UMEM_SG_FLAG);
1427	xs->queue_id = qid;
1428	xp_add_xsk(pool: xs->pool, xs);
1429
1430	if (qid < dev->real_num_rx_queues) {
1431	struct netdev_rx_queue *rxq;
1432
1433	rxq = __netif_get_rx_queue(dev, rxq: qid);
1434	if (rxq->napi)
1435	__sk_mark_napi_id_once(sk, napi_id: rxq->napi->napi_id);
1436	}
1437
1438	out_unlock:
1439	if (err) {
1440	dev_put(dev);
1441	} else {
1442	/* Matches smp_rmb() in bind() for shared umem
1443	* sockets, and xsk_is_bound().
1444	*/
1445	smp_wmb();
1446	WRITE_ONCE(xs->state, XSK_BOUND);
1447	}
1448	netdev_unlock_ops(dev);
1449	out_release:
1450	mutex_unlock(lock: &xs->mutex);
1451	rtnl_unlock();
1452	return err;
1453	}
1454
1455	struct xdp_umem_reg_v1 {
1456	__u64 addr; /* Start of packet data area */
1457	__u64 len; /* Length of packet data area */
1458	__u32 chunk_size;
1459	__u32 headroom;
1460	};
1461
1462	static int xsk_setsockopt(struct socket *sock, int level, int optname,
1463	sockptr_t optval, unsigned int optlen)
1464	{
1465	struct sock *sk = sock->sk;
1466	struct xdp_sock *xs = xdp_sk(sk);
1467	int err;
1468
1469	if (level != SOL_XDP)
1470	return -ENOPROTOOPT;
1471
1472	switch (optname) {
1473	case XDP_RX_RING:
1474	case XDP_TX_RING:
1475	{
1476	struct xsk_queue **q;
1477	int entries;
1478
1479	if (optlen < sizeof(entries))
1480	return -EINVAL;
1481	if (copy_from_sockptr(dst: &entries, src: optval, size: sizeof(entries)))
1482	return -EFAULT;
1483
1484	mutex_lock(&xs->mutex);
1485	if (xs->state != XSK_READY) {
1486	mutex_unlock(lock: &xs->mutex);
1487	return -EBUSY;
1488	}
1489	q = (optname == XDP_TX_RING) ? &xs->tx : &xs->rx;
1490	err = xsk_init_queue(entries, queue: q, umem_queue: false);
1491	if (!err && optname == XDP_TX_RING)
1492	/* Tx needs to be explicitly woken up the first time */
1493	xs->tx->ring->flags |= XDP_RING_NEED_WAKEUP;
1494	mutex_unlock(lock: &xs->mutex);
1495	return err;
1496	}
1497	case XDP_UMEM_REG:
1498	{
1499	size_t mr_size = sizeof(struct xdp_umem_reg);
1500	struct xdp_umem_reg mr = {};
1501	struct xdp_umem *umem;
1502
1503	if (optlen < sizeof(struct xdp_umem_reg_v1))
1504	return -EINVAL;
1505	else if (optlen < sizeof(mr))
1506	mr_size = sizeof(struct xdp_umem_reg_v1);
1507
1508	BUILD_BUG_ON(sizeof(struct xdp_umem_reg_v1) >= sizeof(struct xdp_umem_reg));
1509
1510	/* Make sure the last field of the struct doesn't have
1511	* uninitialized padding. All padding has to be explicit
1512	* and has to be set to zero by the userspace to make
1513	* struct xdp_umem_reg extensible in the future.
1514	*/
1515	BUILD_BUG_ON(offsetof(struct xdp_umem_reg, tx_metadata_len) +
1516	sizeof_field(struct xdp_umem_reg, tx_metadata_len) !=
1517	sizeof(struct xdp_umem_reg));
1518
1519	if (copy_from_sockptr(dst: &mr, src: optval, size: mr_size))
1520	return -EFAULT;
1521
1522	mutex_lock(&xs->mutex);
1523	if (xs->state != XSK_READY || xs->umem) {
1524	mutex_unlock(lock: &xs->mutex);
1525	return -EBUSY;
1526	}
1527
1528	umem = xdp_umem_create(mr: &mr);
1529	if (IS_ERR(ptr: umem)) {
1530	mutex_unlock(lock: &xs->mutex);
1531	return PTR_ERR(ptr: umem);
1532	}
1533
1534	/* Make sure umem is ready before it can be seen by others */
1535	smp_wmb();
1536	WRITE_ONCE(xs->umem, umem);
1537	mutex_unlock(lock: &xs->mutex);
1538	return 0;
1539	}
1540	case XDP_UMEM_FILL_RING:
1541	case XDP_UMEM_COMPLETION_RING:
1542	{
1543	struct xsk_queue **q;
1544	int entries;
1545
1546	if (optlen < sizeof(entries))
1547	return -EINVAL;
1548	if (copy_from_sockptr(dst: &entries, src: optval, size: sizeof(entries)))
1549	return -EFAULT;
1550
1551	mutex_lock(&xs->mutex);
1552	if (xs->state != XSK_READY) {
1553	mutex_unlock(lock: &xs->mutex);
1554	return -EBUSY;
1555	}
1556
1557	q = (optname == XDP_UMEM_FILL_RING) ? &xs->fq_tmp :
1558	&xs->cq_tmp;
1559	err = xsk_init_queue(entries, queue: q, umem_queue: true);
1560	mutex_unlock(lock: &xs->mutex);
1561	return err;
1562	}
1563	case XDP_MAX_TX_SKB_BUDGET:
1564	{
1565	unsigned int budget;
1566
1567	if (optlen != sizeof(budget))
1568	return -EINVAL;
1569	if (copy_from_sockptr(dst: &budget, src: optval, size: sizeof(budget)))
1570	return -EFAULT;
1571	if (!xs->tx ||
1572	budget < TX_BATCH_SIZE || budget > xs->tx->nentries)
1573	return -EACCES;
1574
1575	WRITE_ONCE(xs->max_tx_budget, budget);
1576	return 0;
1577	}
1578	default:
1579	break;
1580	}
1581
1582	return -ENOPROTOOPT;
1583	}
1584
1585	static void xsk_enter_rxtx_offsets(struct xdp_ring_offset_v1 *ring)
1586	{
1587	ring->producer = offsetof(struct xdp_rxtx_ring, ptrs.producer);
1588	ring->consumer = offsetof(struct xdp_rxtx_ring, ptrs.consumer);
1589	ring->desc = offsetof(struct xdp_rxtx_ring, desc);
1590	}
1591
1592	static void xsk_enter_umem_offsets(struct xdp_ring_offset_v1 *ring)
1593	{
1594	ring->producer = offsetof(struct xdp_umem_ring, ptrs.producer);
1595	ring->consumer = offsetof(struct xdp_umem_ring, ptrs.consumer);
1596	ring->desc = offsetof(struct xdp_umem_ring, desc);
1597	}
1598
1599	struct xdp_statistics_v1 {
1600	__u64 rx_dropped;
1601	__u64 rx_invalid_descs;
1602	__u64 tx_invalid_descs;
1603	};
1604
1605	static int xsk_getsockopt(struct socket *sock, int level, int optname,
1606	char __user *optval, int __user *optlen)
1607	{
1608	struct sock *sk = sock->sk;
1609	struct xdp_sock *xs = xdp_sk(sk);
1610	int len;
1611
1612	if (level != SOL_XDP)
1613	return -ENOPROTOOPT;
1614
1615	if (get_user(len, optlen))
1616	return -EFAULT;
1617	if (len < 0)
1618	return -EINVAL;
1619
1620	switch (optname) {
1621	case XDP_STATISTICS:
1622	{
1623	struct xdp_statistics stats = {};
1624	bool extra_stats = true;
1625	size_t stats_size;
1626
1627	if (len < sizeof(struct xdp_statistics_v1)) {
1628	return -EINVAL;
1629	} else if (len < sizeof(stats)) {
1630	extra_stats = false;
1631	stats_size = sizeof(struct xdp_statistics_v1);
1632	} else {
1633	stats_size = sizeof(stats);
1634	}
1635
1636	mutex_lock(&xs->mutex);
1637	stats.rx_dropped = xs->rx_dropped;
1638	if (extra_stats) {
1639	stats.rx_ring_full = xs->rx_queue_full;
1640	stats.rx_fill_ring_empty_descs =
1641	xs->pool ? xskq_nb_queue_empty_descs(q: xs->pool->fq) : 0;
1642	stats.tx_ring_empty_descs = xskq_nb_queue_empty_descs(q: xs->tx);
1643	} else {
1644	stats.rx_dropped += xs->rx_queue_full;
1645	}
1646	stats.rx_invalid_descs = xskq_nb_invalid_descs(q: xs->rx);
1647	stats.tx_invalid_descs = xskq_nb_invalid_descs(q: xs->tx);
1648	mutex_unlock(lock: &xs->mutex);
1649
1650	if (copy_to_user(to: optval, from: &stats, n: stats_size))
1651	return -EFAULT;
1652	if (put_user(stats_size, optlen))
1653	return -EFAULT;
1654
1655	return 0;
1656	}
1657	case XDP_MMAP_OFFSETS:
1658	{
1659	struct xdp_mmap_offsets off;
1660	struct xdp_mmap_offsets_v1 off_v1;
1661	bool flags_supported = true;
1662	void *to_copy;
1663
1664	if (len < sizeof(off_v1))
1665	return -EINVAL;
1666	else if (len < sizeof(off))
1667	flags_supported = false;
1668
1669	if (flags_supported) {
1670	/* xdp_ring_offset is identical to xdp_ring_offset_v1
1671	* except for the flags field added to the end.
1672	*/
1673	xsk_enter_rxtx_offsets(ring: (struct xdp_ring_offset_v1 *)
1674	&off.rx);
1675	xsk_enter_rxtx_offsets(ring: (struct xdp_ring_offset_v1 *)
1676	&off.tx);
1677	xsk_enter_umem_offsets(ring: (struct xdp_ring_offset_v1 *)
1678	&off.fr);
1679	xsk_enter_umem_offsets(ring: (struct xdp_ring_offset_v1 *)
1680	&off.cr);
1681	off.rx.flags = offsetof(struct xdp_rxtx_ring,
1682	ptrs.flags);
1683	off.tx.flags = offsetof(struct xdp_rxtx_ring,
1684	ptrs.flags);
1685	off.fr.flags = offsetof(struct xdp_umem_ring,
1686	ptrs.flags);
1687	off.cr.flags = offsetof(struct xdp_umem_ring,
1688	ptrs.flags);
1689
1690	len = sizeof(off);
1691	to_copy = &off;
1692	} else {
1693	xsk_enter_rxtx_offsets(ring: &off_v1.rx);
1694	xsk_enter_rxtx_offsets(ring: &off_v1.tx);
1695	xsk_enter_umem_offsets(ring: &off_v1.fr);
1696	xsk_enter_umem_offsets(ring: &off_v1.cr);
1697
1698	len = sizeof(off_v1);
1699	to_copy = &off_v1;
1700	}
1701
1702	if (copy_to_user(to: optval, from: to_copy, n: len))
1703	return -EFAULT;
1704	if (put_user(len, optlen))
1705	return -EFAULT;
1706
1707	return 0;
1708	}
1709	case XDP_OPTIONS:
1710	{
1711	struct xdp_options opts = {};
1712
1713	if (len < sizeof(opts))
1714	return -EINVAL;
1715
1716	mutex_lock(&xs->mutex);
1717	if (xs->zc)
1718	opts.flags |= XDP_OPTIONS_ZEROCOPY;
1719	mutex_unlock(lock: &xs->mutex);
1720
1721	len = sizeof(opts);
1722	if (copy_to_user(to: optval, from: &opts, n: len))
1723	return -EFAULT;
1724	if (put_user(len, optlen))
1725	return -EFAULT;
1726
1727	return 0;
1728	}
1729	default:
1730	break;
1731	}
1732
1733	return -EOPNOTSUPP;
1734	}
1735
1736	static int xsk_mmap(struct file *file, struct socket *sock,
1737	struct vm_area_struct *vma)
1738	{
1739	loff_t offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT;
1740	unsigned long size = vma->vm_end - vma->vm_start;
1741	struct xdp_sock *xs = xdp_sk(sk: sock->sk);
1742	int state = READ_ONCE(xs->state);
1743	struct xsk_queue *q = NULL;
1744
1745	if (state != XSK_READY && state != XSK_BOUND)
1746	return -EBUSY;
1747
1748	if (offset == XDP_PGOFF_RX_RING) {
1749	q = READ_ONCE(xs->rx);
1750	} else if (offset == XDP_PGOFF_TX_RING) {
1751	q = READ_ONCE(xs->tx);
1752	} else {
1753	/* Matches the smp_wmb() in XDP_UMEM_REG */
1754	smp_rmb();
1755	if (offset == XDP_UMEM_PGOFF_FILL_RING)
1756	q = state == XSK_READY ? READ_ONCE(xs->fq_tmp) :
1757	READ_ONCE(xs->pool->fq);
1758	else if (offset == XDP_UMEM_PGOFF_COMPLETION_RING)
1759	q = state == XSK_READY ? READ_ONCE(xs->cq_tmp) :
1760	READ_ONCE(xs->pool->cq);
1761	}
1762
1763	if (!q)
1764	return -EINVAL;
1765
1766	/* Matches the smp_wmb() in xsk_init_queue */
1767	smp_rmb();
1768	if (size > q->ring_vmalloc_size)
1769	return -EINVAL;
1770
1771	return remap_vmalloc_range(vma, addr: q->ring, pgoff: 0);
1772	}
1773
1774	static int xsk_notifier(struct notifier_block *this,
1775	unsigned long msg, void *ptr)
1776	{
1777	struct net_device *dev = netdev_notifier_info_to_dev(info: ptr);
1778	struct net *net = dev_net(dev);
1779	struct sock *sk;
1780
1781	switch (msg) {
1782	case NETDEV_UNREGISTER:
1783	mutex_lock(&net->xdp.lock);
1784	sk_for_each(sk, &net->xdp.list) {
1785	struct xdp_sock *xs = xdp_sk(sk);
1786
1787	mutex_lock(&xs->mutex);
1788	if (xs->dev == dev) {
1789	sk->sk_err = ENETDOWN;
1790	if (!sock_flag(sk, flag: SOCK_DEAD))
1791	sk_error_report(sk);
1792
1793	xsk_unbind_dev(xs);
1794
1795	/* Clear device references. */
1796	xp_clear_dev(pool: xs->pool);
1797	}
1798	mutex_unlock(lock: &xs->mutex);
1799	}
1800	mutex_unlock(lock: &net->xdp.lock);
1801	break;
1802	}
1803	return NOTIFY_DONE;
1804	}
1805
1806	static struct proto xsk_proto = {
1807	.name = "XDP",
1808	.owner = THIS_MODULE,
1809	.obj_size = sizeof(struct xdp_sock),
1810	};
1811
1812	static const struct proto_ops xsk_proto_ops = {
1813	.family = PF_XDP,
1814	.owner = THIS_MODULE,
1815	.release = xsk_release,
1816	.bind = xsk_bind,
1817	.connect = sock_no_connect,
1818	.socketpair = sock_no_socketpair,
1819	.accept = sock_no_accept,
1820	.getname = sock_no_getname,
1821	.poll = xsk_poll,
1822	.ioctl = sock_no_ioctl,
1823	.listen = sock_no_listen,
1824	.shutdown = sock_no_shutdown,
1825	.setsockopt = xsk_setsockopt,
1826	.getsockopt = xsk_getsockopt,
1827	.sendmsg = xsk_sendmsg,
1828	.recvmsg = xsk_recvmsg,
1829	.mmap = xsk_mmap,
1830	};
1831
1832	static void xsk_destruct(struct sock *sk)
1833	{
1834	struct xdp_sock *xs = xdp_sk(sk);
1835
1836	if (!sock_flag(sk, flag: SOCK_DEAD))
1837	return;
1838
1839	if (!xp_put_pool(pool: xs->pool))
1840	xdp_put_umem(umem: xs->umem, defer_cleanup: !xs->pool);
1841	}
1842
1843	static int xsk_create(struct net *net, struct socket *sock, int protocol,
1844	int kern)
1845	{
1846	struct xdp_sock *xs;
1847	struct sock *sk;
1848
1849	if (!ns_capable(ns: net->user_ns, CAP_NET_RAW))
1850	return -EPERM;
1851	if (sock->type != SOCK_RAW)
1852	return -ESOCKTNOSUPPORT;
1853
1854	if (protocol)
1855	return -EPROTONOSUPPORT;
1856
1857	sock->state = SS_UNCONNECTED;
1858
1859	sk = sk_alloc(net, PF_XDP, GFP_KERNEL, prot: &xsk_proto, kern);
1860	if (!sk)
1861	return -ENOBUFS;
1862
1863	sock->ops = &xsk_proto_ops;
1864
1865	sock_init_data(sock, sk);
1866
1867	sk->sk_family = PF_XDP;
1868
1869	sk->sk_destruct = xsk_destruct;
1870
1871	sock_set_flag(sk, flag: SOCK_RCU_FREE);
1872
1873	xs = xdp_sk(sk);
1874	xs->state = XSK_READY;
1875	xs->max_tx_budget = TX_BATCH_SIZE;
1876	mutex_init(&xs->mutex);
1877
1878	INIT_LIST_HEAD(list: &xs->map_list);
1879	spin_lock_init(&xs->map_list_lock);
1880
1881	mutex_lock(&net->xdp.lock);
1882	sk_add_node_rcu(sk, list: &net->xdp.list);
1883	mutex_unlock(lock: &net->xdp.lock);
1884
1885	sock_prot_inuse_add(net, prot: &xsk_proto, val: 1);
1886
1887	return 0;
1888	}
1889
1890	static const struct net_proto_family xsk_family_ops = {
1891	.family = PF_XDP,
1892	.create = xsk_create,
1893	.owner = THIS_MODULE,
1894	};
1895
1896	static struct notifier_block xsk_netdev_notifier = {
1897	.notifier_call = xsk_notifier,
1898	};
1899
1900	static int __net_init xsk_net_init(struct net *net)
1901	{
1902	mutex_init(&net->xdp.lock);
1903	INIT_HLIST_HEAD(&net->xdp.list);
1904	return 0;
1905	}
1906
1907	static void __net_exit xsk_net_exit(struct net *net)
1908	{
1909	WARN_ON_ONCE(!hlist_empty(&net->xdp.list));
1910	}
1911
1912	static struct pernet_operations xsk_net_ops = {
1913	.init = xsk_net_init,
1914	.exit = xsk_net_exit,
1915	};
1916
1917	static int __init xsk_init(void)
1918	{
1919	int err;
1920
1921	err = proto_register(prot: &xsk_proto, alloc_slab: 0 /* no slab */);
1922	if (err)
1923	goto out;
1924
1925	err = sock_register(fam: &xsk_family_ops);
1926	if (err)
1927	goto out_proto;
1928
1929	err = register_pernet_subsys(&xsk_net_ops);
1930	if (err)
1931	goto out_sk;
1932
1933	err = register_netdevice_notifier(nb: &xsk_netdev_notifier);
1934	if (err)
1935	goto out_pernet;
1936
1937	xsk_tx_generic_cache = kmem_cache_create("xsk_generic_xmit_cache",
1938	sizeof(struct xsk_addrs),
1939	0, SLAB_HWCACHE_ALIGN, NULL);
1940	if (!xsk_tx_generic_cache) {
1941	err = -ENOMEM;
1942	goto out_unreg_notif;
1943	}
1944
1945	return 0;
1946
1947	out_unreg_notif:
1948	unregister_netdevice_notifier(nb: &xsk_netdev_notifier);
1949	out_pernet:
1950	unregister_pernet_subsys(&xsk_net_ops);
1951	out_sk:
1952	sock_unregister(PF_XDP);
1953	out_proto:
1954	proto_unregister(prot: &xsk_proto);
1955	out:
1956	return err;
1957	}
1958
1959	fs_initcall(xsk_init);
1960