dev.c source code [linux/net/core/dev.c]

1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* NET3 Protocol independent device support routines.
4	*
5	* Derived from the non IP parts of dev.c 1.0.19
6	* Authors: Ross Biro
7	* Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
8	* Mark Evans, <evansmp@uhura.aston.ac.uk>
9	*
10	* Additional Authors:
11	* Florian la Roche <rzsfl@rz.uni-sb.de>
12	* Alan Cox <gw4pts@gw4pts.ampr.org>
13	* David Hinds <dahinds@users.sourceforge.net>
14	* Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
15	* Adam Sulmicki <adam@cfar.umd.edu>
16	* Pekka Riikonen <priikone@poesidon.pspt.fi>
17	*
18	* Changes:
19	* D.J. Barrow : Fixed bug where dev->refcnt gets set
20	* to 2 if register_netdev gets called
21	* before net_dev_init & also removed a
22	* few lines of code in the process.
23	* Alan Cox : device private ioctl copies fields back.
24	* Alan Cox : Transmit queue code does relevant
25	* stunts to keep the queue safe.
26	* Alan Cox : Fixed double lock.
27	* Alan Cox : Fixed promisc NULL pointer trap
28	* ???????? : Support the full private ioctl range
29	* Alan Cox : Moved ioctl permission check into
30	* drivers
31	* Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
32	* Alan Cox : 100 backlog just doesn't cut it when
33	* you start doing multicast video 8)
34	* Alan Cox : Rewrote net_bh and list manager.
35	* Alan Cox : Fix ETH_P_ALL echoback lengths.
36	* Alan Cox : Took out transmit every packet pass
37	* Saved a few bytes in the ioctl handler
38	* Alan Cox : Network driver sets packet type before
39	* calling netif_rx. Saves a function
40	* call a packet.
41	* Alan Cox : Hashed net_bh()
42	* Richard Kooijman: Timestamp fixes.
43	* Alan Cox : Wrong field in SIOCGIFDSTADDR
44	* Alan Cox : Device lock protection.
45	* Alan Cox : Fixed nasty side effect of device close
46	* changes.
47	* Rudi Cilibrasi : Pass the right thing to
48	* set_mac_address()
49	* Dave Miller : 32bit quantity for the device lock to
50	* make it work out on a Sparc.
51	* Bjorn Ekwall : Added KERNELD hack.
52	* Alan Cox : Cleaned up the backlog initialise.
53	* Craig Metz : SIOCGIFCONF fix if space for under
54	* 1 device.
55	* Thomas Bogendoerfer : Return ENODEV for dev_open, if there
56	* is no device open function.
57	* Andi Kleen : Fix error reporting for SIOCGIFCONF
58	* Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
59	* Cyrus Durgin : Cleaned for KMOD
60	* Adam Sulmicki : Bug Fix : Network Device Unload
61	* A network device unload needs to purge
62	* the backlog queue.
63	* Paul Rusty Russell : SIOCSIFNAME
64	* Pekka Riikonen : Netdev boot-time settings code
65	* Andrew Morton : Make unregister_netdevice wait
66	* indefinitely on dev->refcnt
67	* J Hadi Salim : - Backlog queue sampling
68	* - netif_rx() feedback
69	*/
70
71	#include <linux/uaccess.h>
72	#include <linux/bitops.h>
73	#include <linux/capability.h>
74	#include <linux/cpu.h>
75	#include <linux/types.h>
76	#include <linux/kernel.h>
77	#include <linux/hash.h>
78	#include <linux/slab.h>
79	#include <linux/sched.h>
80	#include <linux/sched/mm.h>
81	#include <linux/mutex.h>
82	#include <linux/rwsem.h>
83	#include <linux/string.h>
84	#include <linux/mm.h>
85	#include <linux/socket.h>
86	#include <linux/sockios.h>
87	#include <linux/errno.h>
88	#include <linux/interrupt.h>
89	#include <linux/if_ether.h>
90	#include <linux/netdevice.h>
91	#include <linux/etherdevice.h>
92	#include <linux/ethtool.h>
93	#include <linux/skbuff.h>
94	#include <linux/kthread.h>
95	#include <linux/bpf.h>
96	#include <linux/bpf_trace.h>
97	#include <net/net_namespace.h>
98	#include <net/sock.h>
99	#include <net/busy_poll.h>
100	#include <linux/rtnetlink.h>
101	#include <linux/stat.h>
102	#include <net/dsa.h>
103	#include <net/dst.h>
104	#include <net/dst_metadata.h>
105	#include <net/gro.h>
106	#include <net/pkt_sched.h>
107	#include <net/pkt_cls.h>
108	#include <net/checksum.h>
109	#include <net/xfrm.h>
110	#include <linux/highmem.h>
111	#include <linux/init.h>
112	#include <linux/module.h>
113	#include <linux/netpoll.h>
114	#include <linux/rcupdate.h>
115	#include <linux/delay.h>
116	#include <net/iw_handler.h>
117	#include <asm/current.h>
118	#include <linux/audit.h>
119	#include <linux/dmaengine.h>
120	#include <linux/err.h>
121	#include <linux/ctype.h>
122	#include <linux/if_arp.h>
123	#include <linux/if_vlan.h>
124	#include <linux/ip.h>
125	#include <net/ip.h>
126	#include <net/mpls.h>
127	#include <linux/ipv6.h>
128	#include <linux/in.h>
129	#include <linux/jhash.h>
130	#include <linux/random.h>
131	#include <trace/events/napi.h>
132	#include <trace/events/net.h>
133	#include <trace/events/skb.h>
134	#include <trace/events/qdisc.h>
135	#include <linux/inetdevice.h>
136	#include <linux/cpu_rmap.h>
137	#include <linux/static_key.h>
138	#include <linux/hashtable.h>
139	#include <linux/vmalloc.h>
140	#include <linux/if_macvlan.h>
141	#include <linux/errqueue.h>
142	#include <linux/hrtimer.h>
143	#include <linux/netfilter_netdev.h>
144	#include <linux/crash_dump.h>
145	#include <linux/sctp.h>
146	#include <net/udp_tunnel.h>
147	#include <linux/net_namespace.h>
148	#include <linux/indirect_call_wrapper.h>
149	#include <net/devlink.h>
150	#include <linux/pm_runtime.h>
151	#include <linux/prandom.h>
152	#include <linux/once_lite.h>
153
154	#include "dev.h"
155	#include "net-sysfs.h"
156
157
158	static DEFINE_SPINLOCK(ptype_lock);
159	struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
160	struct list_head ptype_all __read_mostly; / Taps /
161
162	static int netif_rx_internal(struct sk_buff *skb);
163	static int call_netdevice_notifiers_info(unsigned long val,
164	struct netdev_notifier_info *info);
165	static int call_netdevice_notifiers_extack(unsigned long val,
166	struct net_device *dev,
167	struct netlink_ext_ack *extack);
168	static struct napi_struct napi_by_id(unsigned* int napi_id);
169
170	/*
171	* The @dev_base_head list is protected by @dev_base_lock and the rtnl
172	* semaphore.
173	*
174	* Pure readers hold dev_base_lock for reading, or rcu_read_lock()
175	*
176	* Writers must hold the rtnl semaphore while they loop through the
177	* dev_base_head list, and hold dev_base_lock for writing when they do the
178	* actual updates. This allows pure readers to access the list even
179	* while a writer is preparing to update it.
180	*
181	* To put it another way, dev_base_lock is held for writing only to
182	* protect against pure readers; the rtnl semaphore provides the
183	* protection against other writers.
184	*
185	* See, for example usages, register_netdevice() and
186	* unregister_netdevice(), which must be called with the rtnl
187	* semaphore held.
188	*/
189	DEFINE_RWLOCK(dev_base_lock);
190	EXPORT_SYMBOL(dev_base_lock);
191
192	static DEFINE_MUTEX(ifalias_mutex);
193
194	/ protects napi_hash addition/deletion and napi_gen_id /
195	static DEFINE_SPINLOCK(napi_hash_lock);
196
197	static unsigned int napi_gen_id = NR_CPUS;
198	static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, `8`);
199
200	static DECLARE_RWSEM(devnet_rename_sem);
201
202	static inline void dev_base_seq_inc(struct net *net)
203	{
204	while (++net->dev_base_seq == `0`)
205	;
206	}
207
208	static inline struct hlist_head dev_name_hash(struct* net net, const* char *name)
209	{
210	unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ));
211
212	return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
213	}
214
215	static inline struct hlist_head dev_index_hash(struct* net net, int* ifindex)
216	{
217	return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - `1`)];
218	}
219
220	static inline void rps_lock_irqsave(struct softnet_data *sd,
221	unsigned long *flags)
222	{
223	if (IS_ENABLED(CONFIG_RPS))
224	spin_lock_irqsave(&sd->input_pkt_queue.lock, *flags);
225	else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
226	local_irq_save(*flags);
227	}
228
229	static inline void rps_lock_irq_disable(struct softnet_data *sd)
230	{
231	if (IS_ENABLED(CONFIG_RPS))
232	spin_lock_irq(&sd->input_pkt_queue.lock);
233	else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
234	local_irq_disable();
235	}
236
237	static inline void rps_unlock_irq_restore(struct softnet_data *sd,
238	unsigned long *flags)
239	{
240	if (IS_ENABLED(CONFIG_RPS))
241	spin_unlock_irqrestore(&sd->input_pkt_queue.lock, *flags);
242	else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
243	local_irq_restore(*flags);
244	}
245
246	static inline void rps_unlock_irq_enable(struct softnet_data *sd)
247	{
248	if (IS_ENABLED(CONFIG_RPS))
249	spin_unlock_irq(&sd->input_pkt_queue.lock);
250	else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
251	local_irq_enable();
252	}
253
254	static struct netdev_name_node netdev_name_node_alloc(struct* net_device *dev,
255	const char *name)
256	{
257	struct netdev_name_node *name_node;
258
259	name_node = kmalloc(sizeof(*name_node), GFP_KERNEL);
260	if (!name_node)
261	return NULL;
262	INIT_HLIST_NODE(&name_node->hlist);
263	name_node->dev = dev;
264	name_node->name = name;
265	return name_node;
266	}
267
268	static struct netdev_name_node *
269	netdev_name_node_head_alloc(struct net_device *dev)
270	{
271	struct netdev_name_node *name_node;
272
273	name_node = netdev_name_node_alloc(dev, dev->name);
274	if (!name_node)
275	return NULL;
276	INIT_LIST_HEAD(&name_node->list);
277	return name_node;
278	}
279
280	static void netdev_name_node_free(struct netdev_name_node *name_node)
281	{
282	kfree(name_node);
283	}
284
285	static void netdev_name_node_add(struct net *net,
286	struct netdev_name_node *name_node)
287	{
288	hlist_add_head_rcu(&name_node->hlist,
289	dev_name_hash(net, name_node->name));
290	}
291
292	static void netdev_name_node_del(struct netdev_name_node *name_node)
293	{
294	hlist_del_rcu(&name_node->hlist);
295	}
296
297	static struct netdev_name_node netdev_name_node_lookup(struct* net *net,
298	const char *name)
299	{
300	struct hlist_head *head = dev_name_hash(net, name);
301	struct netdev_name_node *name_node;
302
303	hlist_for_each_entry(name_node, head, hlist)
304	if (!strcmp(name_node->name, name))
305	return name_node;
306	return NULL;
307	}
308
309	static struct netdev_name_node netdev_name_node_lookup_rcu(struct* net *net,
310	const char *name)
311	{
312	struct hlist_head *head = dev_name_hash(net, name);
313	struct netdev_name_node *name_node;
314
315	hlist_for_each_entry_rcu(name_node, head, hlist)
316	if (!strcmp(name_node->name, name))
317	return name_node;
318	return NULL;
319	}
320
321	bool netdev_name_in_use(struct net net, const* char *name)
322	{
323	return netdev_name_node_lookup(net, name);
324	}
325	EXPORT_SYMBOL(netdev_name_in_use);
326
327	int netdev_name_node_alt_create(struct net_device dev, const* char *name)
328	{
329	struct netdev_name_node *name_node;
330	struct net *net = dev_net(dev);
331
332	name_node = netdev_name_node_lookup(net, name);
333	if (name_node)
334	return -EEXIST;
335	name_node = netdev_name_node_alloc(dev, name);
336	if (!name_node)
337	return -ENOMEM;
338	netdev_name_node_add(net, name_node);
339	/ The node that holds dev->name acts as a head of per-device list. /
340	list_add_tail(&name_node->list, &dev->name_node->list);
341
342	return `0`;
343	}
344
345	static void __netdev_name_node_alt_destroy(struct netdev_name_node *name_node)
346	{
347	list_del(&name_node->list);
348	netdev_name_node_del(name_node);
349	kfree(name_node->name);
350	netdev_name_node_free(name_node);
351	}
352
353	int netdev_name_node_alt_destroy(struct net_device dev, const* char *name)
354	{
355	struct netdev_name_node *name_node;
356	struct net *net = dev_net(dev);
357
358	name_node = netdev_name_node_lookup(net, name);
359	if (!name_node)
360	return -ENOENT;
361	/ lookup might have found our primary name or a name belonging*
362	* to another device.
363	*/
364	if (name_node == dev->name_node \|\| name_node->dev != dev)
365	return -EINVAL;
366
367	__netdev_name_node_alt_destroy(name_node);
368
369	return `0`;
370	}
371
372	static void netdev_name_node_alt_flush(struct net_device *dev)
373	{
374	struct netdev_name_node name_node, tmp;
375
376	list_for_each_entry_safe(name_node, tmp, &dev->name_node->list, list)
377	__netdev_name_node_alt_destroy(name_node);
378	}
379
380	/ Device list insertion /
381	static void list_netdevice(struct net_device *dev)
382	{
383	struct net *net = dev_net(dev);
384
385	ASSERT_RTNL();
386
387	write_lock(&dev_base_lock);
388	list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
389	netdev_name_node_add(net, dev->name_node);
390	hlist_add_head_rcu(&dev->index_hlist,
391	dev_index_hash(net, dev->ifindex));
392	write_unlock(&dev_base_lock);
393
394	dev_base_seq_inc(net);
395	}
396
397	/ Device list removal*
398	* caller must respect a RCU grace period before freeing/reusing dev
399	*/
400	static void unlist_netdevice(struct net_device *dev, bool lock)
401	{
402	ASSERT_RTNL();
403
404	/ Unlink dev from the device chain /
405	if (lock)
406	write_lock(&dev_base_lock);
407	list_del_rcu(&dev->dev_list);
408	netdev_name_node_del(dev->name_node);
409	hlist_del_rcu(&dev->index_hlist);
410	if (lock)
411	write_unlock(&dev_base_lock);
412
413	dev_base_seq_inc(dev_net(dev));
414	}
415
416	/*
417	* Our notifier list
418	*/
419
420	static RAW_NOTIFIER_HEAD(netdev_chain);
421
422	/*
423	* Device drivers call our routines to queue packets here. We empty the
424	* queue in the local softnet handler.
425	*/
426
427	DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
428	EXPORT_PER_CPU_SYMBOL(softnet_data);
429
430	#ifdef CONFIG_LOCKDEP
431	/*
432	* register_netdevice() inits txq->_xmit_lock and sets lockdep class
433	* according to dev->type
434	*/
435	static const unsigned short netdev_lock_type[] = {
436	ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
437	ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
438	ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
439	ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
440	ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
441	ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
442	ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
443	ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
444	ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
445	ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
446	ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
447	ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
448	ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
449	ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
450	ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
451
452	static const char *const netdev_lock_name[] = {
453	"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
454	"_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
455	"_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
456	"_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
457	"_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
458	"_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
459	"_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
460	"_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
461	"_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
462	"_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
463	"_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
464	"_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
465	"_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
466	"_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
467	"_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
468
469	static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
470	static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
471
472	static inline unsigned short netdev_lock_pos(unsigned short dev_type)
473	{
474	int i;
475
476	for (i = `0`; i < ARRAY_SIZE(netdev_lock_type); i++)
477	if (netdev_lock_type[i] == dev_type)
478	return i;
479	/ the last key is used by default /
480	return ARRAY_SIZE(netdev_lock_type) - `1`;
481	}
482
483	static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
484	unsigned short dev_type)
485	{
486	int i;
487
488	i = netdev_lock_pos(dev_type);
489	lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
490	netdev_lock_name[i]);
491	}
492
493	static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
494	{
495	int i;
496
497	i = netdev_lock_pos(dev->type);
498	lockdep_set_class_and_name(&dev->addr_list_lock,
499	&netdev_addr_lock_key[i],
500	netdev_lock_name[i]);
501	}
502	#else
503	static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
504	unsigned short dev_type)
505	{
506	}
507
508	static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
509	{
510	}
511	#endif
512
513	/*******************************************************************************
514	*
515	* Protocol management and registration routines
516	*
517	*******************************************************************************/
518
519
520	/*
521	* Add a protocol ID to the list. Now that the input handler is
522	* smarter we can dispense with all the messy stuff that used to be
523	* here.
524	*
525	* BEWARE!!! Protocol handlers, mangling input packets,
526	* MUST BE last in hash buckets and checking protocol handlers
527	* MUST start from promiscuous ptype_all chain in net_bh.
528	* It is true now, do not change it.
529	* Explanation follows: if protocol handler, mangling packet, will
530	* be the first on list, it is not able to sense, that packet
531	* is cloned and should be copied-on-write, so that it will
532	* change it and subsequent readers will get broken packet.
533	* --ANK (980803)
534	*/
535
536	static inline struct list_head ptype_head(const* struct packet_type *pt)
537	{
538	if (pt->type == htons(ETH_P_ALL))
539	return pt->dev ? &pt->dev->ptype_all : &ptype_all;
540	else
541	return pt->dev ? &pt->dev->ptype_specific :
542	&ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
543	}
544
545	/**
546	* dev_add_pack - add packet handler
547	* @pt: packet type declaration
548	*
549	* Add a protocol handler to the networking stack. The passed &packet_type
550	* is linked into kernel lists and may not be freed until it has been
551	* removed from the kernel lists.
552	*
553	* This call does not sleep therefore it can not
554	* guarantee all CPU's that are in middle of receiving packets
555	* will see the new packet type (until the next received packet).
556	*/
557
558	void dev_add_pack(struct packet_type *pt)
559	{
560	struct list_head *head = ptype_head(pt);
561
562	spin_lock(&ptype_lock);
563	list_add_rcu(&pt->list, head);
564	spin_unlock(&ptype_lock);
565	}
566	EXPORT_SYMBOL(dev_add_pack);
567
568	/**
569	* __dev_remove_pack - remove packet handler
570	* @pt: packet type declaration
571	*
572	* Remove a protocol handler that was previously added to the kernel
573	* protocol handlers by dev_add_pack(). The passed &packet_type is removed
574	* from the kernel lists and can be freed or reused once this function
575	* returns.
576	*
577	* The packet type might still be in use by receivers
578	* and must not be freed until after all the CPU's have gone
579	* through a quiescent state.
580	*/
581	void __dev_remove_pack(struct packet_type *pt)
582	{
583	struct list_head *head = ptype_head(pt);
584	struct packet_type *pt1;
585
586	spin_lock(&ptype_lock);
587
588	list_for_each_entry(pt1, head, list) {
589	if (pt == pt1) {
590	list_del_rcu(&pt->list);
591	goto out;
592	}
593	}
594
595	pr_warn("dev_remove_pack: %p not found\n", pt);
596	out:
597	spin_unlock(&ptype_lock);
598	}
599	EXPORT_SYMBOL(__dev_remove_pack);
600
601	/**
602	* dev_remove_pack - remove packet handler
603	* @pt: packet type declaration
604	*
605	* Remove a protocol handler that was previously added to the kernel
606	* protocol handlers by dev_add_pack(). The passed &packet_type is removed
607	* from the kernel lists and can be freed or reused once this function
608	* returns.
609	*
610	* This call sleeps to guarantee that no CPU is looking at the packet
611	* type after return.
612	*/
613	void dev_remove_pack(struct packet_type *pt)
614	{
615	__dev_remove_pack(pt);
616
617	synchronize_net();
618	}
619	EXPORT_SYMBOL(dev_remove_pack);
620
621
622	/*******************************************************************************
623	*
624	* Device Interface Subroutines
625	*
626	*******************************************************************************/
627
628	/**
629	* dev_get_iflink - get 'iflink' value of a interface
630	* @dev: targeted interface
631	*
632	* Indicates the ifindex the interface is linked to.
633	* Physical interfaces have the same 'ifindex' and 'iflink' values.
634	*/
635
636	int dev_get_iflink(const struct net_device *dev)
637	{
638	if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
639	return dev->netdev_ops->ndo_get_iflink(dev);
640
641	return dev->ifindex;
642	}
643	EXPORT_SYMBOL(dev_get_iflink);
644
645	/**
646	* dev_fill_metadata_dst - Retrieve tunnel egress information.
647	* @dev: targeted interface
648	* @skb: The packet.
649	*
650	* For better visibility of tunnel traffic OVS needs to retrieve
651	* egress tunnel information for a packet. Following API allows
652	* user to get this info.
653	*/
654	int dev_fill_metadata_dst(struct net_device dev, struct* sk_buff *skb)
655	{
656	struct ip_tunnel_info *info;
657
658	if (!dev->netdev_ops \|\| !dev->netdev_ops->ndo_fill_metadata_dst)
659	return -EINVAL;
660
661	info = skb_tunnel_info_unclone(skb);
662	if (!info)
663	return -ENOMEM;
664	if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
665	return -EINVAL;
666
667	return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
668	}
669	EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
670
671	static struct net_device_path dev_fwd_path(struct* net_device_path_stack *stack)
672	{
673	int k = stack->num_paths++;
674
675	if (WARN_ON_ONCE(k >= NET_DEVICE_PATH_STACK_MAX))
676	return NULL;
677
678	return &stack->path[k];
679	}
680
681	int dev_fill_forward_path(const struct net_device dev, const* u8 *daddr,
682	struct net_device_path_stack *stack)
683	{
684	const struct net_device *last_dev;
685	struct net_device_path_ctx ctx = {
686	.dev = dev,
687	};
688	struct net_device_path *path;
689	int ret = `0`;
690
691	memcpy(ctx.daddr, daddr, sizeof(ctx.daddr));
692	stack->num_paths = `0`;
693	while (ctx.dev && ctx.dev->netdev_ops->ndo_fill_forward_path) {
694	last_dev = ctx.dev;
695	path = dev_fwd_path(stack);
696	if (!path)
697	return -`1`;
698
699	memset(path, `0`, sizeof(struct net_device_path));
700	ret = ctx.dev->netdev_ops->ndo_fill_forward_path(&ctx, path);
701	if (ret < `0`)
702	return -`1`;
703
704	if (WARN_ON_ONCE(last_dev == ctx.dev))
705	return -`1`;
706	}
707
708	if (!ctx.dev)
709	return ret;
710
711	path = dev_fwd_path(stack);
712	if (!path)
713	return -`1`;
714	path->type = DEV_PATH_ETHERNET;
715	path->dev = ctx.dev;
716
717	return ret;
718	}
719	EXPORT_SYMBOL_GPL(dev_fill_forward_path);
720
721	/**
722	* __dev_get_by_name - find a device by its name
723	* @net: the applicable net namespace
724	* @name: name to find
725	*
726	* Find an interface by name. Must be called under RTNL semaphore
727	* or @dev_base_lock. If the name is found a pointer to the device
728	* is returned. If the name is not found then %NULL is returned. The
729	* reference counters are not incremented so the caller must be
730	* careful with locks.
731	*/
732
733	struct net_device __dev_get_by_name(struct* net net, const* char *name)
734	{
735	struct netdev_name_node *node_name;
736
737	node_name = netdev_name_node_lookup(net, name);
738	return node_name ? node_name->dev : NULL;
739	}
740	EXPORT_SYMBOL(__dev_get_by_name);
741
742	/**
743	* dev_get_by_name_rcu - find a device by its name
744	* @net: the applicable net namespace
745	* @name: name to find
746	*
747	* Find an interface by name.
748	* If the name is found a pointer to the device is returned.
749	* If the name is not found then %NULL is returned.
750	* The reference counters are not incremented so the caller must be
751	* careful with locks. The caller must hold RCU lock.
752	*/
753
754	struct net_device dev_get_by_name_rcu(struct* net net, const* char *name)
755	{
756	struct netdev_name_node *node_name;
757
758	node_name = netdev_name_node_lookup_rcu(net, name);
759	return node_name ? node_name->dev : NULL;
760	}
761	EXPORT_SYMBOL(dev_get_by_name_rcu);
762
763	/**
764	* dev_get_by_name - find a device by its name
765	* @net: the applicable net namespace
766	* @name: name to find
767	*
768	* Find an interface by name. This can be called from any
769	* context and does its own locking. The returned handle has
770	* the usage count incremented and the caller must use dev_put() to
771	* release it when it is no longer needed. %NULL is returned if no
772	* matching device is found.
773	*/
774
775	struct net_device dev_get_by_name(struct* net net, const* char *name)
776	{
777	struct net_device *dev;
778
779	rcu_read_lock();
780	dev = dev_get_by_name_rcu(net, name);
781	dev_hold(dev);
782	rcu_read_unlock();
783	return dev;
784	}
785	EXPORT_SYMBOL(dev_get_by_name);
786
787	/**
788	* __dev_get_by_index - find a device by its ifindex
789	* @net: the applicable net namespace
790	* @ifindex: index of device
791	*
792	* Search for an interface by index. Returns %NULL if the device
793	* is not found or a pointer to the device. The device has not
794	* had its reference counter increased so the caller must be careful
795	* about locking. The caller must hold either the RTNL semaphore
796	* or @dev_base_lock.
797	*/
798
799	struct net_device __dev_get_by_index(struct* net net, int* ifindex)
800	{
801	struct net_device *dev;
802	struct hlist_head *head = dev_index_hash(net, ifindex);
803
804	hlist_for_each_entry(dev, head, index_hlist)
805	if (dev->ifindex == ifindex)
806	return dev;
807
808	return NULL;
809	}
810	EXPORT_SYMBOL(__dev_get_by_index);
811
812	/**
813	* dev_get_by_index_rcu - find a device by its ifindex
814	* @net: the applicable net namespace
815	* @ifindex: index of device
816	*
817	* Search for an interface by index. Returns %NULL if the device
818	* is not found or a pointer to the device. The device has not
819	* had its reference counter increased so the caller must be careful
820	* about locking. The caller must hold RCU lock.
821	*/
822
823	struct net_device dev_get_by_index_rcu(struct* net net, int* ifindex)
824	{
825	struct net_device *dev;
826	struct hlist_head *head = dev_index_hash(net, ifindex);
827
828	hlist_for_each_entry_rcu(dev, head, index_hlist)
829	if (dev->ifindex == ifindex)
830	return dev;
831
832	return NULL;
833	}
834	EXPORT_SYMBOL(dev_get_by_index_rcu);
835
836
837	/**
838	* dev_get_by_index - find a device by its ifindex
839	* @net: the applicable net namespace
840	* @ifindex: index of device
841	*
842	* Search for an interface by index. Returns NULL if the device
843	* is not found or a pointer to the device. The device returned has
844	* had a reference added and the pointer is safe until the user calls
845	* dev_put to indicate they have finished with it.
846	*/
847
848	struct net_device dev_get_by_index(struct* net net, int* ifindex)
849	{
850	struct net_device *dev;
851
852	rcu_read_lock();
853	dev = dev_get_by_index_rcu(net, ifindex);
854	dev_hold(dev);
855	rcu_read_unlock();
856	return dev;
857	}
858	EXPORT_SYMBOL(dev_get_by_index);
859
860	/**
861	* dev_get_by_napi_id - find a device by napi_id
862	* @napi_id: ID of the NAPI struct
863	*
864	* Search for an interface by NAPI ID. Returns %NULL if the device
865	* is not found or a pointer to the device. The device has not had
866	* its reference counter increased so the caller must be careful
867	* about locking. The caller must hold RCU lock.
868	*/
869
870	struct net_device dev_get_by_napi_id(unsigned* int napi_id)
871	{
872	struct napi_struct *napi;
873
874	WARN_ON_ONCE(!rcu_read_lock_held());
875
876	if (napi_id < MIN_NAPI_ID)
877	return NULL;
878
879	napi = napi_by_id(napi_id);
880
881	return napi ? napi->dev : NULL;
882	}
883	EXPORT_SYMBOL(dev_get_by_napi_id);
884
885	/**
886	* netdev_get_name - get a netdevice name, knowing its ifindex.
887	* @net: network namespace
888	* @name: a pointer to the buffer where the name will be stored.
889	* @ifindex: the ifindex of the interface to get the name from.
890	*/
891	int netdev_get_name(struct net net, char* name, int* ifindex)
892	{
893	struct net_device *dev;
894	int ret;
895
896	down_read(&devnet_rename_sem);
897	rcu_read_lock();
898
899	dev = dev_get_by_index_rcu(net, ifindex);
900	if (!dev) {
901	ret = -ENODEV;
902	goto out;
903	}
904
905	strcpy(name, dev->name);
906
907	ret = `0`;
908	out:
909	rcu_read_unlock();
910	up_read(&devnet_rename_sem);
911	return ret;
912	}
913
914	/**
915	* dev_getbyhwaddr_rcu - find a device by its hardware address
916	* @net: the applicable net namespace
917	* @type: media type of device
918	* @ha: hardware address
919	*
920	* Search for an interface by MAC address. Returns NULL if the device
921	* is not found or a pointer to the device.
922	* The caller must hold RCU or RTNL.
923	* The returned device has not had its ref count increased
924	* and the caller must therefore be careful about locking
925	*
926	*/
927
928	struct net_device dev_getbyhwaddr_rcu(struct* net net, unsigned* short type,
929	const char *ha)
930	{
931	struct net_device *dev;
932
933	for_each_netdev_rcu(net, dev)
934	if (dev->type == type &&
935	!memcmp(dev->dev_addr, ha, dev->addr_len))
936	return dev;
937
938	return NULL;
939	}
940	EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
941
942	struct net_device dev_getfirstbyhwtype(struct* net net, unsigned* short type)
943	{
944	struct net_device dev, ret = NULL;
945
946	rcu_read_lock();
947	for_each_netdev_rcu(net, dev)
948	if (dev->type == type) {
949	dev_hold(dev);
950	ret = dev;
951	break;
952	}
953	rcu_read_unlock();
954	return ret;
955	}
956	EXPORT_SYMBOL(dev_getfirstbyhwtype);
957
958	/**
959	* __dev_get_by_flags - find any device with given flags
960	* @net: the applicable net namespace
961	* @if_flags: IFF_* values
962	* @mask: bitmask of bits in if_flags to check
963	*
964	* Search for any interface with the given flags. Returns NULL if a device
965	* is not found or a pointer to the device. Must be called inside
966	* rtnl_lock(), and result refcount is unchanged.
967	*/
968
969	struct net_device __dev_get_by_flags(struct* net net, unsigned* short if_flags,
970	unsigned short mask)
971	{
972	struct net_device dev, ret;
973
974	ASSERT_RTNL();
975
976	ret = NULL;
977	for_each_netdev(net, dev) {
978	if (((dev->flags ^ if_flags) & mask) == `0`) {
979	ret = dev;
980	break;
981	}
982	}
983	return ret;
984	}
985	EXPORT_SYMBOL(__dev_get_by_flags);
986
987	/**
988	* dev_valid_name - check if name is okay for network device
989	* @name: name string
990	*
991	* Network device names need to be valid file names to
992	* allow sysfs to work. We also disallow any kind of
993	* whitespace.
994	*/
995	bool dev_valid_name(const char *name)
996	{
997	if (*name == `'\0'`)
998	return false;
999	if (strnlen(name, IFNAMSIZ) == IFNAMSIZ)
1000	return false;
1001	if (!strcmp(name, ".") \|\| !strcmp(name, ".."))
1002	return false;
1003
1004	while (*name) {
1005	if (name == `'/'` \|\| name == `':'` \|\| isspace(*name))
1006	return false;
1007	name++;
1008	}
1009	return true;
1010	}
1011	EXPORT_SYMBOL(dev_valid_name);
1012
1013	/**
1014	* __dev_alloc_name - allocate a name for a device
1015	* @net: network namespace to allocate the device name in
1016	* @name: name format string
1017	* @buf: scratch buffer and result name string
1018	*
1019	* Passed a format string - eg "lt%d" it will try and find a suitable
1020	* id. It scans list of devices to build up a free map, then chooses
1021	* the first empty slot. The caller must hold the dev_base or rtnl lock
1022	* while allocating the name and adding the device in order to avoid
1023	* duplicates.
1024	* Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1025	* Returns the number of the unit assigned or a negative errno code.
1026	*/
1027
1028	static int __dev_alloc_name(struct net net, const* char name, char* *buf)
1029	{
1030	int i = `0`;
1031	const char *p;
1032	const int max_netdevices = `8`*PAGE_SIZE;
1033	unsigned long *inuse;
1034	struct net_device *d;
1035
1036	if (!dev_valid_name(name))
1037	return -EINVAL;
1038
1039	p = strchr(name, `'%'`);
1040	if (p) {
1041	/*
1042	* Verify the string as this thing may have come from
1043	* the user. There must be either one "%d" and no other "%"
1044	* characters.
1045	*/
1046	if (p[`1`] != `'d'` \|\| strchr(p + `2`, `'%'`))
1047	return -EINVAL;
1048
1049	/ Use one page as a bit array of possible slots /
1050	inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
1051	if (!inuse)
1052	return -ENOMEM;
1053
1054	for_each_netdev(net, d) {
1055	struct netdev_name_node *name_node;
1056	list_for_each_entry(name_node, &d->name_node->list, list) {
1057	if (!sscanf(name_node->name, name, &i))
1058	continue;
1059	if (i < `0` \|\| i >= max_netdevices)
1060	continue;
1061
1062	/ avoid cases where sscanf is not exact inverse of printf /
1063	snprintf(buf, IFNAMSIZ, name, i);
1064	if (!strncmp(buf, name_node->name, IFNAMSIZ))
1065	__set_bit(i, inuse);
1066	}
1067	if (!sscanf(d->name, name, &i))
1068	continue;
1069	if (i < `0` \|\| i >= max_netdevices)
1070	continue;
1071
1072	/ avoid cases where sscanf is not exact inverse of printf /
1073	snprintf(buf, IFNAMSIZ, name, i);
1074	if (!strncmp(buf, d->name, IFNAMSIZ))
1075	__set_bit(i, inuse);
1076	}
1077
1078	i = find_first_zero_bit(inuse, max_netdevices);
1079	free_page((unsigned long) inuse);
1080	}
1081
1082	snprintf(buf, IFNAMSIZ, name, i);
1083	if (!netdev_name_in_use(net, buf))
1084	return i;
1085
1086	/ It is possible to run out of possible slots*
1087	* when the name is long and there isn't enough space left
1088	* for the digits, or if all bits are used.
1089	*/
1090	return -ENFILE;
1091	}
1092
1093	static int dev_alloc_name_ns(struct net *net,
1094	struct net_device *dev,
1095	const char *name)
1096	{
1097	char buf[IFNAMSIZ];
1098	int ret;
1099
1100	BUG_ON(!net);
1101	ret = __dev_alloc_name(net, name, buf);
1102	if (ret >= `0`)
1103	strlcpy(dev->name, buf, IFNAMSIZ);
1104	return ret;
1105	}
1106
1107	/**
1108	* dev_alloc_name - allocate a name for a device
1109	* @dev: device
1110	* @name: name format string
1111	*
1112	* Passed a format string - eg "lt%d" it will try and find a suitable
1113	* id. It scans list of devices to build up a free map, then chooses
1114	* the first empty slot. The caller must hold the dev_base or rtnl lock
1115	* while allocating the name and adding the device in order to avoid
1116	* duplicates.
1117	* Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1118	* Returns the number of the unit assigned or a negative errno code.
1119	*/
1120
1121	int dev_alloc_name(struct net_device dev, const* char *name)
1122	{
1123	return dev_alloc_name_ns(dev_net(dev), dev, name);
1124	}
1125	EXPORT_SYMBOL(dev_alloc_name);
1126
1127	static int dev_get_valid_name(struct net net, struct* net_device *dev,
1128	const char *name)
1129	{
1130	BUG_ON(!net);
1131
1132	if (!dev_valid_name(name))
1133	return -EINVAL;
1134
1135	if (strchr(name, `'%'`))
1136	return dev_alloc_name_ns(net, dev, name);
1137	else if (netdev_name_in_use(net, name))
1138	return -EEXIST;
1139	else if (dev->name != name)
1140	strlcpy(dev->name, name, IFNAMSIZ);
1141
1142	return `0`;
1143	}
1144
1145	/**
1146	* dev_change_name - change name of a device
1147	* @dev: device
1148	* @newname: name (or format string) must be at least IFNAMSIZ
1149	*
1150	* Change name of a device, can pass format strings "eth%d".
1151	* for wildcarding.
1152	*/
1153	int dev_change_name(struct net_device dev, const* char *newname)
1154	{
1155	unsigned char old_assign_type;
1156	char oldname[IFNAMSIZ];
1157	int err = `0`;
1158	int ret;
1159	struct net *net;
1160
1161	ASSERT_RTNL();
1162	BUG_ON(!dev_net(dev));
1163
1164	net = dev_net(dev);
1165
1166	/ Some auto-enslaved devices e.g. failover slaves are*
1167	* special, as userspace might rename the device after
1168	* the interface had been brought up and running since
1169	* the point kernel initiated auto-enslavement. Allow
1170	* live name change even when these slave devices are
1171	* up and running.
1172	*
1173	* Typically, users of these auto-enslaving devices
1174	* don't actually care about slave name change, as
1175	* they are supposed to operate on master interface
1176	* directly.
1177	*/
1178	if (dev->flags & IFF_UP &&
1179	likely(!(dev->priv_flags & IFF_LIVE_RENAME_OK)))
1180	return -EBUSY;
1181
1182	down_write(&devnet_rename_sem);
1183
1184	if (strncmp(newname, dev->name, IFNAMSIZ) == `0`) {
1185	up_write(&devnet_rename_sem);
1186	return `0`;
1187	}
1188
1189	memcpy(oldname, dev->name, IFNAMSIZ);
1190
1191	err = dev_get_valid_name(net, dev, newname);
1192	if (err < `0`) {
1193	up_write(&devnet_rename_sem);
1194	return err;
1195	}
1196
1197	if (oldname[`0`] && !strchr(oldname, `'%'`))
1198	netdev_info(dev, "renamed from %s\n", oldname);
1199
1200	old_assign_type = dev->name_assign_type;
1201	dev->name_assign_type = NET_NAME_RENAMED;
1202
1203	rollback:
1204	ret = device_rename(&dev->dev, dev->name);
1205	if (ret) {
1206	memcpy(dev->name, oldname, IFNAMSIZ);
1207	dev->name_assign_type = old_assign_type;
1208	up_write(&devnet_rename_sem);
1209	return ret;
1210	}
1211
1212	up_write(&devnet_rename_sem);
1213
1214	netdev_adjacent_rename_links(dev, oldname);
1215
1216	write_lock(&dev_base_lock);
1217	netdev_name_node_del(dev->name_node);
1218	write_unlock(&dev_base_lock);
1219
1220	synchronize_rcu();
1221
1222	write_lock(&dev_base_lock);
1223	netdev_name_node_add(net, dev->name_node);
1224	write_unlock(&dev_base_lock);
1225
1226	ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1227	ret = notifier_to_errno(ret);
1228
1229	if (ret) {
1230	/ err >= 0 after dev_alloc_name() or stores the first errno /
1231	if (err >= `0`) {
1232	err = ret;
1233	down_write(&devnet_rename_sem);
1234	memcpy(dev->name, oldname, IFNAMSIZ);
1235	memcpy(oldname, newname, IFNAMSIZ);
1236	dev->name_assign_type = old_assign_type;
1237	old_assign_type = NET_NAME_RENAMED;
1238	goto rollback;
1239	} else {
1240	netdev_err(dev, "name change rollback failed: %d\n",
1241	ret);
1242	}
1243	}
1244
1245	return err;
1246	}
1247
1248	/**
1249	* dev_set_alias - change ifalias of a device
1250	* @dev: device
1251	* @alias: name up to IFALIASZ
1252	* @len: limit of bytes to copy from info
1253	*
1254	* Set ifalias for a device,
1255	*/
1256	int dev_set_alias(struct net_device dev, const* char *alias, size_t len)
1257	{
1258	struct dev_ifalias *new_alias = NULL;
1259
1260	if (len >= IFALIASZ)
1261	return -EINVAL;
1262
1263	if (len) {
1264	new_alias = kmalloc(sizeof(*new_alias) + len + `1`, GFP_KERNEL);
1265	if (!new_alias)
1266	return -ENOMEM;
1267
1268	memcpy(new_alias->ifalias, alias, len);
1269	new_alias->ifalias[len] = `0`;
1270	}
1271
1272	mutex_lock(&ifalias_mutex);
1273	new_alias = rcu_replace_pointer(dev->ifalias, new_alias,
1274	mutex_is_locked(&ifalias_mutex));
1275	mutex_unlock(&ifalias_mutex);
1276
1277	if (new_alias)
1278	kfree_rcu(new_alias, rcuhead);
1279
1280	return len;
1281	}
1282	EXPORT_SYMBOL(dev_set_alias);
1283
1284	/**
1285	* dev_get_alias - get ifalias of a device
1286	* @dev: device
1287	* @name: buffer to store name of ifalias
1288	* @len: size of buffer
1289	*
1290	* get ifalias for a device. Caller must make sure dev cannot go
1291	* away, e.g. rcu read lock or own a reference count to device.
1292	*/
1293	int dev_get_alias(const struct net_device dev, char* *name, size_t len)
1294	{
1295	const struct dev_ifalias *alias;
1296	int ret = `0`;
1297
1298	rcu_read_lock();
1299	alias = rcu_dereference(dev->ifalias);
1300	if (alias)
1301	ret = snprintf(name, len, "%s", alias->ifalias);
1302	rcu_read_unlock();
1303
1304	return ret;
1305	}
1306
1307	/**
1308	* netdev_features_change - device changes features
1309	* @dev: device to cause notification
1310	*
1311	* Called to indicate a device has changed features.
1312	*/
1313	void netdev_features_change(struct net_device *dev)
1314	{
1315	call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1316	}
1317	EXPORT_SYMBOL(netdev_features_change);
1318
1319	/**
1320	* netdev_state_change - device changes state
1321	* @dev: device to cause notification
1322	*
1323	* Called to indicate a device has changed state. This function calls
1324	* the notifier chains for netdev_chain and sends a NEWLINK message
1325	* to the routing socket.
1326	*/
1327	void netdev_state_change(struct net_device *dev)
1328	{
1329	if (dev->flags & IFF_UP) {
1330	struct netdev_notifier_change_info change_info = {
1331	.info.dev = dev,
1332	};
1333
1334	call_netdevice_notifiers_info(NETDEV_CHANGE,
1335	&change_info.info);
1336	rtmsg_ifinfo(RTM_NEWLINK, dev, `0`, GFP_KERNEL);
1337	}
1338	}
1339	EXPORT_SYMBOL(netdev_state_change);
1340
1341	/**
1342	* __netdev_notify_peers - notify network peers about existence of @dev,
1343	* to be called when rtnl lock is already held.
1344	* @dev: network device
1345	*
1346	* Generate traffic such that interested network peers are aware of
1347	* @dev, such as by generating a gratuitous ARP. This may be used when
1348	* a device wants to inform the rest of the network about some sort of
1349	* reconfiguration such as a failover event or virtual machine
1350	* migration.
1351	*/
1352	void __netdev_notify_peers(struct net_device *dev)
1353	{
1354	ASSERT_RTNL();
1355	call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1356	call_netdevice_notifiers(NETDEV_RESEND_IGMP, dev);
1357	}
1358	EXPORT_SYMBOL(__netdev_notify_peers);
1359
1360	/**
1361	* netdev_notify_peers - notify network peers about existence of @dev
1362	* @dev: network device
1363	*
1364	* Generate traffic such that interested network peers are aware of
1365	* @dev, such as by generating a gratuitous ARP. This may be used when
1366	* a device wants to inform the rest of the network about some sort of
1367	* reconfiguration such as a failover event or virtual machine
1368	* migration.
1369	*/
1370	void netdev_notify_peers(struct net_device *dev)
1371	{
1372	rtnl_lock();
1373	__netdev_notify_peers(dev);
1374	rtnl_unlock();
1375	}
1376	EXPORT_SYMBOL(netdev_notify_peers);
1377
1378	static int napi_threaded_poll(void *data);
1379
1380	static int napi_kthread_create(struct napi_struct *n)
1381	{
1382	int err = `0`;
1383
1384	/ Create and wake up the kthread once to put it in*
1385	* TASK_INTERRUPTIBLE mode to avoid the blocked task
1386	* warning and work with loadavg.
1387	*/
1388	n->thread = kthread_run(napi_threaded_poll, n, "napi/%s-%d",
1389	n->dev->name, n->napi_id);
1390	if (IS_ERR(n->thread)) {
1391	err = PTR_ERR(n->thread);
1392	pr_err("kthread_run failed with err %d\n", err);
1393	n->thread = NULL;
1394	}
1395
1396	return err;
1397	}
1398
1399	static int __dev_open(struct net_device dev, struct* netlink_ext_ack *extack)
1400	{
1401	const struct net_device_ops *ops = dev->netdev_ops;
1402	int ret;
1403
1404	ASSERT_RTNL();
1405	dev_addr_check(dev);
1406
1407	if (!netif_device_present(dev)) {
1408	/ may be detached because parent is runtime-suspended /
1409	if (dev->dev.parent)
1410	pm_runtime_resume(dev->dev.parent);
1411	if (!netif_device_present(dev))
1412	return -ENODEV;
1413	}
1414
1415	/ Block netpoll from trying to do any rx path servicing.*
1416	* If we don't do this there is a chance ndo_poll_controller
1417	* or ndo_poll may be running while we open the device
1418	*/
1419	netpoll_poll_disable(dev);
1420
1421	ret = call_netdevice_notifiers_extack(NETDEV_PRE_UP, dev, extack);
1422	ret = notifier_to_errno(ret);
1423	if (ret)
1424	return ret;
1425
1426	set_bit(__LINK_STATE_START, &dev->state);
1427
1428	if (ops->ndo_validate_addr)
1429	ret = ops->ndo_validate_addr(dev);
1430
1431	if (!ret && ops->ndo_open)
1432	ret = ops->ndo_open(dev);
1433
1434	netpoll_poll_enable(dev);
1435
1436	if (ret)
1437	clear_bit(__LINK_STATE_START, &dev->state);
1438	else {
1439	dev->flags \|= IFF_UP;
1440	dev_set_rx_mode(dev);
1441	dev_activate(dev);
1442	add_device_randomness(dev->dev_addr, dev->addr_len);
1443	}
1444
1445	return ret;
1446	}
1447
1448	/**
1449	* dev_open - prepare an interface for use.
1450	* @dev: device to open
1451	* @extack: netlink extended ack
1452	*
1453	* Takes a device from down to up state. The device's private open
1454	* function is invoked and then the multicast lists are loaded. Finally
1455	* the device is moved into the up state and a %NETDEV_UP message is
1456	* sent to the netdev notifier chain.
1457	*
1458	* Calling this function on an active interface is a nop. On a failure
1459	* a negative errno code is returned.
1460	*/
1461	int dev_open(struct net_device dev, struct* netlink_ext_ack *extack)
1462	{
1463	int ret;
1464
1465	if (dev->flags & IFF_UP)
1466	return `0`;
1467
1468	ret = __dev_open(dev, extack);
1469	if (ret < `0`)
1470	return ret;
1471
1472	rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP\|IFF_RUNNING, GFP_KERNEL);
1473	call_netdevice_notifiers(NETDEV_UP, dev);
1474
1475	return ret;
1476	}
1477	EXPORT_SYMBOL(dev_open);
1478
1479	static void __dev_close_many(struct list_head *head)
1480	{
1481	struct net_device *dev;
1482
1483	ASSERT_RTNL();
1484	might_sleep();
1485
1486	list_for_each_entry(dev, head, close_list) {
1487	/ Temporarily disable netpoll until the interface is down /
1488	netpoll_poll_disable(dev);
1489
1490	call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1491
1492	clear_bit(__LINK_STATE_START, &dev->state);
1493
1494	/ Synchronize to scheduled poll. We cannot touch poll list, it*
1495	* can be even on different cpu. So just clear netif_running().
1496	*
1497	* dev->stop() will invoke napi_disable() on all of it's
1498	* napi_struct instances on this device.
1499	*/
1500	smp_mb__after_atomic(); / Commit netif_running(). /
1501	}
1502
1503	dev_deactivate_many(head);
1504
1505	list_for_each_entry(dev, head, close_list) {
1506	const struct net_device_ops *ops = dev->netdev_ops;
1507
1508	/*
1509	* Call the device specific close. This cannot fail.
1510	* Only if device is UP
1511	*
1512	* We allow it to be called even after a DETACH hot-plug
1513	* event.
1514	*/
1515	if (ops->ndo_stop)
1516	ops->ndo_stop(dev);
1517
1518	dev->flags &= ~IFF_UP;
1519	netpoll_poll_enable(dev);
1520	}
1521	}
1522
1523	static void __dev_close(struct net_device *dev)
1524	{
1525	LIST_HEAD(single);
1526
1527	list_add(&dev->close_list, &single);
1528	__dev_close_many(&single);
1529	list_del(&single);
1530	}
1531
1532	void dev_close_many(struct list_head *head, bool unlink)
1533	{
1534	struct net_device dev, tmp;
1535
1536	/ Remove the devices that don't need to be closed /
1537	list_for_each_entry_safe(dev, tmp, head, close_list)
1538	if (!(dev->flags & IFF_UP))
1539	list_del_init(&dev->close_list);
1540
1541	__dev_close_many(head);
1542
1543	list_for_each_entry_safe(dev, tmp, head, close_list) {
1544	rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP\|IFF_RUNNING, GFP_KERNEL);
1545	call_netdevice_notifiers(NETDEV_DOWN, dev);
1546	if (unlink)
1547	list_del_init(&dev->close_list);
1548	}
1549	}
1550	EXPORT_SYMBOL(dev_close_many);
1551
1552	/**
1553	* dev_close - shutdown an interface.
1554	* @dev: device to shutdown
1555	*
1556	* This function moves an active device into down state. A
1557	* %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1558	* is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1559	* chain.
1560	*/
1561	void dev_close(struct net_device *dev)
1562	{
1563	if (dev->flags & IFF_UP) {
1564	LIST_HEAD(single);
1565
1566	list_add(&dev->close_list, &single);
1567	dev_close_many(&single, true);
1568	list_del(&single);
1569	}
1570	}
1571	EXPORT_SYMBOL(dev_close);
1572
1573
1574	/**
1575	* dev_disable_lro - disable Large Receive Offload on a device
1576	* @dev: device
1577	*
1578	* Disable Large Receive Offload (LRO) on a net device. Must be
1579	* called under RTNL. This is needed if received packets may be
1580	* forwarded to another interface.
1581	*/
1582	void dev_disable_lro(struct net_device *dev)
1583	{
1584	struct net_device *lower_dev;
1585	struct list_head *iter;
1586
1587	dev->wanted_features &= ~NETIF_F_LRO;
1588	netdev_update_features(dev);
1589
1590	if (unlikely(dev->features & NETIF_F_LRO))
1591	netdev_WARN(dev, "failed to disable LRO!\n");
1592
1593	netdev_for_each_lower_dev(dev, lower_dev, iter)
1594	dev_disable_lro(lower_dev);
1595	}
1596	EXPORT_SYMBOL(dev_disable_lro);
1597
1598	/**
1599	* dev_disable_gro_hw - disable HW Generic Receive Offload on a device
1600	* @dev: device
1601	*
1602	* Disable HW Generic Receive Offload (GRO_HW) on a net device. Must be
1603	* called under RTNL. This is needed if Generic XDP is installed on
1604	* the device.
1605	*/
1606	static void dev_disable_gro_hw(struct net_device *dev)
1607	{
1608	dev->wanted_features &= ~NETIF_F_GRO_HW;
1609	netdev_update_features(dev);
1610
1611	if (unlikely(dev->features & NETIF_F_GRO_HW))
1612	netdev_WARN(dev, "failed to disable GRO_HW!\n");
1613	}
1614
1615	const char netdev_cmd_to_name(enum* netdev_cmd cmd)
1616	{
1617	#define N(val) \
1618	case NETDEV_##val: \
1619	return "NETDEV_" __stringify(val);
1620	switch (cmd) {
1621	N(UP) N(DOWN) N(REBOOT) N(CHANGE) N(REGISTER) N(UNREGISTER)
1622	N(CHANGEMTU) N(CHANGEADDR) N(GOING_DOWN) N(CHANGENAME) N(FEAT_CHANGE)
1623	N(BONDING_FAILOVER) N(PRE_UP) N(PRE_TYPE_CHANGE) N(POST_TYPE_CHANGE)
1624	N(POST_INIT) N(RELEASE) N(NOTIFY_PEERS) N(JOIN) N(CHANGEUPPER)
1625	N(RESEND_IGMP) N(PRECHANGEMTU) N(CHANGEINFODATA) N(BONDING_INFO)
1626	N(PRECHANGEUPPER) N(CHANGELOWERSTATE) N(UDP_TUNNEL_PUSH_INFO)
1627	N(UDP_TUNNEL_DROP_INFO) N(CHANGE_TX_QUEUE_LEN)
1628	N(CVLAN_FILTER_PUSH_INFO) N(CVLAN_FILTER_DROP_INFO)
1629	N(SVLAN_FILTER_PUSH_INFO) N(SVLAN_FILTER_DROP_INFO)
1630	N(PRE_CHANGEADDR) N(OFFLOAD_XSTATS_ENABLE) N(OFFLOAD_XSTATS_DISABLE)
1631	N(OFFLOAD_XSTATS_REPORT_USED) N(OFFLOAD_XSTATS_REPORT_DELTA)
1632	}
1633	#undef N
1634	return "UNKNOWN_NETDEV_EVENT";
1635	}
1636	EXPORT_SYMBOL_GPL(netdev_cmd_to_name);
1637
1638	static int call_netdevice_notifier(struct notifier_block nb, unsigned* long val,
1639	struct net_device *dev)
1640	{
1641	struct netdev_notifier_info info = {
1642	.dev = dev,
1643	};
1644
1645	return nb->notifier_call(nb, val, &info);
1646	}
1647
1648	static int call_netdevice_register_notifiers(struct notifier_block *nb,
1649	struct net_device *dev)
1650	{
1651	int err;
1652
1653	err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
1654	err = notifier_to_errno(err);
1655	if (err)
1656	return err;
1657
1658	if (!(dev->flags & IFF_UP))
1659	return `0`;
1660
1661	call_netdevice_notifier(nb, NETDEV_UP, dev);
1662	return `0`;
1663	}
1664
1665	static void call_netdevice_unregister_notifiers(struct notifier_block *nb,
1666	struct net_device *dev)
1667	{
1668	if (dev->flags & IFF_UP) {
1669	call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1670	dev);
1671	call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1672	}
1673	call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1674	}
1675
1676	static int call_netdevice_register_net_notifiers(struct notifier_block *nb,
1677	struct net *net)
1678	{
1679	struct net_device *dev;
1680	int err;
1681
1682	for_each_netdev(net, dev) {
1683	err = call_netdevice_register_notifiers(nb, dev);
1684	if (err)
1685	goto rollback;
1686	}
1687	return `0`;
1688
1689	rollback:
1690	for_each_netdev_continue_reverse(net, dev)
1691	call_netdevice_unregister_notifiers(nb, dev);
1692	return err;
1693	}
1694
1695	static void call_netdevice_unregister_net_notifiers(struct notifier_block *nb,
1696	struct net *net)
1697	{
1698	struct net_device *dev;
1699
1700	for_each_netdev(net, dev)
1701	call_netdevice_unregister_notifiers(nb, dev);
1702	}
1703
1704	static int dev_boot_phase = `1`;
1705
1706	/**
1707	* register_netdevice_notifier - register a network notifier block
1708	* @nb: notifier
1709	*
1710	* Register a notifier to be called when network device events occur.
1711	* The notifier passed is linked into the kernel structures and must
1712	* not be reused until it has been unregistered. A negative errno code
1713	* is returned on a failure.
1714	*
1715	* When registered all registration and up events are replayed
1716	* to the new notifier to allow device to have a race free
1717	* view of the network device list.
1718	*/
1719
1720	int register_netdevice_notifier(struct notifier_block *nb)
1721	{
1722	struct net *net;
1723	int err;
1724
1725	/ Close race with setup_net() and cleanup_net() /
1726	down_write(&pernet_ops_rwsem);
1727	rtnl_lock();
1728	err = raw_notifier_chain_register(&netdev_chain, nb);
1729	if (err)
1730	goto unlock;
1731	if (dev_boot_phase)
1732	goto unlock;
1733	for_each_net(net) {
1734	err = call_netdevice_register_net_notifiers(nb, net);
1735	if (err)
1736	goto rollback;
1737	}
1738
1739	unlock:
1740	rtnl_unlock();
1741	up_write(&pernet_ops_rwsem);
1742	return err;
1743
1744	rollback:
1745	for_each_net_continue_reverse(net)
1746	call_netdevice_unregister_net_notifiers(nb, net);
1747
1748	raw_notifier_chain_unregister(&netdev_chain, nb);
1749	goto unlock;
1750	}
1751	EXPORT_SYMBOL(register_netdevice_notifier);
1752
1753	/**
1754	* unregister_netdevice_notifier - unregister a network notifier block
1755	* @nb: notifier
1756	*
1757	* Unregister a notifier previously registered by
1758	* register_netdevice_notifier(). The notifier is unlinked into the
1759	* kernel structures and may then be reused. A negative errno code
1760	* is returned on a failure.
1761	*
1762	* After unregistering unregister and down device events are synthesized
1763	* for all devices on the device list to the removed notifier to remove
1764	* the need for special case cleanup code.
1765	*/
1766
1767	int unregister_netdevice_notifier(struct notifier_block *nb)
1768	{
1769	struct net *net;
1770	int err;
1771
1772	/ Close race with setup_net() and cleanup_net() /
1773	down_write(&pernet_ops_rwsem);
1774	rtnl_lock();
1775	err = raw_notifier_chain_unregister(&netdev_chain, nb);
1776	if (err)
1777	goto unlock;
1778
1779	for_each_net(net)
1780	call_netdevice_unregister_net_notifiers(nb, net);
1781
1782	unlock:
1783	rtnl_unlock();
1784	up_write(&pernet_ops_rwsem);
1785	return err;
1786	}
1787	EXPORT_SYMBOL(unregister_netdevice_notifier);
1788
1789	static int __register_netdevice_notifier_net(struct net *net,
1790	struct notifier_block *nb,
1791	bool ignore_call_fail)
1792	{
1793	int err;
1794
1795	err = raw_notifier_chain_register(&net->netdev_chain, nb);
1796	if (err)
1797	return err;
1798	if (dev_boot_phase)
1799	return `0`;
1800
1801	err = call_netdevice_register_net_notifiers(nb, net);
1802	if (err && !ignore_call_fail)
1803	goto chain_unregister;
1804
1805	return `0`;
1806
1807	chain_unregister:
1808	raw_notifier_chain_unregister(&net->netdev_chain, nb);
1809	return err;
1810	}
1811
1812	static int __unregister_netdevice_notifier_net(struct net *net,
1813	struct notifier_block *nb)
1814	{
1815	int err;
1816
1817	err = raw_notifier_chain_unregister(&net->netdev_chain, nb);
1818	if (err)
1819	return err;
1820
1821	call_netdevice_unregister_net_notifiers(nb, net);
1822	return `0`;
1823	}
1824
1825	/**
1826	* register_netdevice_notifier_net - register a per-netns network notifier block
1827	* @net: network namespace
1828	* @nb: notifier
1829	*
1830	* Register a notifier to be called when network device events occur.
1831	* The notifier passed is linked into the kernel structures and must
1832	* not be reused until it has been unregistered. A negative errno code
1833	* is returned on a failure.
1834	*
1835	* When registered all registration and up events are replayed
1836	* to the new notifier to allow device to have a race free
1837	* view of the network device list.
1838	*/
1839
1840	int register_netdevice_notifier_net(struct net net, struct* notifier_block *nb)
1841	{
1842	int err;
1843
1844	rtnl_lock();
1845	err = __register_netdevice_notifier_net(net, nb, false);
1846	rtnl_unlock();
1847	return err;
1848	}
1849	EXPORT_SYMBOL(register_netdevice_notifier_net);
1850
1851	/**
1852	* unregister_netdevice_notifier_net - unregister a per-netns
1853	* network notifier block
1854	* @net: network namespace
1855	* @nb: notifier
1856	*
1857	* Unregister a notifier previously registered by
1858	* register_netdevice_notifier(). The notifier is unlinked into the
1859	* kernel structures and may then be reused. A negative errno code
1860	* is returned on a failure.
1861	*
1862	* After unregistering unregister and down device events are synthesized
1863	* for all devices on the device list to the removed notifier to remove
1864	* the need for special case cleanup code.
1865	*/
1866
1867	int unregister_netdevice_notifier_net(struct net *net,
1868	struct notifier_block *nb)
1869	{
1870	int err;
1871
1872	rtnl_lock();
1873	err = __unregister_netdevice_notifier_net(net, nb);
1874	rtnl_unlock();
1875	return err;
1876	}
1877	EXPORT_SYMBOL(unregister_netdevice_notifier_net);
1878
1879	int register_netdevice_notifier_dev_net(struct net_device *dev,
1880	struct notifier_block *nb,
1881	struct netdev_net_notifier *nn)
1882	{
1883	int err;
1884
1885	rtnl_lock();
1886	err = __register_netdevice_notifier_net(dev_net(dev), nb, false);
1887	if (!err) {
1888	nn->nb = nb;
1889	list_add(&nn->list, &dev->net_notifier_list);
1890	}
1891	rtnl_unlock();
1892	return err;
1893	}
1894	EXPORT_SYMBOL(register_netdevice_notifier_dev_net);
1895
1896	int unregister_netdevice_notifier_dev_net(struct net_device *dev,
1897	struct notifier_block *nb,
1898	struct netdev_net_notifier *nn)
1899	{
1900	int err;
1901
1902	rtnl_lock();
1903	list_del(&nn->list);
1904	err = __unregister_netdevice_notifier_net(dev_net(dev), nb);
1905	rtnl_unlock();
1906	return err;
1907	}
1908	EXPORT_SYMBOL(unregister_netdevice_notifier_dev_net);
1909
1910	static void move_netdevice_notifiers_dev_net(struct net_device *dev,
1911	struct net *net)
1912	{
1913	struct netdev_net_notifier *nn;
1914
1915	list_for_each_entry(nn, &dev->net_notifier_list, list) {
1916	__unregister_netdevice_notifier_net(dev_net(dev), nn->nb);
1917	__register_netdevice_notifier_net(net, nn->nb, true);
1918	}
1919	}
1920
1921	/**
1922	* call_netdevice_notifiers_info - call all network notifier blocks
1923	* @val: value passed unmodified to notifier function
1924	* @info: notifier information data
1925	*
1926	* Call all network notifier blocks. Parameters and return value
1927	* are as for raw_notifier_call_chain().
1928	*/
1929
1930	static int call_netdevice_notifiers_info(unsigned long val,
1931	struct netdev_notifier_info *info)
1932	{
1933	struct net *net = dev_net(info->dev);
1934	int ret;
1935
1936	ASSERT_RTNL();
1937
1938	/ Run per-netns notifier block chain first, then run the global one.*
1939	* Hopefully, one day, the global one is going to be removed after
1940	* all notifier block registrators get converted to be per-netns.
1941	*/
1942	ret = raw_notifier_call_chain(&net->netdev_chain, val, info);
1943	if (ret & NOTIFY_STOP_MASK)
1944	return ret;
1945	return raw_notifier_call_chain(&netdev_chain, val, info);
1946	}
1947
1948	/**
1949	* call_netdevice_notifiers_info_robust - call per-netns notifier blocks
1950	* for and rollback on error
1951	* @val_up: value passed unmodified to notifier function
1952	* @val_down: value passed unmodified to the notifier function when
1953	* recovering from an error on @val_up
1954	* @info: notifier information data
1955	*
1956	* Call all per-netns network notifier blocks, but not notifier blocks on
1957	* the global notifier chain. Parameters and return value are as for
1958	* raw_notifier_call_chain_robust().
1959	*/
1960
1961	static int
1962	call_netdevice_notifiers_info_robust(unsigned long val_up,
1963	unsigned long val_down,
1964	struct netdev_notifier_info *info)
1965	{
1966	struct net *net = dev_net(info->dev);
1967
1968	ASSERT_RTNL();
1969
1970	return raw_notifier_call_chain_robust(&net->netdev_chain,
1971	val_up, val_down, info);
1972	}
1973
1974	static int call_netdevice_notifiers_extack(unsigned long val,
1975	struct net_device *dev,
1976	struct netlink_ext_ack *extack)
1977	{
1978	struct netdev_notifier_info info = {
1979	.dev = dev,
1980	.extack = extack,
1981	};
1982
1983	return call_netdevice_notifiers_info(val, &info);
1984	}
1985
1986	/**
1987	* call_netdevice_notifiers - call all network notifier blocks
1988	* @val: value passed unmodified to notifier function
1989	* @dev: net_device pointer passed unmodified to notifier function
1990	*
1991	* Call all network notifier blocks. Parameters and return value
1992	* are as for raw_notifier_call_chain().
1993	*/
1994
1995	int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
1996	{
1997	return call_netdevice_notifiers_extack(val, dev, NULL);
1998	}
1999	EXPORT_SYMBOL(call_netdevice_notifiers);
2000
2001	/**
2002	* call_netdevice_notifiers_mtu - call all network notifier blocks
2003	* @val: value passed unmodified to notifier function
2004	* @dev: net_device pointer passed unmodified to notifier function
2005	* @arg: additional u32 argument passed to the notifier function
2006	*
2007	* Call all network notifier blocks. Parameters and return value
2008	* are as for raw_notifier_call_chain().
2009	*/
2010	static int call_netdevice_notifiers_mtu(unsigned long val,
2011	struct net_device *dev, u32 arg)
2012	{
2013	struct netdev_notifier_info_ext info = {
2014	.info.dev = dev,
2015	.ext.mtu = arg,
2016	};
2017
2018	BUILD_BUG_ON(offsetof(struct netdev_notifier_info_ext, info) != `0`);
2019
2020	return call_netdevice_notifiers_info(val, &info.info);
2021	}
2022
2023	#ifdef CONFIG_NET_INGRESS
2024	static DEFINE_STATIC_KEY_FALSE(ingress_needed_key);
2025
2026	void net_inc_ingress_queue(void)
2027	{
2028	static_branch_inc(&ingress_needed_key);
2029	}
2030	EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
2031
2032	void net_dec_ingress_queue(void)
2033	{
2034	static_branch_dec(&ingress_needed_key);
2035	}
2036	EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
2037	#endif
2038
2039	#ifdef CONFIG_NET_EGRESS
2040	static DEFINE_STATIC_KEY_FALSE(egress_needed_key);
2041
2042	void net_inc_egress_queue(void)
2043	{
2044	static_branch_inc(&egress_needed_key);
2045	}
2046	EXPORT_SYMBOL_GPL(net_inc_egress_queue);
2047
2048	void net_dec_egress_queue(void)
2049	{
2050	static_branch_dec(&egress_needed_key);
2051	}
2052	EXPORT_SYMBOL_GPL(net_dec_egress_queue);
2053	#endif
2054
2055	DEFINE_STATIC_KEY_FALSE(netstamp_needed_key);
2056	EXPORT_SYMBOL(netstamp_needed_key);
2057	#ifdef CONFIG_JUMP_LABEL
2058	static atomic_t netstamp_needed_deferred;
2059	static atomic_t netstamp_wanted;
2060	static void netstamp_clear(struct work_struct *work)
2061	{
2062	int deferred = atomic_xchg(&netstamp_needed_deferred, `0`);
2063	int wanted;
2064
2065	wanted = atomic_add_return(deferred, &netstamp_wanted);
2066	if (wanted > `0`)
2067	static_branch_enable(&netstamp_needed_key);
2068	else
2069	static_branch_disable(&netstamp_needed_key);
2070	}
2071	static DECLARE_WORK(netstamp_work, netstamp_clear);
2072	#endif
2073
2074	void net_enable_timestamp(void)
2075	{
2076	#ifdef CONFIG_JUMP_LABEL
2077	int wanted;
2078
2079	while (`1`) {
2080	wanted = atomic_read(&netstamp_wanted);
2081	if (wanted <= `0`)
2082	break;
2083	if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted + `1`) == wanted)
2084	return;
2085	}
2086	atomic_inc(&netstamp_needed_deferred);
2087	schedule_work(&netstamp_work);
2088	#else
2089	static_branch_inc(&netstamp_needed_key);
2090	#endif
2091	}
2092	EXPORT_SYMBOL(net_enable_timestamp);
2093
2094	void net_disable_timestamp(void)
2095	{
2096	#ifdef CONFIG_JUMP_LABEL
2097	int wanted;
2098
2099	while (`1`) {
2100	wanted = atomic_read(&netstamp_wanted);
2101	if (wanted <= `1`)
2102	break;
2103	if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted - `1`) == wanted)
2104	return;
2105	}
2106	atomic_dec(&netstamp_needed_deferred);
2107	schedule_work(&netstamp_work);
2108	#else
2109	static_branch_dec(&netstamp_needed_key);
2110	#endif
2111	}
2112	EXPORT_SYMBOL(net_disable_timestamp);
2113
2114	static inline void net_timestamp_set(struct sk_buff *skb)
2115	{
2116	skb->tstamp = `0`;
2117	skb->mono_delivery_time = `0`;
2118	if (static_branch_unlikely(&netstamp_needed_key))
2119	skb->tstamp = ktime_get_real();
2120	}
2121
2122	#define net_timestamp_check(COND, SKB) \
2123	if (static_branch_unlikely(&netstamp_needed_key)) { \
2124	if ((COND) && !(SKB)->tstamp) \
2125	(SKB)->tstamp = ktime_get_real(); \
2126	} \
2127
2128	bool is_skb_forwardable(const struct net_device dev, const* struct sk_buff *skb)
2129	{
2130	return __is_skb_forwardable(dev, skb, true);
2131	}
2132	EXPORT_SYMBOL_GPL(is_skb_forwardable);
2133
2134	static int __dev_forward_skb2(struct net_device dev, struct* sk_buff *skb,
2135	bool check_mtu)
2136	{
2137	int ret = ____dev_forward_skb(dev, skb, check_mtu);
2138
2139	if (likely(!ret)) {
2140	skb->protocol = eth_type_trans(skb, dev);
2141	skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
2142	}
2143
2144	return ret;
2145	}
2146
2147	int __dev_forward_skb(struct net_device dev, struct* sk_buff *skb)
2148	{
2149	return __dev_forward_skb2(dev, skb, true);
2150	}
2151	EXPORT_SYMBOL_GPL(__dev_forward_skb);
2152
2153	/**
2154	* dev_forward_skb - loopback an skb to another netif
2155	*
2156	* @dev: destination network device
2157	* @skb: buffer to forward
2158	*
2159	* return values:
2160	* NET_RX_SUCCESS (no congestion)
2161	* NET_RX_DROP (packet was dropped, but freed)
2162	*
2163	* dev_forward_skb can be used for injecting an skb from the
2164	* start_xmit function of one device into the receive queue
2165	* of another device.
2166	*
2167	* The receiving device may be in another namespace, so
2168	* we have to clear all information in the skb that could
2169	* impact namespace isolation.
2170	*/
2171	int dev_forward_skb(struct net_device dev, struct* sk_buff *skb)
2172	{
2173	return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
2174	}
2175	EXPORT_SYMBOL_GPL(dev_forward_skb);
2176
2177	int dev_forward_skb_nomtu(struct net_device dev, struct* sk_buff *skb)
2178	{
2179	return __dev_forward_skb2(dev, skb, false) ?: netif_rx_internal(skb);
2180	}
2181
2182	static inline int deliver_skb(struct sk_buff *skb,
2183	struct packet_type *pt_prev,
2184	struct net_device *orig_dev)
2185	{
2186	if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
2187	return -ENOMEM;
2188	refcount_inc(&skb->users);
2189	return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
2190	}
2191
2192	static inline void deliver_ptype_list_skb(struct sk_buff *skb,
2193	struct packet_type **pt,
2194	struct net_device *orig_dev,
2195	__be16 type,
2196	struct list_head *ptype_list)
2197	{
2198	struct packet_type ptype, pt_prev = *pt;
2199
2200	list_for_each_entry_rcu(ptype, ptype_list, list) {
2201	if (ptype->type != type)
2202	continue;
2203	if (pt_prev)
2204	deliver_skb(skb, pt_prev, orig_dev);
2205	pt_prev = ptype;
2206	}
2207	*pt = pt_prev;
2208	}
2209
2210	static inline bool skb_loop_sk(struct packet_type ptype, struct* sk_buff *skb)
2211	{
2212	if (!ptype->af_packet_priv \|\| !skb->sk)
2213	return false;
2214
2215	if (ptype->id_match)
2216	return ptype->id_match(ptype, skb->sk);
2217	else if ((struct sock *)ptype->af_packet_priv == skb->sk)
2218	return true;
2219
2220	return false;
2221	}
2222
2223	/**
2224	* dev_nit_active - return true if any network interface taps are in use
2225	*
2226	* @dev: network device to check for the presence of taps
2227	*/
2228	bool dev_nit_active(struct net_device *dev)
2229	{
2230	return !list_empty(&ptype_all) \|\| !list_empty(&dev->ptype_all);
2231	}
2232	EXPORT_SYMBOL_GPL(dev_nit_active);
2233
2234	/*
2235	* Support routine. Sends outgoing frames to any network
2236	* taps currently in use.
2237	*/
2238
2239	void dev_queue_xmit_nit(struct sk_buff skb, struct* net_device *dev)
2240	{
2241	struct packet_type *ptype;
2242	struct sk_buff *skb2 = NULL;
2243	struct packet_type *pt_prev = NULL;
2244	struct list_head *ptype_list = &ptype_all;
2245
2246	rcu_read_lock();
2247	again:
2248	list_for_each_entry_rcu(ptype, ptype_list, list) {
2249	if (ptype->ignore_outgoing)
2250	continue;
2251
2252	/ Never send packets back to the socket*
2253	* they originated from - MvS (miquels@drinkel.ow.org)
2254	*/
2255	if (skb_loop_sk(ptype, skb))
2256	continue;
2257
2258	if (pt_prev) {
2259	deliver_skb(skb2, pt_prev, skb->dev);
2260	pt_prev = ptype;
2261	continue;
2262	}
2263
2264	/ need to clone skb, done only once /
2265	skb2 = skb_clone(skb, GFP_ATOMIC);
2266	if (!skb2)
2267	goto out_unlock;
2268
2269	net_timestamp_set(skb2);
2270
2271	/ skb->nh should be correctly*
2272	* set by sender, so that the second statement is
2273	* just protection against buggy protocols.
2274	*/
2275	skb_reset_mac_header(skb2);
2276
2277	if (skb_network_header(skb2) < skb2->data \|\|
2278	skb_network_header(skb2) > skb_tail_pointer(skb2)) {
2279	net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
2280	ntohs(skb2->protocol),
2281	dev->name);
2282	skb_reset_network_header(skb2);
2283	}
2284
2285	skb2->transport_header = skb2->network_header;
2286	skb2->pkt_type = PACKET_OUTGOING;
2287	pt_prev = ptype;
2288	}
2289
2290	if (ptype_list == &ptype_all) {
2291	ptype_list = &dev->ptype_all;
2292	goto again;
2293	}
2294	out_unlock:
2295	if (pt_prev) {
2296	if (!skb_orphan_frags_rx(skb2, GFP_ATOMIC))
2297	pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
2298	else
2299	kfree_skb(skb2);
2300	}
2301	rcu_read_unlock();
2302	}
2303	EXPORT_SYMBOL_GPL(dev_queue_xmit_nit);
2304
2305	/**
2306	* netif_setup_tc - Handle tc mappings on real_num_tx_queues change
2307	* @dev: Network device
2308	* @txq: number of queues available
2309	*
2310	* If real_num_tx_queues is changed the tc mappings may no longer be
2311	* valid. To resolve this verify the tc mapping remains valid and if
2312	* not NULL the mapping. With no priorities mapping to this
2313	* offset/count pair it will no longer be used. In the worst case TC0
2314	* is invalid nothing can be done so disable priority mappings. If is
2315	* expected that drivers will fix this mapping if they can before
2316	* calling netif_set_real_num_tx_queues.
2317	*/
2318	static void netif_setup_tc(struct net_device dev, unsigned* int txq)
2319	{
2320	int i;
2321	struct netdev_tc_txq *tc = &dev->tc_to_txq[`0`];
2322
2323	/ If TC0 is invalidated disable TC mapping /
2324	if (tc->offset + tc->count > txq) {
2325	netdev_warn(dev, "Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
2326	dev->num_tc = `0`;
2327	return;
2328	}
2329
2330	/ Invalidated prio to tc mappings set to TC0 /
2331	for (i = `1`; i < TC_BITMASK + `1`; i++) {
2332	int q = netdev_get_prio_tc_map(dev, i);
2333
2334	tc = &dev->tc_to_txq[q];
2335	if (tc->offset + tc->count > txq) {
2336	netdev_warn(dev, "Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
2337	i, q);
2338	netdev_set_prio_tc_map(dev, i, `0`);
2339	}
2340	}
2341	}
2342
2343	int netdev_txq_to_tc(struct net_device dev, unsigned* int txq)
2344	{
2345	if (dev->num_tc) {
2346	struct netdev_tc_txq *tc = &dev->tc_to_txq[`0`];
2347	int i;
2348
2349	/ walk through the TCs and see if it falls into any of them /
2350	for (i = `0`; i < TC_MAX_QUEUE; i++, tc++) {
2351	if ((txq - tc->offset) < tc->count)
2352	return i;
2353	}
2354
2355	/ didn't find it, just return -1 to indicate no match /
2356	return -`1`;
2357	}
2358
2359	return `0`;
2360	}
2361	EXPORT_SYMBOL(netdev_txq_to_tc);
2362
2363	#ifdef CONFIG_XPS
2364	static struct static_key xps_needed __read_mostly;
2365	static struct static_key xps_rxqs_needed __read_mostly;
2366	static DEFINE_MUTEX(xps_map_mutex);
2367	#define xmap_dereference(P) \
2368	rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
2369
2370	static bool remove_xps_queue(struct xps_dev_maps *dev_maps,
2371	struct xps_dev_maps old_maps, int* tci, u16 index)
2372	{
2373	struct xps_map *map = NULL;
2374	int pos;
2375
2376	if (dev_maps)
2377	map = xmap_dereference(dev_maps->attr_map[tci]);
2378	if (!map)
2379	return false;
2380
2381	for (pos = map->len; pos--;) {
2382	if (map->queues[pos] != index)
2383	continue;
2384
2385	if (map->len > `1`) {
2386	map->queues[pos] = map->queues[--map->len];
2387	break;
2388	}
2389
2390	if (old_maps)
2391	RCU_INIT_POINTER(old_maps->attr_map[tci], NULL);
2392	RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
2393	kfree_rcu(map, rcu);
2394	return false;
2395	}
2396
2397	return true;
2398	}
2399
2400	static bool remove_xps_queue_cpu(struct net_device *dev,
2401	struct xps_dev_maps *dev_maps,
2402	int cpu, u16 offset, u16 count)
2403	{
2404	int num_tc = dev_maps->num_tc;
2405	bool active = false;
2406	int tci;
2407
2408	for (tci = cpu * num_tc; num_tc--; tci++) {
2409	int i, j;
2410
2411	for (i = count, j = offset; i--; j++) {
2412	if (!remove_xps_queue(dev_maps, NULL, tci, j))
2413	break;
2414	}
2415
2416	active \|= i < `0`;
2417	}
2418
2419	return active;
2420	}
2421
2422	static void reset_xps_maps(struct net_device *dev,
2423	struct xps_dev_maps *dev_maps,
2424	enum xps_map_type type)
2425	{
2426	static_key_slow_dec_cpuslocked(&xps_needed);
2427	if (type == XPS_RXQS)
2428	static_key_slow_dec_cpuslocked(&xps_rxqs_needed);
2429
2430	RCU_INIT_POINTER(dev->xps_maps[type], NULL);
2431
2432	kfree_rcu(dev_maps, rcu);
2433	}
2434
2435	static void clean_xps_maps(struct net_device dev, enum* xps_map_type type,
2436	u16 offset, u16 count)
2437	{
2438	struct xps_dev_maps *dev_maps;
2439	bool active = false;
2440	int i, j;
2441
2442	dev_maps = xmap_dereference(dev->xps_maps[type]);
2443	if (!dev_maps)
2444	return;
2445
2446	for (j = `0`; j < dev_maps->nr_ids; j++)
2447	active \|= remove_xps_queue_cpu(dev, dev_maps, j, offset, count);
2448	if (!active)
2449	reset_xps_maps(dev, dev_maps, type);
2450
2451	if (type == XPS_CPUS) {
2452	for (i = offset + (count - `1`); count--; i--)
2453	netdev_queue_numa_node_write(
2454	netdev_get_tx_queue(dev, i), NUMA_NO_NODE);
2455	}
2456	}
2457
2458	static void netif_reset_xps_queues(struct net_device *dev, u16 offset,
2459	u16 count)
2460	{
2461	if (!static_key_false(&xps_needed))
2462	return;
2463
2464	cpus_read_lock();
2465	mutex_lock(&xps_map_mutex);
2466
2467	if (static_key_false(&xps_rxqs_needed))
2468	clean_xps_maps(dev, XPS_RXQS, offset, count);
2469
2470	clean_xps_maps(dev, XPS_CPUS, offset, count);
2471
2472	mutex_unlock(&xps_map_mutex);
2473	cpus_read_unlock();
2474	}
2475
2476	static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
2477	{
2478	netif_reset_xps_queues(dev, index, dev->num_tx_queues - index);
2479	}
2480
2481	static struct xps_map expand_xps_map(struct* xps_map map, int* attr_index,
2482	u16 index, bool is_rxqs_map)
2483	{
2484	struct xps_map *new_map;
2485	int alloc_len = XPS_MIN_MAP_ALLOC;
2486	int i, pos;
2487
2488	for (pos = `0`; map && pos < map->len; pos++) {
2489	if (map->queues[pos] != index)
2490	continue;
2491	return map;
2492	}
2493
2494	/ Need to add tx-queue to this CPU's/rx-queue's existing map /
2495	if (map) {
2496	if (pos < map->alloc_len)
2497	return map;
2498
2499	alloc_len = map->alloc_len * `2`;
2500	}
2501
2502	/ Need to allocate new map to store tx-queue on this CPU's/rx-queue's*
2503	* map
2504	*/
2505	if (is_rxqs_map)
2506	new_map = kzalloc(XPS_MAP_SIZE(alloc_len), GFP_KERNEL);
2507	else
2508	new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
2509	cpu_to_node(attr_index));
2510	if (!new_map)
2511	return NULL;
2512
2513	for (i = `0`; i < pos; i++)
2514	new_map->queues[i] = map->queues[i];
2515	new_map->alloc_len = alloc_len;
2516	new_map->len = pos;
2517
2518	return new_map;
2519	}
2520
2521	/ Copy xps maps at a given index /
2522	static void xps_copy_dev_maps(struct xps_dev_maps *dev_maps,
2523	struct xps_dev_maps new_dev_maps, int* index,
2524	int tc, bool skip_tc)
2525	{
2526	int i, tci = index * dev_maps->num_tc;
2527	struct xps_map *map;
2528
2529	/ copy maps belonging to foreign traffic classes /
2530	for (i = `0`; i < dev_maps->num_tc; i++, tci++) {
2531	if (i == tc && skip_tc)
2532	continue;
2533
2534	/ fill in the new device map from the old device map /
2535	map = xmap_dereference(dev_maps->attr_map[tci]);
2536	RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2537	}
2538	}
2539
2540	/ Must be called under cpus_read_lock /
2541	int __netif_set_xps_queue(struct net_device dev, const* unsigned long *mask,
2542	u16 index, enum xps_map_type type)
2543	{
2544	struct xps_dev_maps dev_maps, new_dev_maps = NULL, *old_dev_maps = NULL;
2545	const unsigned long *online_mask = NULL;
2546	bool active = false, copy = false;
2547	int i, j, tci, numa_node_id = -`2`;
2548	int maps_sz, num_tc = `1`, tc = `0`;
2549	struct xps_map map, new_map;
2550	unsigned int nr_ids;
2551
2552	if (dev->num_tc) {
2553	/ Do not allow XPS on subordinate device directly /
2554	num_tc = dev->num_tc;
2555	if (num_tc < `0`)
2556	return -EINVAL;
2557
2558	/ If queue belongs to subordinate dev use its map /
2559	dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev;
2560
2561	tc = netdev_txq_to_tc(dev, index);
2562	if (tc < `0`)
2563	return -EINVAL;
2564	}
2565
2566	mutex_lock(&xps_map_mutex);
2567
2568	dev_maps = xmap_dereference(dev->xps_maps[type]);
2569	if (type == XPS_RXQS) {
2570	maps_sz = XPS_RXQ_DEV_MAPS_SIZE(num_tc, dev->num_rx_queues);
2571	nr_ids = dev->num_rx_queues;
2572	} else {
2573	maps_sz = XPS_CPU_DEV_MAPS_SIZE(num_tc);
2574	if (num_possible_cpus() > `1`)
2575	online_mask = cpumask_bits(cpu_online_mask);
2576	nr_ids = nr_cpu_ids;
2577	}
2578
2579	if (maps_sz < L1_CACHE_BYTES)
2580	maps_sz = L1_CACHE_BYTES;
2581
2582	/ The old dev_maps could be larger or smaller than the one we're*
2583	* setting up now, as dev->num_tc or nr_ids could have been updated in
2584	* between. We could try to be smart, but let's be safe instead and only
2585	* copy foreign traffic classes if the two map sizes match.
2586	*/
2587	if (dev_maps &&
2588	dev_maps->num_tc == num_tc && dev_maps->nr_ids == nr_ids)
2589	copy = true;
2590
2591	/ allocate memory for queue storage /
2592	for (j = -`1`; j = netif_attrmask_next_and(j, online_mask, mask, nr_ids),
2593	j < nr_ids;) {
2594	if (!new_dev_maps) {
2595	new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
2596	if (!new_dev_maps) {
2597	mutex_unlock(&xps_map_mutex);
2598	return -ENOMEM;
2599	}
2600
2601	new_dev_maps->nr_ids = nr_ids;
2602	new_dev_maps->num_tc = num_tc;
2603	}
2604
2605	tci = j * num_tc + tc;
2606	map = copy ? xmap_dereference(dev_maps->attr_map[tci]) : NULL;
2607
2608	map = expand_xps_map(map, j, index, type == XPS_RXQS);
2609	if (!map)
2610	goto error;
2611
2612	RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2613	}
2614
2615	if (!new_dev_maps)
2616	goto out_no_new_maps;
2617
2618	if (!dev_maps) {
2619	/ Increment static keys at most once per type /
2620	static_key_slow_inc_cpuslocked(&xps_needed);
2621	if (type == XPS_RXQS)
2622	static_key_slow_inc_cpuslocked(&xps_rxqs_needed);
2623	}
2624
2625	for (j = `0`; j < nr_ids; j++) {
2626	bool skip_tc = false;
2627
2628	tci = j * num_tc + tc;
2629	if (netif_attr_test_mask(j, mask, nr_ids) &&
2630	netif_attr_test_online(j, online_mask, nr_ids)) {
2631	/ add tx-queue to CPU/rx-queue maps /
2632	int pos = `0`;
2633
2634	skip_tc = true;
2635
2636	map = xmap_dereference(new_dev_maps->attr_map[tci]);
2637	while ((pos < map->len) && (map->queues[pos] != index))
2638	pos++;
2639
2640	if (pos == map->len)
2641	map->queues[map->len++] = index;
2642	#ifdef CONFIG_NUMA
2643	if (type == XPS_CPUS) {
2644	if (numa_node_id == -`2`)
2645	numa_node_id = cpu_to_node(j);
2646	else if (numa_node_id != cpu_to_node(j))
2647	numa_node_id = -`1`;
2648	}
2649	#endif
2650	}
2651
2652	if (copy)
2653	xps_copy_dev_maps(dev_maps, new_dev_maps, j, tc,
2654	skip_tc);
2655	}
2656
2657	rcu_assign_pointer(dev->xps_maps[type], new_dev_maps);
2658
2659	/ Cleanup old maps /
2660	if (!dev_maps)
2661	goto out_no_old_maps;
2662
2663	for (j = `0`; j < dev_maps->nr_ids; j++) {
2664	for (i = num_tc, tci = j * dev_maps->num_tc; i--; tci++) {
2665	map = xmap_dereference(dev_maps->attr_map[tci]);
2666	if (!map)
2667	continue;
2668
2669	if (copy) {
2670	new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
2671	if (map == new_map)
2672	continue;
2673	}
2674
2675	RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
2676	kfree_rcu(map, rcu);
2677	}
2678	}
2679
2680	old_dev_maps = dev_maps;
2681
2682	out_no_old_maps:
2683	dev_maps = new_dev_maps;
2684	active = true;
2685
2686	out_no_new_maps:
2687	if (type == XPS_CPUS)
2688	/ update Tx queue numa node /
2689	netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2690	(numa_node_id >= `0`) ?
2691	numa_node_id : NUMA_NO_NODE);
2692
2693	if (!dev_maps)
2694	goto out_no_maps;
2695
2696	/ removes tx-queue from unused CPUs/rx-queues /
2697	for (j = `0`; j < dev_maps->nr_ids; j++) {
2698	tci = j * dev_maps->num_tc;
2699
2700	for (i = `0`; i < dev_maps->num_tc; i++, tci++) {
2701	if (i == tc &&
2702	netif_attr_test_mask(j, mask, dev_maps->nr_ids) &&
2703	netif_attr_test_online(j, online_mask, dev_maps->nr_ids))
2704	continue;
2705
2706	active \|= remove_xps_queue(dev_maps,
2707	copy ? old_dev_maps : NULL,
2708	tci, index);
2709	}
2710	}
2711
2712	if (old_dev_maps)
2713	kfree_rcu(old_dev_maps, rcu);
2714
2715	/ free map if not active /
2716	if (!active)
2717	reset_xps_maps(dev, dev_maps, type);
2718
2719	out_no_maps:
2720	mutex_unlock(&xps_map_mutex);
2721
2722	return `0`;
2723	error:
2724	/ remove any maps that we added /
2725	for (j = `0`; j < nr_ids; j++) {
2726	for (i = num_tc, tci = j * num_tc; i--; tci++) {
2727	new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
2728	map = copy ?
2729	xmap_dereference(dev_maps->attr_map[tci]) :
2730	NULL;
2731	if (new_map && new_map != map)
2732	kfree(new_map);
2733	}
2734	}
2735
2736	mutex_unlock(&xps_map_mutex);
2737
2738	kfree(new_dev_maps);
2739	return -ENOMEM;
2740	}
2741	EXPORT_SYMBOL_GPL(__netif_set_xps_queue);
2742
2743	int netif_set_xps_queue(struct net_device dev, const* struct cpumask *mask,
2744	u16 index)
2745	{
2746	int ret;
2747
2748	cpus_read_lock();
2749	ret = __netif_set_xps_queue(dev, cpumask_bits(mask), index, XPS_CPUS);
2750	cpus_read_unlock();
2751
2752	return ret;
2753	}
2754	EXPORT_SYMBOL(netif_set_xps_queue);
2755
2756	#endif
2757	static void netdev_unbind_all_sb_channels(struct net_device *dev)
2758	{
2759	struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
2760
2761	/ Unbind any subordinate channels /
2762	while (txq-- != &dev->_tx[`0`]) {
2763	if (txq->sb_dev)
2764	netdev_unbind_sb_channel(dev, txq->sb_dev);
2765	}
2766	}
2767
2768	void netdev_reset_tc(struct net_device *dev)
2769	{
2770	#ifdef CONFIG_XPS
2771	netif_reset_xps_queues_gt(dev, `0`);
2772	#endif
2773	netdev_unbind_all_sb_channels(dev);
2774
2775	/ Reset TC configuration of device /
2776	dev->num_tc = `0`;
2777	memset(dev->tc_to_txq, `0`, sizeof(dev->tc_to_txq));
2778	memset(dev->prio_tc_map, `0`, sizeof(dev->prio_tc_map));
2779	}
2780	EXPORT_SYMBOL(netdev_reset_tc);
2781
2782	int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset)
2783	{
2784	if (tc >= dev->num_tc)
2785	return -EINVAL;
2786
2787	#ifdef CONFIG_XPS
2788	netif_reset_xps_queues(dev, offset, count);
2789	#endif
2790	dev->tc_to_txq[tc].count = count;
2791	dev->tc_to_txq[tc].offset = offset;
2792	return `0`;
2793	}
2794	EXPORT_SYMBOL(netdev_set_tc_queue);
2795
2796	int netdev_set_num_tc(struct net_device *dev, u8 num_tc)
2797	{
2798	if (num_tc > TC_MAX_QUEUE)
2799	return -EINVAL;
2800
2801	#ifdef CONFIG_XPS
2802	netif_reset_xps_queues_gt(dev, `0`);
2803	#endif
2804	netdev_unbind_all_sb_channels(dev);
2805
2806	dev->num_tc = num_tc;
2807	return `0`;
2808	}
2809	EXPORT_SYMBOL(netdev_set_num_tc);
2810
2811	void netdev_unbind_sb_channel(struct net_device *dev,
2812	struct net_device *sb_dev)
2813	{
2814	struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
2815
2816	#ifdef CONFIG_XPS
2817	netif_reset_xps_queues_gt(sb_dev, `0`);
2818	#endif
2819	memset(sb_dev->tc_to_txq, `0`, sizeof(sb_dev->tc_to_txq));
2820	memset(sb_dev->prio_tc_map, `0`, sizeof(sb_dev->prio_tc_map));
2821
2822	while (txq-- != &dev->_tx[`0`]) {
2823	if (txq->sb_dev == sb_dev)
2824	txq->sb_dev = NULL;
2825	}
2826	}
2827	EXPORT_SYMBOL(netdev_unbind_sb_channel);
2828
2829	int netdev_bind_sb_channel_queue(struct net_device *dev,
2830	struct net_device *sb_dev,
2831	u8 tc, u16 count, u16 offset)
2832	{
2833	/ Make certain the sb_dev and dev are already configured /
2834	if (sb_dev->num_tc >= `0` \|\| tc >= dev->num_tc)
2835	return -EINVAL;
2836
2837	/ We cannot hand out queues we don't have /
2838	if ((offset + count) > dev->real_num_tx_queues)
2839	return -EINVAL;
2840
2841	/ Record the mapping /
2842	sb_dev->tc_to_txq[tc].count = count;
2843	sb_dev->tc_to_txq[tc].offset = offset;
2844
2845	/ Provide a way for Tx queue to find the tc_to_txq map or*
2846	* XPS map for itself.
2847	*/
2848	while (count--)
2849	netdev_get_tx_queue(dev, count + offset)->sb_dev = sb_dev;
2850
2851	return `0`;
2852	}
2853	EXPORT_SYMBOL(netdev_bind_sb_channel_queue);
2854
2855	int netdev_set_sb_channel(struct net_device *dev, u16 channel)
2856	{
2857	/ Do not use a multiqueue device to represent a subordinate channel /
2858	if (netif_is_multiqueue(dev))
2859	return -ENODEV;
2860
2861	/ We allow channels 1 - 32767 to be used for subordinate channels.*
2862	* Channel 0 is meant to be "native" mode and used only to represent
2863	* the main root device. We allow writing 0 to reset the device back
2864	* to normal mode after being used as a subordinate channel.
2865	*/
2866	if (channel > S16_MAX)
2867	return -EINVAL;
2868
2869	dev->num_tc = -channel;
2870
2871	return `0`;
2872	}
2873	EXPORT_SYMBOL(netdev_set_sb_channel);
2874
2875	/*
2876	* Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2877	* greater than real_num_tx_queues stale skbs on the qdisc must be flushed.
2878	*/
2879	int netif_set_real_num_tx_queues(struct net_device dev, unsigned* int txq)
2880	{
2881	bool disabling;
2882	int rc;
2883
2884	disabling = txq < dev->real_num_tx_queues;
2885
2886	if (txq < `1` \|\| txq > dev->num_tx_queues)
2887	return -EINVAL;
2888
2889	if (dev->reg_state == NETREG_REGISTERED \|\|
2890	dev->reg_state == NETREG_UNREGISTERING) {
2891	ASSERT_RTNL();
2892
2893	rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2894	txq);
2895	if (rc)
2896	return rc;
2897
2898	if (dev->num_tc)
2899	netif_setup_tc(dev, txq);
2900
2901	dev_qdisc_change_real_num_tx(dev, txq);
2902
2903	dev->real_num_tx_queues = txq;
2904
2905	if (disabling) {
2906	synchronize_net();
2907	qdisc_reset_all_tx_gt(dev, txq);
2908	#ifdef CONFIG_XPS
2909	netif_reset_xps_queues_gt(dev, txq);
2910	#endif
2911	}
2912	} else {
2913	dev->real_num_tx_queues = txq;
2914	}
2915
2916	return `0`;
2917	}
2918	EXPORT_SYMBOL(netif_set_real_num_tx_queues);
2919
2920	#ifdef CONFIG_SYSFS
2921	/**
2922	* netif_set_real_num_rx_queues - set actual number of RX queues used
2923	* @dev: Network device
2924	* @rxq: Actual number of RX queues
2925	*
2926	* This must be called either with the rtnl_lock held or before
2927	* registration of the net device. Returns 0 on success, or a
2928	* negative error code. If called before registration, it always
2929	* succeeds.
2930	*/
2931	int netif_set_real_num_rx_queues(struct net_device dev, unsigned* int rxq)
2932	{
2933	int rc;
2934
2935	if (rxq < `1` \|\| rxq > dev->num_rx_queues)
2936	return -EINVAL;
2937
2938	if (dev->reg_state == NETREG_REGISTERED) {
2939	ASSERT_RTNL();
2940
2941	rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2942	rxq);
2943	if (rc)
2944	return rc;
2945	}
2946
2947	dev->real_num_rx_queues = rxq;
2948	return `0`;
2949	}
2950	EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2951	#endif
2952
2953	/**
2954	* netif_set_real_num_queues - set actual number of RX and TX queues used
2955	* @dev: Network device
2956	* @txq: Actual number of TX queues
2957	* @rxq: Actual number of RX queues
2958	*
2959	* Set the real number of both TX and RX queues.
2960	* Does nothing if the number of queues is already correct.
2961	*/
2962	int netif_set_real_num_queues(struct net_device *dev,
2963	unsigned int txq, unsigned int rxq)
2964	{
2965	unsigned int old_rxq = dev->real_num_rx_queues;
2966	int err;
2967
2968	if (txq < `1` \|\| txq > dev->num_tx_queues \|\|
2969	rxq < `1` \|\| rxq > dev->num_rx_queues)
2970	return -EINVAL;
2971
2972	/ Start from increases, so the error path only does decreases -*
2973	* decreases can't fail.
2974	*/
2975	if (rxq > dev->real_num_rx_queues) {
2976	err = netif_set_real_num_rx_queues(dev, rxq);
2977	if (err)
2978	return err;
2979	}
2980	if (txq > dev->real_num_tx_queues) {
2981	err = netif_set_real_num_tx_queues(dev, txq);
2982	if (err)
2983	goto undo_rx;
2984	}
2985	if (rxq < dev->real_num_rx_queues)
2986	WARN_ON(netif_set_real_num_rx_queues(dev, rxq));
2987	if (txq < dev->real_num_tx_queues)
2988	WARN_ON(netif_set_real_num_tx_queues(dev, txq));
2989
2990	return `0`;
2991	undo_rx:
2992	WARN_ON(netif_set_real_num_rx_queues(dev, old_rxq));
2993	return err;
2994	}
2995	EXPORT_SYMBOL(netif_set_real_num_queues);
2996
2997	/**
2998	* netif_set_tso_max_size() - set the max size of TSO frames supported
2999	* @dev: netdev to update
3000	* @size: max skb->len of a TSO frame
3001	*
3002	* Set the limit on the size of TSO super-frames the device can handle.
3003	* Unless explicitly set the stack will assume the value of
3004	* %GSO_LEGACY_MAX_SIZE.
3005	*/
3006	void netif_set_tso_max_size(struct net_device dev, unsigned* int size)
3007	{
3008	dev->tso_max_size = min(GSO_MAX_SIZE, size);
3009	if (size < READ_ONCE(dev->gso_max_size))
3010	netif_set_gso_max_size(dev, size);
3011	}
3012	EXPORT_SYMBOL(netif_set_tso_max_size);
3013
3014	/**
3015	* netif_set_tso_max_segs() - set the max number of segs supported for TSO
3016	* @dev: netdev to update
3017	* @segs: max number of TCP segments
3018	*
3019	* Set the limit on the number of TCP segments the device can generate from
3020	* a single TSO super-frame.
3021	* Unless explicitly set the stack will assume the value of %GSO_MAX_SEGS.
3022	*/
3023	void netif_set_tso_max_segs(struct net_device dev, unsigned* int segs)
3024	{
3025	dev->tso_max_segs = segs;
3026	if (segs < READ_ONCE(dev->gso_max_segs))
3027	netif_set_gso_max_segs(dev, segs);
3028	}
3029	EXPORT_SYMBOL(netif_set_tso_max_segs);
3030
3031	/**
3032	* netif_inherit_tso_max() - copy all TSO limits from a lower device to an upper
3033	* @to: netdev to update
3034	* @from: netdev from which to copy the limits
3035	*/
3036	void netif_inherit_tso_max(struct net_device to, const* struct net_device *from)
3037	{
3038	netif_set_tso_max_size(to, from->tso_max_size);
3039	netif_set_tso_max_segs(to, from->tso_max_segs);
3040	}
3041	EXPORT_SYMBOL(netif_inherit_tso_max);
3042
3043	/**
3044	* netif_get_num_default_rss_queues - default number of RSS queues
3045	*
3046	* Default value is the number of physical cores if there are only 1 or 2, or
3047	* divided by 2 if there are more.
3048	*/
3049	int netif_get_num_default_rss_queues(void)
3050	{
3051	cpumask_var_t cpus;
3052	int cpu, count = `0`;
3053
3054	if (unlikely(is_kdump_kernel() \|\| !zalloc_cpumask_var(&cpus, GFP_KERNEL)))
3055	return `1`;
3056
3057	cpumask_copy(cpus, cpu_online_mask);
3058	for_each_cpu(cpu, cpus) {
3059	++count;
3060	cpumask_andnot(cpus, cpus, topology_sibling_cpumask(cpu));
3061	}
3062	free_cpumask_var(cpus);
3063
3064	return count > `2` ? DIV_ROUND_UP(count, `2`) : count;
3065	}
3066	EXPORT_SYMBOL(netif_get_num_default_rss_queues);
3067
3068	static void __netif_reschedule(struct Qdisc *q)
3069	{
3070	struct softnet_data *sd;
3071	unsigned long flags;
3072
3073	local_irq_save(flags);
3074	sd = this_cpu_ptr(&softnet_data);
3075	q->next_sched = NULL;
3076	*sd->output_queue_tailp = q;
3077	sd->output_queue_tailp = &q->next_sched;
3078	raise_softirq_irqoff(NET_TX_SOFTIRQ);
3079	local_irq_restore(flags);
3080	}
3081
3082	void __netif_schedule(struct Qdisc *q)
3083	{
3084	if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
3085	__netif_reschedule(q);
3086	}
3087	EXPORT_SYMBOL(__netif_schedule);
3088
3089	struct dev_kfree_skb_cb {
3090	enum skb_free_reason reason;
3091	};
3092
3093	static struct dev_kfree_skb_cb get_kfree_skb_cb(const* struct sk_buff *skb)
3094	{
3095	return (struct dev_kfree_skb_cb *)skb->cb;
3096	}
3097
3098	void netif_schedule_queue(struct netdev_queue *txq)
3099	{
3100	rcu_read_lock();
3101	if (!netif_xmit_stopped(txq)) {
3102	struct Qdisc *q = rcu_dereference(txq->qdisc);
3103
3104	__netif_schedule(q);
3105	}
3106	rcu_read_unlock();
3107	}
3108	EXPORT_SYMBOL(netif_schedule_queue);
3109
3110	void netif_tx_wake_queue(struct netdev_queue *dev_queue)
3111	{
3112	if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
3113	struct Qdisc *q;
3114
3115	rcu_read_lock();
3116	q = rcu_dereference(dev_queue->qdisc);
3117	__netif_schedule(q);
3118	rcu_read_unlock();
3119	}
3120	}
3121	EXPORT_SYMBOL(netif_tx_wake_queue);
3122
3123	void __dev_kfree_skb_irq(struct sk_buff skb, enum* skb_free_reason reason)
3124	{
3125	unsigned long flags;
3126
3127	if (unlikely(!skb))
3128	return;
3129
3130	if (likely(refcount_read(&skb->users) == `1`)) {
3131	smp_rmb();
3132	refcount_set(&skb->users, `0`);
3133	} else if (likely(!refcount_dec_and_test(&skb->users))) {
3134	return;
3135	}
3136	get_kfree_skb_cb(skb)->reason = reason;
3137	local_irq_save(flags);
3138	skb->next = __this_cpu_read(softnet_data.completion_queue);
3139	__this_cpu_write(softnet_data.completion_queue, skb);
3140	raise_softirq_irqoff(NET_TX_SOFTIRQ);
3141	local_irq_restore(flags);
3142	}
3143	EXPORT_SYMBOL(__dev_kfree_skb_irq);
3144
3145	void __dev_kfree_skb_any(struct sk_buff skb, enum* skb_free_reason reason)
3146	{
3147	if (in_hardirq() \|\| irqs_disabled())
3148	__dev_kfree_skb_irq(skb, reason);
3149	else
3150	dev_kfree_skb(skb);
3151	}
3152	EXPORT_SYMBOL(__dev_kfree_skb_any);
3153
3154
3155	/**
3156	* netif_device_detach - mark device as removed
3157	* @dev: network device
3158	*
3159	* Mark device as removed from system and therefore no longer available.
3160	*/
3161	void netif_device_detach(struct net_device *dev)
3162	{
3163	if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
3164	netif_running(dev)) {
3165	netif_tx_stop_all_queues(dev);
3166	}
3167	}
3168	EXPORT_SYMBOL(netif_device_detach);
3169
3170	/**
3171	* netif_device_attach - mark device as attached
3172	* @dev: network device
3173	*
3174	* Mark device as attached from system and restart if needed.
3175	*/
3176	void netif_device_attach(struct net_device *dev)
3177	{
3178	if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
3179	netif_running(dev)) {
3180	netif_tx_wake_all_queues(dev);
3181	__netdev_watchdog_up(dev);
3182	}
3183	}
3184	EXPORT_SYMBOL(netif_device_attach);
3185
3186	/*
3187	* Returns a Tx hash based on the given packet descriptor a Tx queues' number
3188	* to be used as a distribution range.
3189	*/
3190	static u16 skb_tx_hash(const struct net_device *dev,
3191	const struct net_device *sb_dev,
3192	struct sk_buff *skb)
3193	{
3194	u32 hash;
3195	u16 qoffset = `0`;
3196	u16 qcount = dev->real_num_tx_queues;
3197
3198	if (dev->num_tc) {
3199	u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
3200
3201	qoffset = sb_dev->tc_to_txq[tc].offset;
3202	qcount = sb_dev->tc_to_txq[tc].count;
3203	if (unlikely(!qcount)) {
3204	net_warn_ratelimited("%s: invalid qcount, qoffset %u for tc %u\n",
3205	sb_dev->name, qoffset, tc);
3206	qoffset = `0`;
3207	qcount = dev->real_num_tx_queues;
3208	}
3209	}
3210
3211	if (skb_rx_queue_recorded(skb)) {
3212	hash = skb_get_rx_queue(skb);
3213	if (hash >= qoffset)
3214	hash -= qoffset;
3215	while (unlikely(hash >= qcount))
3216	hash -= qcount;
3217	return hash + qoffset;
3218	}
3219
3220	return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
3221	}
3222
3223	static void skb_warn_bad_offload(const struct sk_buff *skb)
3224	{
3225	static const netdev_features_t null_features;
3226	struct net_device *dev = skb->dev;
3227	const char *name = "";
3228
3229	if (!net_ratelimit())
3230	return;
3231
3232	if (dev) {
3233	if (dev->dev.parent)
3234	name = dev_driver_string(dev->dev.parent);
3235	else
3236	name = netdev_name(dev);
3237	}
3238	skb_dump(KERN_WARNING, skb, false);
3239	WARN(`1`, "%s: caps=(%pNF, %pNF)\n",
3240	name, dev ? &dev->features : &null_features,
3241	skb->sk ? &skb->sk->sk_route_caps : &null_features);
3242	}
3243
3244	/*
3245	* Invalidate hardware checksum when packet is to be mangled, and
3246	* complete checksum manually on outgoing path.
3247	*/
3248	int skb_checksum_help(struct sk_buff *skb)
3249	{
3250	__wsum csum;
3251	int ret = `0`, offset;
3252
3253	if (skb->ip_summed == CHECKSUM_COMPLETE)
3254	goto out_set_summed;
3255
3256	if (unlikely(skb_is_gso(skb))) {
3257	skb_warn_bad_offload(skb);
3258	return -EINVAL;
3259	}
3260
3261	/ Before computing a checksum, we should make sure no frag could*
3262	* be modified by an external entity : checksum could be wrong.
3263	*/
3264	if (skb_has_shared_frag(skb)) {
3265	ret = __skb_linearize(skb);
3266	if (ret)
3267	goto out;
3268	}
3269
3270	offset = skb_checksum_start_offset(skb);
3271	ret = -EINVAL;
3272	if (WARN_ON_ONCE(offset >= skb_headlen(skb))) {
3273	DO_ONCE_LITE(skb_dump, KERN_ERR, skb, false);
3274	goto out;
3275	}
3276	csum = skb_checksum(skb, offset, skb->len - offset, `0`);
3277
3278	offset += skb->csum_offset;
3279	if (WARN_ON_ONCE(offset + sizeof(__sum16) > skb_headlen(skb))) {
3280	DO_ONCE_LITE(skb_dump, KERN_ERR, skb, false);
3281	goto out;
3282	}
3283	ret = skb_ensure_writable(skb, offset + sizeof(__sum16));
3284	if (ret)
3285	goto out;
3286
3287	(__sum16 )(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
3288	out_set_summed:
3289	skb->ip_summed = CHECKSUM_NONE;
3290	out:
3291	return ret;
3292	}
3293	EXPORT_SYMBOL(skb_checksum_help);
3294
3295	int skb_crc32c_csum_help(struct sk_buff *skb)
3296	{
3297	__le32 crc32c_csum;
3298	int ret = `0`, offset, start;
3299
3300	if (skb->ip_summed != CHECKSUM_PARTIAL)
3301	goto out;
3302
3303	if (unlikely(skb_is_gso(skb)))
3304	goto out;
3305
3306	/ Before computing a checksum, we should make sure no frag could*
3307	* be modified by an external entity : checksum could be wrong.
3308	*/
3309	if (unlikely(skb_has_shared_frag(skb))) {
3310	ret = __skb_linearize(skb);
3311	if (ret)
3312	goto out;
3313	}
3314	start = skb_checksum_start_offset(skb);
3315	offset = start + offsetof(struct sctphdr, checksum);
3316	if (WARN_ON_ONCE(offset >= skb_headlen(skb))) {
3317	ret = -EINVAL;
3318	goto out;
3319	}
3320
3321	ret = skb_ensure_writable(skb, offset + sizeof(__le32));
3322	if (ret)
3323	goto out;
3324
3325	crc32c_csum = cpu_to_le32(~__skb_checksum(skb, start,
3326	skb->len - start, ~(__u32)`0`,
3327	crc32c_csum_stub));
3328	(__le32 )(skb->data + offset) = crc32c_csum;
3329	skb->ip_summed = CHECKSUM_NONE;
3330	skb->csum_not_inet = `0`;
3331	out:
3332	return ret;
3333	}
3334
3335	__be16 skb_network_protocol(struct sk_buff skb, int* *depth)
3336	{
3337	__be16 type = skb->protocol;
3338
3339	/ Tunnel gso handlers can set protocol to ethernet. /
3340	if (type == htons(ETH_P_TEB)) {
3341	struct ethhdr *eth;
3342
3343	if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
3344	return `0`;
3345
3346	eth = (struct ethhdr *)skb->data;
3347	type = eth->h_proto;
3348	}
3349
3350	return __vlan_get_protocol(skb, type, depth);
3351	}
3352
3353	/ openvswitch calls this on rx path, so we need a different check.*
3354	*/
3355	static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
3356	{
3357	if (tx_path)
3358	return skb->ip_summed != CHECKSUM_PARTIAL &&
3359	skb->ip_summed != CHECKSUM_UNNECESSARY;
3360
3361	return skb->ip_summed == CHECKSUM_NONE;
3362	}
3363
3364	/**
3365	* __skb_gso_segment - Perform segmentation on skb.
3366	* @skb: buffer to segment
3367	* @features: features for the output path (see dev->features)
3368	* @tx_path: whether it is called in TX path
3369	*
3370	* This function segments the given skb and returns a list of segments.
3371	*
3372	* It may return NULL if the skb requires no segmentation. This is
3373	* only possible when GSO is used for verifying header integrity.
3374	*
3375	* Segmentation preserves SKB_GSO_CB_OFFSET bytes of previous skb cb.
3376	*/
3377	struct sk_buff __skb_gso_segment(struct* sk_buff *skb,
3378	netdev_features_t features, bool tx_path)
3379	{
3380	struct sk_buff *segs;
3381
3382	if (unlikely(skb_needs_check(skb, tx_path))) {
3383	int err;
3384
3385	/ We're going to init ->check field in TCP or UDP header /
3386	err = skb_cow_head(skb, `0`);
3387	if (err < `0`)
3388	return ERR_PTR(err);
3389	}
3390
3391	/ Only report GSO partial support if it will enable us to*
3392	* support segmentation on this frame without needing additional
3393	* work.
3394	*/
3395	if (features & NETIF_F_GSO_PARTIAL) {
3396	netdev_features_t partial_features = NETIF_F_GSO_ROBUST;
3397	struct net_device *dev = skb->dev;
3398
3399	partial_features \|= dev->features & dev->gso_partial_features;
3400	if (!skb_gso_ok(skb, features \| partial_features))
3401	features &= ~NETIF_F_GSO_PARTIAL;
3402	}
3403
3404	BUILD_BUG_ON(SKB_GSO_CB_OFFSET +
3405	sizeof(SKB_GSO_CB(skb)) > sizeof*(skb->cb));
3406
3407	SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
3408	SKB_GSO_CB(skb)->encap_level = `0`;
3409
3410	skb_reset_mac_header(skb);
3411	skb_reset_mac_len(skb);
3412
3413	segs = skb_mac_gso_segment(skb, features);
3414
3415	if (segs != skb && unlikely(skb_needs_check(skb, tx_path) && !IS_ERR(segs)))
3416	skb_warn_bad_offload(skb);
3417
3418	return segs;
3419	}
3420	EXPORT_SYMBOL(__skb_gso_segment);
3421
3422	/ Take action when hardware reception checksum errors are detected. /
3423	#ifdef CONFIG_BUG
3424	static void do_netdev_rx_csum_fault(struct net_device dev, struct* sk_buff *skb)
3425	{
3426	netdev_err(dev, "hw csum failure\n");
3427	skb_dump(KERN_ERR, skb, true);
3428	dump_stack();
3429	}
3430
3431	void netdev_rx_csum_fault(struct net_device dev, struct* sk_buff *skb)
3432	{
3433	DO_ONCE_LITE(do_netdev_rx_csum_fault, dev, skb);
3434	}
3435	EXPORT_SYMBOL(netdev_rx_csum_fault);
3436	#endif
3437
3438	/ XXX: check that highmem exists at all on the given machine. /
3439	static int illegal_highdma(struct net_device dev, struct* sk_buff *skb)
3440	{
3441	#ifdef CONFIG_HIGHMEM
3442	int i;
3443
3444	if (!(dev->features & NETIF_F_HIGHDMA)) {
3445	for (i = `0`; i < skb_shinfo(skb)->nr_frags; i++) {
3446	skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3447
3448	if (PageHighMem(skb_frag_page(frag)))
3449	return `1`;
3450	}
3451	}
3452	#endif
3453	return `0`;
3454	}
3455
3456	/ If MPLS offload request, verify we are testing hardware MPLS features*
3457	* instead of standard features for the netdev.
3458	*/
3459	#if IS_ENABLED(CONFIG_NET_MPLS_GSO)
3460	static netdev_features_t net_mpls_features(struct sk_buff *skb,
3461	netdev_features_t features,
3462	__be16 type)
3463	{
3464	if (eth_p_mpls(type))
3465	features &= skb->dev->mpls_features;
3466
3467	return features;
3468	}
3469	#else
3470	static netdev_features_t net_mpls_features(struct sk_buff *skb,
3471	netdev_features_t features,
3472	__be16 type)
3473	{
3474	return features;
3475	}
3476	#endif
3477
3478	static netdev_features_t harmonize_features(struct sk_buff *skb,
3479	netdev_features_t features)
3480	{
3481	__be16 type;
3482
3483	type = skb_network_protocol(skb, NULL);
3484	features = net_mpls_features(skb, features, type);
3485
3486	if (skb->ip_summed != CHECKSUM_NONE &&
3487	!can_checksum_protocol(features, type)) {
3488	features &= ~(NETIF_F_CSUM_MASK \| NETIF_F_GSO_MASK);
3489	}
3490	if (illegal_highdma(skb->dev, skb))
3491	features &= ~NETIF_F_SG;
3492
3493	return features;
3494	}
3495
3496	netdev_features_t passthru_features_check(struct sk_buff *skb,
3497	struct net_device *dev,
3498	netdev_features_t features)
3499	{
3500	return features;
3501	}
3502	EXPORT_SYMBOL(passthru_features_check);
3503
3504	static netdev_features_t dflt_features_check(struct sk_buff *skb,
3505	struct net_device *dev,
3506	netdev_features_t features)
3507	{
3508	return vlan_features_check(skb, features);
3509	}
3510
3511	static netdev_features_t gso_features_check(const struct sk_buff *skb,
3512	struct net_device *dev,
3513	netdev_features_t features)
3514	{
3515	u16 gso_segs = skb_shinfo(skb)->gso_segs;
3516
3517	if (gso_segs > READ_ONCE(dev->gso_max_segs))
3518	return features & ~NETIF_F_GSO_MASK;
3519
3520	if (!skb_shinfo(skb)->gso_type) {
3521	skb_warn_bad_offload(skb);
3522	return features & ~NETIF_F_GSO_MASK;
3523	}
3524
3525	/ Support for GSO partial features requires software*
3526	* intervention before we can actually process the packets
3527	* so we need to strip support for any partial features now
3528	* and we can pull them back in after we have partially
3529	* segmented the frame.
3530	*/
3531	if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL))
3532	features &= ~dev->gso_partial_features;
3533
3534	/ Make sure to clear the IPv4 ID mangling feature if the*
3535	* IPv4 header has the potential to be fragmented.
3536	*/
3537	if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
3538	struct iphdr *iph = skb->encapsulation ?
3539	inner_ip_hdr(skb) : ip_hdr(skb);
3540
3541	if (!(iph->frag_off & htons(IP_DF)))
3542	features &= ~NETIF_F_TSO_MANGLEID;
3543	}
3544
3545	return features;
3546	}
3547
3548	netdev_features_t netif_skb_features(struct sk_buff *skb)
3549	{
3550	struct net_device *dev = skb->dev;
3551	netdev_features_t features = dev->features;
3552
3553	if (skb_is_gso(skb))
3554	features = gso_features_check(skb, dev, features);
3555
3556	/ If encapsulation offload request, verify we are testing*
3557	* hardware encapsulation features instead of standard
3558	* features for the netdev
3559	*/
3560	if (skb->encapsulation)
3561	features &= dev->hw_enc_features;
3562
3563	if (skb_vlan_tagged(skb))
3564	features = netdev_intersect_features(features,
3565	dev->vlan_features \|
3566	NETIF_F_HW_VLAN_CTAG_TX \|
3567	NETIF_F_HW_VLAN_STAG_TX);
3568
3569	if (dev->netdev_ops->ndo_features_check)
3570	features &= dev->netdev_ops->ndo_features_check(skb, dev,
3571	features);
3572	else
3573	features &= dflt_features_check(skb, dev, features);
3574
3575	return harmonize_features(skb, features);
3576	}
3577	EXPORT_SYMBOL(netif_skb_features);
3578
3579	static int xmit_one(struct sk_buff skb, struct* net_device *dev,
3580	struct netdev_queue *txq, bool more)
3581	{
3582	unsigned int len;
3583	int rc;
3584
3585	if (dev_nit_active(dev))
3586	dev_queue_xmit_nit(skb, dev);
3587
3588	len = skb->len;
3589	trace_net_dev_start_xmit(skb, dev);
3590	rc = netdev_start_xmit(skb, dev, txq, more);
3591	trace_net_dev_xmit(skb, rc, dev, len);
3592
3593	return rc;
3594	}
3595
3596	struct sk_buff dev_hard_start_xmit(struct* sk_buff first, struct* net_device *dev,
3597	struct netdev_queue txq, int* *ret)
3598	{
3599	struct sk_buff *skb = first;
3600	int rc = NETDEV_TX_OK;
3601
3602	while (skb) {
3603	struct sk_buff *next = skb->next;
3604
3605	skb_mark_not_on_list(skb);
3606	rc = xmit_one(skb, dev, txq, next != NULL);
3607	if (unlikely(!dev_xmit_complete(rc))) {
3608	skb->next = next;
3609	goto out;
3610	}
3611
3612	skb = next;
3613	if (netif_tx_queue_stopped(txq) && skb) {
3614	rc = NETDEV_TX_BUSY;
3615	break;
3616	}
3617	}
3618
3619	out:
3620	*ret = rc;
3621	return skb;
3622	}
3623
3624	static struct sk_buff validate_xmit_vlan(struct* sk_buff *skb,
3625	netdev_features_t features)
3626	{
3627	if (skb_vlan_tag_present(skb) &&
3628	!vlan_hw_offload_capable(features, skb->vlan_proto))
3629	skb = __vlan_hwaccel_push_inside(skb);
3630	return skb;
3631	}
3632
3633	int skb_csum_hwoffload_help(struct sk_buff *skb,
3634	const netdev_features_t features)
3635	{
3636	if (unlikely(skb_csum_is_sctp(skb)))
3637	return !!(features & NETIF_F_SCTP_CRC) ? `0` :
3638	skb_crc32c_csum_help(skb);
3639
3640	if (features & NETIF_F_HW_CSUM)
3641	return `0`;
3642
3643	if (features & (NETIF_F_IP_CSUM \| NETIF_F_IPV6_CSUM)) {
3644	switch (skb->csum_offset) {
3645	case offsetof(struct tcphdr, check):
3646	case offsetof(struct udphdr, check):
3647	return `0`;
3648	}
3649	}
3650
3651	return skb_checksum_help(skb);
3652	}
3653	EXPORT_SYMBOL(skb_csum_hwoffload_help);
3654
3655	static struct sk_buff validate_xmit_skb(struct* sk_buff skb, struct* net_device dev, bool again)
3656	{
3657	netdev_features_t features;
3658
3659	features = netif_skb_features(skb);
3660	skb = validate_xmit_vlan(skb, features);
3661	if (unlikely(!skb))
3662	goto out_null;
3663
3664	skb = sk_validate_xmit_skb(skb, dev);
3665	if (unlikely(!skb))
3666	goto out_null;
3667
3668	if (netif_needs_gso(skb, features)) {
3669	struct sk_buff *segs;
3670
3671	segs = skb_gso_segment(skb, features);
3672	if (IS_ERR(segs)) {
3673	goto out_kfree_skb;
3674	} else if (segs) {
3675	consume_skb(skb);
3676	skb = segs;
3677	}
3678	} else {
3679	if (skb_needs_linearize(skb, features) &&
3680	__skb_linearize(skb))
3681	goto out_kfree_skb;
3682
3683	/ If packet is not checksummed and device does not*
3684	* support checksumming for this protocol, complete
3685	* checksumming here.
3686	*/
3687	if (skb->ip_summed == CHECKSUM_PARTIAL) {
3688	if (skb->encapsulation)
3689	skb_set_inner_transport_header(skb,
3690	skb_checksum_start_offset(skb));
3691	else
3692	skb_set_transport_header(skb,
3693	skb_checksum_start_offset(skb));
3694	if (skb_csum_hwoffload_help(skb, features))
3695	goto out_kfree_skb;
3696	}
3697	}
3698
3699	skb = validate_xmit_xfrm(skb, features, again);
3700
3701	return skb;
3702
3703	out_kfree_skb:
3704	kfree_skb(skb);
3705	out_null:
3706	dev_core_stats_tx_dropped_inc(dev);
3707	return NULL;
3708	}
3709
3710	struct sk_buff validate_xmit_skb_list(struct* sk_buff skb, struct* net_device dev, bool again)
3711	{
3712	struct sk_buff next, head = NULL, *tail;
3713
3714	for (; skb != NULL; skb = next) {
3715	next = skb->next;
3716	skb_mark_not_on_list(skb);
3717
3718	/ in case skb wont be segmented, point to itself /
3719	skb->prev = skb;
3720
3721	skb = validate_xmit_skb(skb, dev, again);
3722	if (!skb)
3723	continue;
3724
3725	if (!head)
3726	head = skb;
3727	else
3728	tail->next = skb;
3729	/ If skb was segmented, skb->prev points to*
3730	* the last segment. If not, it still contains skb.
3731	*/
3732	tail = skb->prev;
3733	}
3734	return head;
3735	}
3736	EXPORT_SYMBOL_GPL(validate_xmit_skb_list);
3737
3738	static void qdisc_pkt_len_init(struct sk_buff *skb)
3739	{
3740	const struct skb_shared_info *shinfo = skb_shinfo(skb);
3741
3742	qdisc_skb_cb(skb)->pkt_len = skb->len;
3743
3744	/ To get more precise estimation of bytes sent on wire,*
3745	* we add to pkt_len the headers size of all segments
3746	*/
3747	if (shinfo->gso_size && skb_transport_header_was_set(skb)) {
3748	unsigned int hdr_len;
3749	u16 gso_segs = shinfo->gso_segs;
3750
3751	/ mac layer + network layer /
3752	hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
3753
3754	/ + transport layer /
3755	if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 \| SKB_GSO_TCPV6))) {
3756	const struct tcphdr *th;
3757	struct tcphdr _tcphdr;
3758
3759	th = skb_header_pointer(skb, skb_transport_offset(skb),
3760	sizeof(_tcphdr), &_tcphdr);
3761	if (likely(th))
3762	hdr_len += __tcp_hdrlen(th);
3763	} else {
3764	struct udphdr _udphdr;
3765
3766	if (skb_header_pointer(skb, skb_transport_offset(skb),
3767	sizeof(_udphdr), &_udphdr))
3768	hdr_len += sizeof(struct udphdr);
3769	}
3770
3771	if (shinfo->gso_type & SKB_GSO_DODGY)
3772	gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
3773	shinfo->gso_size);
3774
3775	qdisc_skb_cb(skb)->pkt_len += (gso_segs - `1`) * hdr_len;
3776	}
3777	}
3778
3779	static int dev_qdisc_enqueue(struct sk_buff skb, struct* Qdisc *q,
3780	struct sk_buff **to_free,
3781	struct netdev_queue *txq)
3782	{
3783	int rc;
3784
3785	rc = q->enqueue(skb, q, to_free) & NET_XMIT_MASK;
3786	if (rc == NET_XMIT_SUCCESS)
3787	trace_qdisc_enqueue(q, txq, skb);
3788	return rc;
3789	}
3790
3791	static inline int __dev_xmit_skb(struct sk_buff skb, struct* Qdisc *q,
3792	struct net_device *dev,
3793	struct netdev_queue *txq)
3794	{
3795	spinlock_t *root_lock = qdisc_lock(q);
3796	struct sk_buff *to_free = NULL;
3797	bool contended;
3798	int rc;
3799
3800	qdisc_calculate_pkt_len(skb, q);
3801
3802	if (q->flags & TCQ_F_NOLOCK) {
3803	if (q->flags & TCQ_F_CAN_BYPASS && nolock_qdisc_is_empty(q) &&
3804	qdisc_run_begin(q)) {
3805	/ Retest nolock_qdisc_is_empty() within the protection*
3806	* of q->seqlock to protect from racing with requeuing.
3807	*/
3808	if (unlikely(!nolock_qdisc_is_empty(q))) {
3809	rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
3810	__qdisc_run(q);
3811	qdisc_run_end(q);
3812
3813	goto no_lock_out;
3814	}
3815
3816	qdisc_bstats_cpu_update(q, skb);
3817	if (sch_direct_xmit(skb, q, dev, txq, NULL, true) &&
3818	!nolock_qdisc_is_empty(q))
3819	__qdisc_run(q);
3820
3821	qdisc_run_end(q);
3822	return NET_XMIT_SUCCESS;
3823	}
3824
3825	rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
3826	qdisc_run(q);
3827
3828	no_lock_out:
3829	if (unlikely(to_free))
3830	kfree_skb_list_reason(to_free,
3831	SKB_DROP_REASON_QDISC_DROP);
3832	return rc;
3833	}
3834
3835	/*
3836	* Heuristic to force contended enqueues to serialize on a
3837	* separate lock before trying to get qdisc main lock.
3838	* This permits qdisc->running owner to get the lock more
3839	* often and dequeue packets faster.
3840	* On PREEMPT_RT it is possible to preempt the qdisc owner during xmit
3841	* and then other tasks will only enqueue packets. The packets will be
3842	* sent after the qdisc owner is scheduled again. To prevent this
3843	* scenario the task always serialize on the lock.
3844	*/
3845	contended = qdisc_is_running(q) \|\| IS_ENABLED(CONFIG_PREEMPT_RT);
3846	if (unlikely(contended))
3847	spin_lock(&q->busylock);
3848
3849	spin_lock(root_lock);
3850	if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
3851	__qdisc_drop(skb, &to_free);
3852	rc = NET_XMIT_DROP;
3853	} else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
3854	qdisc_run_begin(q)) {
3855	/*
3856	* This is a work-conserving queue; there are no old skbs
3857	* waiting to be sent out; and the qdisc is not running -
3858	* xmit the skb directly.
3859	*/
3860
3861	qdisc_bstats_update(q, skb);
3862
3863	if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
3864	if (unlikely(contended)) {
3865	spin_unlock(&q->busylock);
3866	contended = false;
3867	}
3868	__qdisc_run(q);
3869	}
3870
3871	qdisc_run_end(q);
3872	rc = NET_XMIT_SUCCESS;
3873	} else {
3874	rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
3875	if (qdisc_run_begin(q)) {
3876	if (unlikely(contended)) {
3877	spin_unlock(&q->busylock);
3878	contended = false;
3879	}
3880	__qdisc_run(q);
3881	qdisc_run_end(q);
3882	}
3883	}
3884	spin_unlock(root_lock);
3885	if (unlikely(to_free))
3886	kfree_skb_list_reason(to_free, SKB_DROP_REASON_QDISC_DROP);
3887	if (unlikely(contended))
3888	spin_unlock(&q->busylock);
3889	return rc;
3890	}
3891
3892	#if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
3893	static void skb_update_prio(struct sk_buff *skb)
3894	{
3895	const struct netprio_map *map;
3896	const struct sock *sk;
3897	unsigned int prioidx;
3898
3899	if (skb->priority)
3900	return;
3901	map = rcu_dereference_bh(skb->dev->priomap);
3902	if (!map)
3903	return;
3904	sk = skb_to_full_sk(skb);
3905	if (!sk)
3906	return;
3907
3908	prioidx = sock_cgroup_prioidx(&sk->sk_cgrp_data);
3909
3910	if (prioidx < map->priomap_len)
3911	skb->priority = map->priomap[prioidx];
3912	}
3913	#else
3914	#define skb_update_prio(skb)
3915	#endif
3916
3917	/**
3918	* dev_loopback_xmit - loop back @skb
3919	* @net: network namespace this loopback is happening in
3920	* @sk: sk needed to be a netfilter okfn
3921	* @skb: buffer to transmit
3922	*/
3923	int dev_loopback_xmit(struct net net, struct* sock sk, struct* sk_buff *skb)
3924	{
3925	skb_reset_mac_header(skb);
3926	__skb_pull(skb, skb_network_offset(skb));
3927	skb->pkt_type = PACKET_LOOPBACK;
3928	if (skb->ip_summed == CHECKSUM_NONE)
3929	skb->ip_summed = CHECKSUM_UNNECESSARY;
3930	DEBUG_NET_WARN_ON_ONCE(!skb_dst(skb));
3931	skb_dst_force(skb);
3932	netif_rx(skb);
3933	return `0`;
3934	}
3935	EXPORT_SYMBOL(dev_loopback_xmit);
3936
3937	#ifdef CONFIG_NET_EGRESS
3938	static struct sk_buff *
3939	sch_handle_egress(struct sk_buff skb, int* ret, struct* net_device *dev)
3940	{
3941	#ifdef CONFIG_NET_CLS_ACT
3942	struct mini_Qdisc *miniq = rcu_dereference_bh(dev->miniq_egress);
3943	struct tcf_result cl_res;
3944
3945	if (!miniq)
3946	return skb;
3947
3948	/ qdisc_skb_cb(skb)->pkt_len was already set by the caller. /
3949	tc_skb_cb(skb)->mru = `0`;
3950	tc_skb_cb(skb)->post_ct = false;
3951	mini_qdisc_bstats_cpu_update(miniq, skb);
3952
3953	switch (tcf_classify(skb, miniq->block, miniq->filter_list, &cl_res, false)) {
3954	case TC_ACT_OK:
3955	case TC_ACT_RECLASSIFY:
3956	skb->tc_index = TC_H_MIN(cl_res.classid);
3957	break;
3958	case TC_ACT_SHOT:
3959	mini_qdisc_qstats_cpu_drop(miniq);
3960	*ret = NET_XMIT_DROP;
3961	kfree_skb_reason(skb, SKB_DROP_REASON_TC_EGRESS);
3962	return NULL;
3963	case TC_ACT_STOLEN:
3964	case TC_ACT_QUEUED:
3965	case TC_ACT_TRAP:
3966	*ret = NET_XMIT_SUCCESS;
3967	consume_skb(skb);
3968	return NULL;
3969	case TC_ACT_REDIRECT:
3970	/ No need to push/pop skb's mac_header here on egress! /
3971	skb_do_redirect(skb);
3972	*ret = NET_XMIT_SUCCESS;
3973	return NULL;
3974	default:
3975	break;
3976	}
3977	#endif /* CONFIG_NET_CLS_ACT */
3978
3979	return skb;
3980	}
3981
3982	static struct netdev_queue *
3983	netdev_tx_queue_mapping(struct net_device dev, struct* sk_buff *skb)
3984	{
3985	int qm = skb_get_queue_mapping(skb);
3986
3987	return netdev_get_tx_queue(dev, netdev_cap_txqueue(dev, qm));
3988	}
3989
3990	static bool netdev_xmit_txqueue_skipped(void)
3991	{
3992	return __this_cpu_read(softnet_data.xmit.skip_txqueue);
3993	}
3994
3995	void netdev_xmit_skip_txqueue(bool skip)
3996	{
3997	__this_cpu_write(softnet_data.xmit.skip_txqueue, skip);
3998	}
3999	EXPORT_SYMBOL_GPL(netdev_xmit_skip_txqueue);
4000	#endif /* CONFIG_NET_EGRESS */
4001
4002	#ifdef CONFIG_XPS
4003	static int __get_xps_queue_idx(struct net_device dev, struct* sk_buff *skb,
4004	struct xps_dev_maps dev_maps, unsigned* int tci)
4005	{
4006	int tc = netdev_get_prio_tc_map(dev, skb->priority);
4007	struct xps_map *map;
4008	int queue_index = -`1`;
4009
4010	if (tc >= dev_maps->num_tc \|\| tci >= dev_maps->nr_ids)
4011	return queue_index;
4012
4013	tci *= dev_maps->num_tc;
4014	tci += tc;
4015
4016	map = rcu_dereference(dev_maps->attr_map[tci]);
4017	if (map) {
4018	if (map->len == `1`)
4019	queue_index = map->queues[`0`];
4020	else
4021	queue_index = map->queues[reciprocal_scale(
4022	skb_get_hash(skb), map->len)];
4023	if (unlikely(queue_index >= dev->real_num_tx_queues))
4024	queue_index = -`1`;
4025	}
4026	return queue_index;
4027	}
4028	#endif
4029
4030	static int get_xps_queue(struct net_device dev, struct* net_device *sb_dev,
4031	struct sk_buff *skb)
4032	{
4033	#ifdef CONFIG_XPS
4034	struct xps_dev_maps *dev_maps;
4035	struct sock *sk = skb->sk;
4036	int queue_index = -`1`;
4037
4038	if (!static_key_false(&xps_needed))
4039	return -`1`;
4040
4041	rcu_read_lock();
4042	if (!static_key_false(&xps_rxqs_needed))
4043	goto get_cpus_map;
4044
4045	dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_RXQS]);
4046	if (dev_maps) {
4047	int tci = sk_rx_queue_get(sk);
4048
4049	if (tci >= `0`)
4050	queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
4051	tci);
4052	}
4053
4054	get_cpus_map:
4055	if (queue_index < `0`) {
4056	dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_CPUS]);
4057	if (dev_maps) {
4058	unsigned int tci = skb->sender_cpu - `1`;
4059
4060	queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
4061	tci);
4062	}
4063	}
4064	rcu_read_unlock();
4065
4066	return queue_index;
4067	#else
4068	return -`1`;
4069	#endif
4070	}
4071
4072	u16 dev_pick_tx_zero(struct net_device dev, struct* sk_buff *skb,
4073	struct net_device *sb_dev)
4074	{
4075	return `0`;
4076	}
4077	EXPORT_SYMBOL(dev_pick_tx_zero);
4078
4079	u16 dev_pick_tx_cpu_id(struct net_device dev, struct* sk_buff *skb,
4080	struct net_device *sb_dev)
4081	{
4082	return (u16)raw_smp_processor_id() % dev->real_num_tx_queues;
4083	}
4084	EXPORT_SYMBOL(dev_pick_tx_cpu_id);
4085
4086	u16 netdev_pick_tx(struct net_device dev, struct* sk_buff *skb,
4087	struct net_device *sb_dev)
4088	{
4089	struct sock *sk = skb->sk;
4090	int queue_index = sk_tx_queue_get(sk);
4091
4092	sb_dev = sb_dev ? : dev;
4093
4094	if (queue_index < `0` \|\| skb->ooo_okay \|\|
4095	queue_index >= dev->real_num_tx_queues) {
4096	int new_index = get_xps_queue(dev, sb_dev, skb);
4097
4098	if (new_index < `0`)
4099	new_index = skb_tx_hash(dev, sb_dev, skb);
4100
4101	if (queue_index != new_index && sk &&
4102	sk_fullsock(sk) &&
4103	rcu_access_pointer(sk->sk_dst_cache))
4104	sk_tx_queue_set(sk, new_index);
4105
4106	queue_index = new_index;
4107	}
4108
4109	return queue_index;
4110	}
4111	EXPORT_SYMBOL(netdev_pick_tx);
4112
4113	struct netdev_queue netdev_core_pick_tx(struct* net_device *dev,
4114	struct sk_buff *skb,
4115	struct net_device *sb_dev)
4116	{
4117	int queue_index = `0`;
4118
4119	#ifdef CONFIG_XPS
4120	u32 sender_cpu = skb->sender_cpu - `1`;
4121
4122	if (sender_cpu >= (u32)NR_CPUS)
4123	skb->sender_cpu = raw_smp_processor_id() + `1`;
4124	#endif
4125
4126	if (dev->real_num_tx_queues != `1`) {
4127	const struct net_device_ops *ops = dev->netdev_ops;
4128
4129	if (ops->ndo_select_queue)
4130	queue_index = ops->ndo_select_queue(dev, skb, sb_dev);
4131	else
4132	queue_index = netdev_pick_tx(dev, skb, sb_dev);
4133
4134	queue_index = netdev_cap_txqueue(dev, queue_index);
4135	}
4136
4137	skb_set_queue_mapping(skb, queue_index);
4138	return netdev_get_tx_queue(dev, queue_index);
4139	}
4140
4141	/**
4142	* __dev_queue_xmit() - transmit a buffer
4143	* @skb: buffer to transmit
4144	* @sb_dev: suboordinate device used for L2 forwarding offload
4145	*
4146	* Queue a buffer for transmission to a network device. The caller must
4147	* have set the device and priority and built the buffer before calling
4148	* this function. The function can be called from an interrupt.
4149	*
4150	* When calling this method, interrupts MUST be enabled. This is because
4151	* the BH enable code must have IRQs enabled so that it will not deadlock.
4152	*
4153	* Regardless of the return value, the skb is consumed, so it is currently
4154	* difficult to retry a send to this method. (You can bump the ref count
4155	* before sending to hold a reference for retry if you are careful.)
4156	*
4157	* Return:
4158	* * 0 - buffer successfully transmitted
4159	* * positive qdisc return code - NET_XMIT_DROP etc.
4160	* * negative errno - other errors
4161	*/
4162	int __dev_queue_xmit(struct sk_buff skb, struct* net_device *sb_dev)
4163	{
4164	struct net_device *dev = skb->dev;
4165	struct netdev_queue *txq = NULL;
4166	struct Qdisc *q;
4167	int rc = -ENOMEM;
4168	bool again = false;
4169
4170	skb_reset_mac_header(skb);
4171	skb_assert_len(skb);
4172
4173	if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
4174	__skb_tstamp_tx(skb, NULL, NULL, skb->sk, SCM_TSTAMP_SCHED);
4175
4176	/ Disable soft irqs for various locks below. Also*
4177	* stops preemption for RCU.
4178	*/
4179	rcu_read_lock_bh();
4180
4181	skb_update_prio(skb);
4182
4183	qdisc_pkt_len_init(skb);
4184	#ifdef CONFIG_NET_CLS_ACT
4185	skb->tc_at_ingress = `0`;
4186	#endif
4187	#ifdef CONFIG_NET_EGRESS
4188	if (static_branch_unlikely(&egress_needed_key)) {
4189	if (nf_hook_egress_active()) {
4190	skb = nf_hook_egress(skb, &rc, dev);
4191	if (!skb)
4192	goto out;
4193	}
4194
4195	netdev_xmit_skip_txqueue(false);
4196
4197	nf_skip_egress(skb, true);
4198	skb = sch_handle_egress(skb, &rc, dev);
4199	if (!skb)
4200	goto out;
4201	nf_skip_egress(skb, false);
4202
4203	if (netdev_xmit_txqueue_skipped())
4204	txq = netdev_tx_queue_mapping(dev, skb);
4205	}
4206	#endif
4207	/ If device/qdisc don't need skb->dst, release it right now while*
4208	* its hot in this cpu cache.
4209	*/
4210	if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
4211	skb_dst_drop(skb);
4212	else
4213	skb_dst_force(skb);
4214
4215	if (!txq)
4216	txq = netdev_core_pick_tx(dev, skb, sb_dev);
4217
4218	q = rcu_dereference_bh(txq->qdisc);
4219
4220	trace_net_dev_queue(skb);
4221	if (q->enqueue) {
4222	rc = __dev_xmit_skb(skb, q, dev, txq);
4223	goto out;
4224	}
4225
4226	/ The device has no queue. Common case for software devices:*
4227	* loopback, all the sorts of tunnels...
4228
4229	* Really, it is unlikely that netif_tx_lock protection is necessary
4230	* here. (f.e. loopback and IP tunnels are clean ignoring statistics
4231	* counters.)
4232	* However, it is possible, that they rely on protection
4233	* made by us here.
4234
4235	* Check this and shot the lock. It is not prone from deadlocks.
4236	*Either shot noqueue qdisc, it is even simpler 8)
4237	*/
4238	if (dev->flags & IFF_UP) {
4239	int cpu = smp_processor_id(); / ok because BHs are off /
4240
4241	/ Other cpus might concurrently change txq->xmit_lock_owner*
4242	* to -1 or to their cpu id, but not to our id.
4243	*/
4244	if (READ_ONCE(txq->xmit_lock_owner) != cpu) {
4245	if (dev_xmit_recursion())
4246	goto recursion_alert;
4247
4248	skb = validate_xmit_skb(skb, dev, &again);
4249	if (!skb)
4250	goto out;
4251
4252	HARD_TX_LOCK(dev, txq, cpu);
4253
4254	if (!netif_xmit_stopped(txq)) {
4255	dev_xmit_recursion_inc();
4256	skb = dev_hard_start_xmit(skb, dev, txq, &rc);
4257	dev_xmit_recursion_dec();
4258	if (dev_xmit_complete(rc)) {
4259	HARD_TX_UNLOCK(dev, txq);
4260	goto out;
4261	}
4262	}
4263	HARD_TX_UNLOCK(dev, txq);
4264	net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
4265	dev->name);
4266	} else {
4267	/ Recursion is detected! It is possible,*
4268	* unfortunately
4269	*/
4270	recursion_alert:
4271	net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
4272	dev->name);
4273	}
4274	}
4275
4276	rc = -ENETDOWN;
4277	rcu_read_unlock_bh();
4278
4279	dev_core_stats_tx_dropped_inc(dev);
4280	kfree_skb_list(skb);
4281	return rc;
4282	out:
4283	rcu_read_unlock_bh();
4284	return rc;
4285	}
4286	EXPORT_SYMBOL(__dev_queue_xmit);
4287
4288	int __dev_direct_xmit(struct sk_buff *skb, u16 queue_id)
4289	{
4290	struct net_device *dev = skb->dev;
4291	struct sk_buff *orig_skb = skb;
4292	struct netdev_queue *txq;
4293	int ret = NETDEV_TX_BUSY;
4294	bool again = false;
4295
4296	if (unlikely(!netif_running(dev) \|\|
4297	!netif_carrier_ok(dev)))
4298	goto drop;
4299
4300	skb = validate_xmit_skb_list(skb, dev, &again);
4301	if (skb != orig_skb)
4302	goto drop;
4303
4304	skb_set_queue_mapping(skb, queue_id);
4305	txq = skb_get_tx_queue(dev, skb);
4306
4307	local_bh_disable();
4308
4309	dev_xmit_recursion_inc();
4310	HARD_TX_LOCK(dev, txq, smp_processor_id());
4311	if (!netif_xmit_frozen_or_drv_stopped(txq))
4312	ret = netdev_start_xmit(skb, dev, txq, false);
4313	HARD_TX_UNLOCK(dev, txq);
4314	dev_xmit_recursion_dec();
4315
4316	local_bh_enable();
4317	return ret;
4318	drop:
4319	dev_core_stats_tx_dropped_inc(dev);
4320	kfree_skb_list(skb);
4321	return NET_XMIT_DROP;
4322	}
4323	EXPORT_SYMBOL(__dev_direct_xmit);
4324
4325	/*************************************************************************
4326	* Receiver routines
4327	*************************************************************************/
4328
4329	int netdev_max_backlog __read_mostly = `1000`;
4330	EXPORT_SYMBOL(netdev_max_backlog);
4331
4332	int netdev_tstamp_prequeue __read_mostly = `1`;
4333	unsigned int sysctl_skb_defer_max __read_mostly = `64`;
4334	int netdev_budget __read_mostly = `300`;
4335	/ Must be at least 2 jiffes to guarantee 1 jiffy timeout /
4336	unsigned int __read_mostly netdev_budget_usecs = `2` * USEC_PER_SEC / HZ;
4337	int weight_p __read_mostly = `64`; / old backlog weight /
4338	int dev_weight_rx_bias __read_mostly = `1`; / bias for backlog weight /
4339	int dev_weight_tx_bias __read_mostly = `1`; / bias for output_queue quota /
4340	int dev_rx_weight __read_mostly = `64`;
4341	int dev_tx_weight __read_mostly = `64`;
4342
4343	/ Called with irq disabled /
4344	static inline void ____napi_schedule(struct softnet_data *sd,
4345	struct napi_struct *napi)
4346	{
4347	struct task_struct *thread;
4348
4349	lockdep_assert_irqs_disabled();
4350
4351	if (test_bit(NAPI_STATE_THREADED, &napi->state)) {
4352	/ Paired with smp_mb__before_atomic() in*
4353	* napi_enable()/dev_set_threaded().
4354	* Use READ_ONCE() to guarantee a complete
4355	* read on napi->thread. Only call
4356	* wake_up_process() when it's not NULL.
4357	*/
4358	thread = READ_ONCE(napi->thread);
4359	if (thread) {
4360	/ Avoid doing set_bit() if the thread is in*
4361	* INTERRUPTIBLE state, cause napi_thread_wait()
4362	* makes sure to proceed with napi polling
4363	* if the thread is explicitly woken from here.
4364	*/
4365	if (READ_ONCE(thread->__state) != TASK_INTERRUPTIBLE)
4366	set_bit(NAPI_STATE_SCHED_THREADED, &napi->state);
4367	wake_up_process(thread);
4368	return;
4369	}
4370	}
4371
4372	list_add_tail(&napi->poll_list, &sd->poll_list);
4373	__raise_softirq_irqoff(NET_RX_SOFTIRQ);
4374	}
4375
4376	#ifdef CONFIG_RPS
4377
4378	/ One global table that all flow-based protocols share. /
4379	struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
4380	EXPORT_SYMBOL(rps_sock_flow_table);
4381	u32 rps_cpu_mask __read_mostly;
4382	EXPORT_SYMBOL(rps_cpu_mask);
4383
4384	struct static_key_false rps_needed __read_mostly;
4385	EXPORT_SYMBOL(rps_needed);
4386	struct static_key_false rfs_needed __read_mostly;
4387	EXPORT_SYMBOL(rfs_needed);
4388
4389	static struct rps_dev_flow *
4390	set_rps_cpu(struct net_device dev, struct* sk_buff *skb,
4391	struct rps_dev_flow *rflow, u16 next_cpu)
4392	{
4393	if (next_cpu < nr_cpu_ids) {
4394	#ifdef CONFIG_RFS_ACCEL
4395	struct netdev_rx_queue *rxqueue;
4396	struct rps_dev_flow_table *flow_table;
4397	struct rps_dev_flow *old_rflow;
4398	u32 flow_id;
4399	u16 rxq_index;
4400	int rc;
4401
4402	/ Should we steer this flow to a different hardware queue? /
4403	if (!skb_rx_queue_recorded(skb) \|\| !dev->rx_cpu_rmap \|\|
4404	!(dev->features & NETIF_F_NTUPLE))
4405	goto out;
4406	rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
4407	if (rxq_index == skb_get_rx_queue(skb))
4408	goto out;
4409
4410	rxqueue = dev->_rx + rxq_index;
4411	flow_table = rcu_dereference(rxqueue->rps_flow_table);
4412	if (!flow_table)
4413	goto out;
4414	flow_id = skb_get_hash(skb) & flow_table->mask;
4415	rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
4416	rxq_index, flow_id);
4417	if (rc < `0`)
4418	goto out;
4419	old_rflow = rflow;
4420	rflow = &flow_table->flows[flow_id];
4421	rflow->filter = rc;
4422	if (old_rflow->filter == rflow->filter)
4423	old_rflow->filter = RPS_NO_FILTER;
4424	out:
4425	#endif
4426	rflow->last_qtail =
4427	per_cpu(softnet_data, next_cpu).input_queue_head;
4428	}
4429
4430	rflow->cpu = next_cpu;
4431	return rflow;
4432	}
4433
4434	/*
4435	* get_rps_cpu is called from netif_receive_skb and returns the target
4436	* CPU from the RPS map of the receiving queue for a given skb.
4437	* rcu_read_lock must be held on entry.
4438	*/
4439	static int get_rps_cpu(struct net_device dev, struct* sk_buff *skb,
4440	struct rps_dev_flow **rflowp)
4441	{
4442	const struct rps_sock_flow_table *sock_flow_table;
4443	struct netdev_rx_queue *rxqueue = dev->_rx;
4444	struct rps_dev_flow_table *flow_table;
4445	struct rps_map *map;
4446	int cpu = -`1`;
4447	u32 tcpu;
4448	u32 hash;
4449
4450	if (skb_rx_queue_recorded(skb)) {
4451	u16 index = skb_get_rx_queue(skb);
4452
4453	if (unlikely(index >= dev->real_num_rx_queues)) {
4454	WARN_ONCE(dev->real_num_rx_queues > `1`,
4455	"%s received packet on queue %u, but number "
4456	"of RX queues is %u\n",
4457	dev->name, index, dev->real_num_rx_queues);
4458	goto done;
4459	}
4460	rxqueue += index;
4461	}
4462
4463	/ Avoid computing hash if RFS/RPS is not active for this rxqueue /
4464
4465	flow_table = rcu_dereference(rxqueue->rps_flow_table);
4466	map = rcu_dereference(rxqueue->rps_map);
4467	if (!flow_table && !map)
4468	goto done;
4469
4470	skb_reset_network_header(skb);
4471	hash = skb_get_hash(skb);
4472	if (!hash)
4473	goto done;
4474
4475	sock_flow_table = rcu_dereference(rps_sock_flow_table);
4476	if (flow_table && sock_flow_table) {
4477	struct rps_dev_flow *rflow;
4478	u32 next_cpu;
4479	u32 ident;
4480
4481	/ First check into global flow table if there is a match /
4482	ident = sock_flow_table->ents[hash & sock_flow_table->mask];
4483	if ((ident ^ hash) & ~rps_cpu_mask)
4484	goto try_rps;
4485
4486	next_cpu = ident & rps_cpu_mask;
4487
4488	/ OK, now we know there is a match,*
4489	* we can look at the local (per receive queue) flow table
4490	*/
4491	rflow = &flow_table->flows[hash & flow_table->mask];
4492	tcpu = rflow->cpu;
4493
4494	/*
4495	* If the desired CPU (where last recvmsg was done) is
4496	* different from current CPU (one in the rx-queue flow
4497	* table entry), switch if one of the following holds:
4498	* - Current CPU is unset (>= nr_cpu_ids).
4499	* - Current CPU is offline.
4500	* - The current CPU's queue tail has advanced beyond the
4501	* last packet that was enqueued using this table entry.
4502	* This guarantees that all previous packets for the flow
4503	* have been dequeued, thus preserving in order delivery.
4504	*/
4505	if (unlikely(tcpu != next_cpu) &&
4506	(tcpu >= nr_cpu_ids \|\| !cpu_online(tcpu) \|\|
4507	((int)(per_cpu(softnet_data, tcpu).input_queue_head -
4508	rflow->last_qtail)) >= `0`)) {
4509	tcpu = next_cpu;
4510	rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
4511	}
4512
4513	if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
4514	*rflowp = rflow;
4515	cpu = tcpu;
4516	goto done;
4517	}
4518	}
4519
4520	try_rps:
4521
4522	if (map) {
4523	tcpu = map->cpus[reciprocal_scale(hash, map->len)];
4524	if (cpu_online(tcpu)) {
4525	cpu = tcpu;
4526	goto done;
4527	}
4528	}
4529
4530	done:
4531	return cpu;
4532	}
4533
4534	#ifdef CONFIG_RFS_ACCEL
4535
4536	/**
4537	* rps_may_expire_flow - check whether an RFS hardware filter may be removed
4538	* @dev: Device on which the filter was set
4539	* @rxq_index: RX queue index
4540	* @flow_id: Flow ID passed to ndo_rx_flow_steer()
4541	* @filter_id: Filter ID returned by ndo_rx_flow_steer()
4542	*
4543	* Drivers that implement ndo_rx_flow_steer() should periodically call
4544	* this function for each installed filter and remove the filters for
4545	* which it returns %true.
4546	*/
4547	bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
4548	u32 flow_id, u16 filter_id)
4549	{
4550	struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
4551	struct rps_dev_flow_table *flow_table;
4552	struct rps_dev_flow *rflow;
4553	bool expire = true;
4554	unsigned int cpu;
4555
4556	rcu_read_lock();
4557	flow_table = rcu_dereference(rxqueue->rps_flow_table);
4558	if (flow_table && flow_id <= flow_table->mask) {
4559	rflow = &flow_table->flows[flow_id];
4560	cpu = READ_ONCE(rflow->cpu);
4561	if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
4562	((int)(per_cpu(softnet_data, cpu).input_queue_head -
4563	rflow->last_qtail) <
4564	(int)(`10` * flow_table->mask)))
4565	expire = false;
4566	}
4567	rcu_read_unlock();
4568	return expire;
4569	}
4570	EXPORT_SYMBOL(rps_may_expire_flow);
4571
4572	#endif /* CONFIG_RFS_ACCEL */
4573
4574	/ Called from hardirq (IPI) context /
4575	static void rps_trigger_softirq(void *data)
4576	{
4577	struct softnet_data *sd = data;
4578
4579	____napi_schedule(sd, &sd->backlog);
4580	sd->received_rps++;
4581	}
4582
4583	#endif /* CONFIG_RPS */
4584
4585	/ Called from hardirq (IPI) context /
4586	static void trigger_rx_softirq(void *data)
4587	{
4588	struct softnet_data *sd = data;
4589
4590	__raise_softirq_irqoff(NET_RX_SOFTIRQ);
4591	smp_store_release(&sd->defer_ipi_scheduled, `0`);
4592	}
4593
4594	/*
4595	* Check if this softnet_data structure is another cpu one
4596	* If yes, queue it to our IPI list and return 1
4597	* If no, return 0
4598	*/
4599	static int napi_schedule_rps(struct softnet_data *sd)
4600	{
4601	struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
4602
4603	#ifdef CONFIG_RPS
4604	if (sd != mysd) {
4605	sd->rps_ipi_next = mysd->rps_ipi_list;
4606	mysd->rps_ipi_list = sd;
4607
4608	__raise_softirq_irqoff(NET_RX_SOFTIRQ);
4609	return `1`;
4610	}
4611	#endif /* CONFIG_RPS */
4612	__napi_schedule_irqoff(&mysd->backlog);
4613	return `0`;
4614	}
4615
4616	#ifdef CONFIG_NET_FLOW_LIMIT
4617	int netdev_flow_limit_table_len __read_mostly = (`1` << `12`);
4618	#endif
4619
4620	static bool skb_flow_limit(struct sk_buff skb, unsigned* int qlen)
4621	{
4622	#ifdef CONFIG_NET_FLOW_LIMIT
4623	struct sd_flow_limit *fl;
4624	struct softnet_data *sd;
4625	unsigned int old_flow, new_flow;
4626
4627	if (qlen < (netdev_max_backlog >> `1`))
4628	return false;
4629
4630	sd = this_cpu_ptr(&softnet_data);
4631
4632	rcu_read_lock();
4633	fl = rcu_dereference(sd->flow_limit);
4634	if (fl) {
4635	new_flow = skb_get_hash(skb) & (fl->num_buckets - `1`);
4636	old_flow = fl->history[fl->history_head];
4637	fl->history[fl->history_head] = new_flow;
4638
4639	fl->history_head++;
4640	fl->history_head &= FLOW_LIMIT_HISTORY - `1`;
4641
4642	if (likely(fl->buckets[old_flow]))
4643	fl->buckets[old_flow]--;
4644
4645	if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> `1`)) {
4646	fl->count++;
4647	rcu_read_unlock();
4648	return true;
4649	}
4650	}
4651	rcu_read_unlock();
4652	#endif
4653	return false;
4654	}
4655
4656	/*
4657	* enqueue_to_backlog is called to queue an skb to a per CPU backlog
4658	* queue (may be a remote CPU queue).
4659	*/
4660	static int enqueue_to_backlog(struct sk_buff skb, int* cpu,
4661	unsigned int *qtail)
4662	{
4663	enum skb_drop_reason reason;
4664	struct softnet_data *sd;
4665	unsigned long flags;
4666	unsigned int qlen;
4667
4668	reason = SKB_DROP_REASON_NOT_SPECIFIED;
4669	sd = &per_cpu(softnet_data, cpu);
4670
4671	rps_lock_irqsave(sd, &flags);
4672	if (!netif_running(skb->dev))
4673	goto drop;
4674	qlen = skb_queue_len(&sd->input_pkt_queue);
4675	if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) {
4676	if (qlen) {
4677	enqueue:
4678	__skb_queue_tail(&sd->input_pkt_queue, skb);
4679	input_queue_tail_incr_save(sd, qtail);
4680	rps_unlock_irq_restore(sd, &flags);
4681	return NET_RX_SUCCESS;
4682	}
4683
4684	/ Schedule NAPI for backlog device*
4685	* We can use non atomic operation since we own the queue lock
4686	*/
4687	if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state))
4688	napi_schedule_rps(sd);
4689	goto enqueue;
4690	}
4691	reason = SKB_DROP_REASON_CPU_BACKLOG;
4692
4693	drop:
4694	sd->dropped++;
4695	rps_unlock_irq_restore(sd, &flags);
4696
4697	dev_core_stats_rx_dropped_inc(skb->dev);
4698	kfree_skb_reason(skb, reason);
4699	return NET_RX_DROP;
4700	}
4701
4702	static struct netdev_rx_queue netif_get_rxqueue(struct* sk_buff *skb)
4703	{
4704	struct net_device *dev = skb->dev;
4705	struct netdev_rx_queue *rxqueue;
4706
4707	rxqueue = dev->_rx;
4708
4709	if (skb_rx_queue_recorded(skb)) {
4710	u16 index = skb_get_rx_queue(skb);
4711
4712	if (unlikely(index >= dev->real_num_rx_queues)) {
4713	WARN_ONCE(dev->real_num_rx_queues > `1`,
4714	"%s received packet on queue %u, but number "
4715	"of RX queues is %u\n",
4716	dev->name, index, dev->real_num_rx_queues);
4717
4718	return rxqueue; / Return first rxqueue /
4719	}
4720	rxqueue += index;
4721	}
4722	return rxqueue;
4723	}
4724
4725	u32 bpf_prog_run_generic_xdp(struct sk_buff skb, struct* xdp_buff *xdp,
4726	struct bpf_prog *xdp_prog)
4727	{
4728	void orig_data, orig_data_end, *hard_start;
4729	struct netdev_rx_queue *rxqueue;
4730	bool orig_bcast, orig_host;
4731	u32 mac_len, frame_sz;
4732	__be16 orig_eth_type;
4733	struct ethhdr *eth;
4734	u32 metalen, act;
4735	int off;
4736
4737	/ The XDP program wants to see the packet starting at the MAC*
4738	* header.
4739	*/
4740	mac_len = skb->data - skb_mac_header(skb);
4741	hard_start = skb->data - skb_headroom(skb);
4742
4743	/ SKB "head" area always have tailroom for skb_shared_info /
4744	frame_sz = (void *)skb_end_pointer(skb) - hard_start;
4745	frame_sz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
4746
4747	rxqueue = netif_get_rxqueue(skb);
4748	xdp_init_buff(xdp, frame_sz, &rxqueue->xdp_rxq);
4749	xdp_prepare_buff(xdp, hard_start, skb_headroom(skb) - mac_len,
4750	skb_headlen(skb) + mac_len, true);
4751
4752	orig_data_end = xdp->data_end;
4753	orig_data = xdp->data;
4754	eth = (struct ethhdr *)xdp->data;
4755	orig_host = ether_addr_equal_64bits(eth->h_dest, skb->dev->dev_addr);
4756	orig_bcast = is_multicast_ether_addr_64bits(eth->h_dest);
4757	orig_eth_type = eth->h_proto;
4758
4759	act = bpf_prog_run_xdp(xdp_prog, xdp);
4760
4761	/ check if bpf_xdp_adjust_head was used /
4762	off = xdp->data - orig_data;
4763	if (off) {
4764	if (off > `0`)
4765	__skb_pull(skb, off);
4766	else if (off < `0`)
4767	__skb_push(skb, -off);
4768
4769	skb->mac_header += off;
4770	skb_reset_network_header(skb);
4771	}
4772
4773	/ check if bpf_xdp_adjust_tail was used /
4774	off = xdp->data_end - orig_data_end;
4775	if (off != `0`) {
4776	skb_set_tail_pointer(skb, xdp->data_end - xdp->data);
4777	skb->len += off; / positive on grow, negative on shrink /
4778	}
4779
4780	/ check if XDP changed eth hdr such SKB needs update /
4781	eth = (struct ethhdr *)xdp->data;
4782	if ((orig_eth_type != eth->h_proto) \|\|
4783	(orig_host != ether_addr_equal_64bits(eth->h_dest,
4784	skb->dev->dev_addr)) \|\|
4785	(orig_bcast != is_multicast_ether_addr_64bits(eth->h_dest))) {
4786	__skb_push(skb, ETH_HLEN);
4787	skb->pkt_type = PACKET_HOST;
4788	skb->protocol = eth_type_trans(skb, skb->dev);
4789	}
4790
4791	/ Redirect/Tx gives L2 packet, code that will reuse skb must __skb_pull*
4792	* before calling us again on redirect path. We do not call do_redirect
4793	* as we leave that up to the caller.
4794	*
4795	* Caller is responsible for managing lifetime of skb (i.e. calling
4796	* kfree_skb in response to actions it cannot handle/XDP_DROP).
4797	*/
4798	switch (act) {
4799	case XDP_REDIRECT:
4800	case XDP_TX:
4801	__skb_push(skb, mac_len);
4802	break;
4803	case XDP_PASS:
4804	metalen = xdp->data - xdp->data_meta;
4805	if (metalen)
4806	skb_metadata_set(skb, metalen);
4807	break;
4808	}
4809
4810	return act;
4811	}
4812
4813	static u32 netif_receive_generic_xdp(struct sk_buff *skb,
4814	struct xdp_buff *xdp,
4815	struct bpf_prog *xdp_prog)
4816	{
4817	u32 act = XDP_DROP;
4818
4819	/ Reinjected packets coming from act_mirred or similar should*
4820	* not get XDP generic processing.
4821	*/
4822	if (skb_is_redirected(skb))
4823	return XDP_PASS;
4824
4825	/ XDP packets must be linear and must have sufficient headroom*
4826	* of XDP_PACKET_HEADROOM bytes. This is the guarantee that also
4827	* native XDP provides, thus we need to do it here as well.
4828	*/
4829	if (skb_cloned(skb) \|\| skb_is_nonlinear(skb) \|\|
4830	skb_headroom(skb) < XDP_PACKET_HEADROOM) {
4831	int hroom = XDP_PACKET_HEADROOM - skb_headroom(skb);
4832	int troom = skb->tail + skb->data_len - skb->end;
4833
4834	/ In case we have to go down the path and also linearize,*
4835	* then lets do the pskb_expand_head() work just once here.
4836	*/
4837	if (pskb_expand_head(skb,
4838	hroom > `0` ? ALIGN(hroom, NET_SKB_PAD) : `0`,
4839	troom > `0` ? troom + `128` : `0`, GFP_ATOMIC))
4840	goto do_drop;
4841	if (skb_linearize(skb))
4842	goto do_drop;
4843	}
4844
4845	act = bpf_prog_run_generic_xdp(skb, xdp, xdp_prog);
4846	switch (act) {
4847	case XDP_REDIRECT:
4848	case XDP_TX:
4849	case XDP_PASS:
4850	break;
4851	default:
4852	bpf_warn_invalid_xdp_action(skb->dev, xdp_prog, act);
4853	fallthrough;
4854	case XDP_ABORTED:
4855	trace_xdp_exception(skb->dev, xdp_prog, act);
4856	fallthrough;
4857	case XDP_DROP:
4858	do_drop:
4859	kfree_skb(skb);
4860	break;
4861	}
4862
4863	return act;
4864	}
4865
4866	/ When doing generic XDP we have to bypass the qdisc layer and the*
4867	* network taps in order to match in-driver-XDP behavior. This also means
4868	* that XDP packets are able to starve other packets going through a qdisc,
4869	* and DDOS attacks will be more effective. In-driver-XDP use dedicated TX
4870	* queues, so they do not have this starvation issue.
4871	*/
4872	void generic_xdp_tx(struct sk_buff skb, struct* bpf_prog *xdp_prog)
4873	{
4874	struct net_device *dev = skb->dev;
4875	struct netdev_queue *txq;
4876	bool free_skb = true;
4877	int cpu, rc;
4878
4879	txq = netdev_core_pick_tx(dev, skb, NULL);
4880	cpu = smp_processor_id();
4881	HARD_TX_LOCK(dev, txq, cpu);
4882	if (!netif_xmit_frozen_or_drv_stopped(txq)) {
4883	rc = netdev_start_xmit(skb, dev, txq, `0`);
4884	if (dev_xmit_complete(rc))
4885	free_skb = false;
4886	}
4887	HARD_TX_UNLOCK(dev, txq);
4888	if (free_skb) {
4889	trace_xdp_exception(dev, xdp_prog, XDP_TX);
4890	dev_core_stats_tx_dropped_inc(dev);
4891	kfree_skb(skb);
4892	}
4893	}
4894
4895	static DEFINE_STATIC_KEY_FALSE(generic_xdp_needed_key);
4896
4897	int do_xdp_generic(struct bpf_prog xdp_prog, struct* sk_buff *skb)
4898	{
4899	if (xdp_prog) {
4900	struct xdp_buff xdp;
4901	u32 act;
4902	int err;
4903
4904	act = netif_receive_generic_xdp(skb, &xdp, xdp_prog);
4905	if (act != XDP_PASS) {
4906	switch (act) {
4907	case XDP_REDIRECT:
4908	err = xdp_do_generic_redirect(skb->dev, skb,
4909	&xdp, xdp_prog);
4910	if (err)
4911	goto out_redir;
4912	break;
4913	case XDP_TX:
4914	generic_xdp_tx(skb, xdp_prog);
4915	break;
4916	}
4917	return XDP_DROP;
4918	}
4919	}
4920	return XDP_PASS;
4921	out_redir:
4922	kfree_skb_reason(skb, SKB_DROP_REASON_XDP);
4923	return XDP_DROP;
4924	}
4925	EXPORT_SYMBOL_GPL(do_xdp_generic);
4926
4927	static int netif_rx_internal(struct sk_buff *skb)
4928	{
4929	int ret;
4930
4931	net_timestamp_check(netdev_tstamp_prequeue, skb);
4932
4933	trace_netif_rx(skb);
4934
4935	#ifdef CONFIG_RPS
4936	if (static_branch_unlikely(&rps_needed)) {
4937	struct rps_dev_flow voidflow, *rflow = &voidflow;
4938	int cpu;
4939
4940	rcu_read_lock();
4941
4942	cpu = get_rps_cpu(skb->dev, skb, &rflow);
4943	if (cpu < `0`)
4944	cpu = smp_processor_id();
4945
4946	ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
4947
4948	rcu_read_unlock();
4949	} else
4950	#endif
4951	{
4952	unsigned int qtail;
4953
4954	ret = enqueue_to_backlog(skb, smp_processor_id(), &qtail);
4955	}
4956	return ret;
4957	}
4958
4959	/**
4960	* __netif_rx - Slightly optimized version of netif_rx
4961	* @skb: buffer to post
4962	*
4963	* This behaves as netif_rx except that it does not disable bottom halves.
4964	* As a result this function may only be invoked from the interrupt context
4965	* (either hard or soft interrupt).
4966	*/
4967	int __netif_rx(struct sk_buff *skb)
4968	{
4969	int ret;
4970
4971	lockdep_assert_once(hardirq_count() \| softirq_count());
4972
4973	trace_netif_rx_entry(skb);
4974	ret = netif_rx_internal(skb);
4975	trace_netif_rx_exit(ret);
4976	return ret;
4977	}
4978	EXPORT_SYMBOL(__netif_rx);
4979
4980	/**
4981	* netif_rx - post buffer to the network code
4982	* @skb: buffer to post
4983	*
4984	* This function receives a packet from a device driver and queues it for
4985	* the upper (protocol) levels to process via the backlog NAPI device. It
4986	* always succeeds. The buffer may be dropped during processing for
4987	* congestion control or by the protocol layers.
4988	* The network buffer is passed via the backlog NAPI device. Modern NIC
4989	* driver should use NAPI and GRO.
4990	* This function can used from interrupt and from process context. The
4991	* caller from process context must not disable interrupts before invoking
4992	* this function.
4993	*
4994	* return values:
4995	* NET_RX_SUCCESS (no congestion)
4996	* NET_RX_DROP (packet was dropped)
4997	*
4998	*/
4999	int netif_rx(struct sk_buff *skb)
5000	{
5001	bool need_bh_off = !(hardirq_count() \| softirq_count());
5002	int ret;
5003
5004	if (need_bh_off)
5005	local_bh_disable();
5006	trace_netif_rx_entry(skb);
5007	ret = netif_rx_internal(skb);
5008	trace_netif_rx_exit(ret);
5009	if (need_bh_off)
5010	local_bh_enable();
5011	return ret;
5012	}
5013	EXPORT_SYMBOL(netif_rx);
5014
5015	static __latent_entropy void net_tx_action(struct softirq_action *h)
5016	{
5017	struct softnet_data *sd = this_cpu_ptr(&softnet_data);
5018
5019	if (sd->completion_queue) {
5020	struct sk_buff *clist;
5021
5022	local_irq_disable();
5023	clist = sd->completion_queue;
5024	sd->completion_queue = NULL;
5025	local_irq_enable();
5026
5027	while (clist) {
5028	struct sk_buff *skb = clist;
5029
5030	clist = clist->next;
5031
5032	WARN_ON(refcount_read(&skb->users));
5033	if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED))
5034	trace_consume_skb(skb);
5035	else
5036	trace_kfree_skb(skb, net_tx_action,
5037	SKB_DROP_REASON_NOT_SPECIFIED);
5038
5039	if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
5040	__kfree_skb(skb);
5041	else
5042	__kfree_skb_defer(skb);
5043	}
5044	}
5045
5046	if (sd->output_queue) {
5047	struct Qdisc *head;
5048
5049	local_irq_disable();
5050	head = sd->output_queue;
5051	sd->output_queue = NULL;
5052	sd->output_queue_tailp = &sd->output_queue;
5053	local_irq_enable();
5054
5055	rcu_read_lock();
5056
5057	while (head) {
5058	struct Qdisc *q = head;
5059	spinlock_t *root_lock = NULL;
5060
5061	head = head->next_sched;
5062
5063	/ We need to make sure head->next_sched is read*
5064	* before clearing __QDISC_STATE_SCHED
5065	*/
5066	smp_mb__before_atomic();
5067
5068	if (!(q->flags & TCQ_F_NOLOCK)) {
5069	root_lock =