1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * NET3 Protocol independent device support routines.
4 *
5 * Derived from the non IP parts of dev.c 1.0.19
6 * Authors: Ross Biro
7 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
8 * Mark Evans, <evansmp@uhura.aston.ac.uk>
9 *
10 * Additional Authors:
11 * Florian la Roche <rzsfl@rz.uni-sb.de>
12 * Alan Cox <gw4pts@gw4pts.ampr.org>
13 * David Hinds <dahinds@users.sourceforge.net>
14 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
15 * Adam Sulmicki <adam@cfar.umd.edu>
16 * Pekka Riikonen <priikone@poesidon.pspt.fi>
17 *
18 * Changes:
19 * D.J. Barrow : Fixed bug where dev->refcnt gets set
20 * to 2 if register_netdev gets called
21 * before net_dev_init & also removed a
22 * few lines of code in the process.
23 * Alan Cox : device private ioctl copies fields back.
24 * Alan Cox : Transmit queue code does relevant
25 * stunts to keep the queue safe.
26 * Alan Cox : Fixed double lock.
27 * Alan Cox : Fixed promisc NULL pointer trap
28 * ???????? : Support the full private ioctl range
29 * Alan Cox : Moved ioctl permission check into
30 * drivers
31 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
32 * Alan Cox : 100 backlog just doesn't cut it when
33 * you start doing multicast video 8)
34 * Alan Cox : Rewrote net_bh and list manager.
35 * Alan Cox : Fix ETH_P_ALL echoback lengths.
36 * Alan Cox : Took out transmit every packet pass
37 * Saved a few bytes in the ioctl handler
38 * Alan Cox : Network driver sets packet type before
39 * calling netif_rx. Saves a function
40 * call a packet.
41 * Alan Cox : Hashed net_bh()
42 * Richard Kooijman: Timestamp fixes.
43 * Alan Cox : Wrong field in SIOCGIFDSTADDR
44 * Alan Cox : Device lock protection.
45 * Alan Cox : Fixed nasty side effect of device close
46 * changes.
47 * Rudi Cilibrasi : Pass the right thing to
48 * set_mac_address()
49 * Dave Miller : 32bit quantity for the device lock to
50 * make it work out on a Sparc.
51 * Bjorn Ekwall : Added KERNELD hack.
52 * Alan Cox : Cleaned up the backlog initialise.
53 * Craig Metz : SIOCGIFCONF fix if space for under
54 * 1 device.
55 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
56 * is no device open function.
57 * Andi Kleen : Fix error reporting for SIOCGIFCONF
58 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
59 * Cyrus Durgin : Cleaned for KMOD
60 * Adam Sulmicki : Bug Fix : Network Device Unload
61 * A network device unload needs to purge
62 * the backlog queue.
63 * Paul Rusty Russell : SIOCSIFNAME
64 * Pekka Riikonen : Netdev boot-time settings code
65 * Andrew Morton : Make unregister_netdevice wait
66 * indefinitely on dev->refcnt
67 * J Hadi Salim : - Backlog queue sampling
68 * - netif_rx() feedback
69 */
70
71#include <linux/uaccess.h>
72#include <linux/bitops.h>
73#include <linux/capability.h>
74#include <linux/cpu.h>
75#include <linux/types.h>
76#include <linux/kernel.h>
77#include <linux/hash.h>
78#include <linux/slab.h>
79#include <linux/sched.h>
80#include <linux/sched/mm.h>
81#include <linux/mutex.h>
82#include <linux/rwsem.h>
83#include <linux/string.h>
84#include <linux/mm.h>
85#include <linux/socket.h>
86#include <linux/sockios.h>
87#include <linux/errno.h>
88#include <linux/interrupt.h>
89#include <linux/if_ether.h>
90#include <linux/netdevice.h>
91#include <linux/etherdevice.h>
92#include <linux/ethtool.h>
93#include <linux/skbuff.h>
94#include <linux/kthread.h>
95#include <linux/bpf.h>
96#include <linux/bpf_trace.h>
97#include <net/net_namespace.h>
98#include <net/sock.h>
99#include <net/busy_poll.h>
100#include <linux/rtnetlink.h>
101#include <linux/stat.h>
102#include <net/dsa.h>
103#include <net/dst.h>
104#include <net/dst_metadata.h>
105#include <net/gro.h>
106#include <net/pkt_sched.h>
107#include <net/pkt_cls.h>
108#include <net/checksum.h>
109#include <net/xfrm.h>
110#include <linux/highmem.h>
111#include <linux/init.h>
112#include <linux/module.h>
113#include <linux/netpoll.h>
114#include <linux/rcupdate.h>
115#include <linux/delay.h>
116#include <net/iw_handler.h>
117#include <asm/current.h>
118#include <linux/audit.h>
119#include <linux/dmaengine.h>
120#include <linux/err.h>
121#include <linux/ctype.h>
122#include <linux/if_arp.h>
123#include <linux/if_vlan.h>
124#include <linux/ip.h>
125#include <net/ip.h>
126#include <net/mpls.h>
127#include <linux/ipv6.h>
128#include <linux/in.h>
129#include <linux/jhash.h>
130#include <linux/random.h>
131#include <trace/events/napi.h>
132#include <trace/events/net.h>
133#include <trace/events/skb.h>
134#include <trace/events/qdisc.h>
135#include <linux/inetdevice.h>
136#include <linux/cpu_rmap.h>
137#include <linux/static_key.h>
138#include <linux/hashtable.h>
139#include <linux/vmalloc.h>
140#include <linux/if_macvlan.h>
141#include <linux/errqueue.h>
142#include <linux/hrtimer.h>
143#include <linux/netfilter_netdev.h>
144#include <linux/crash_dump.h>
145#include <linux/sctp.h>
146#include <net/udp_tunnel.h>
147#include <linux/net_namespace.h>
148#include <linux/indirect_call_wrapper.h>
149#include <net/devlink.h>
150#include <linux/pm_runtime.h>
151#include <linux/prandom.h>
152#include <linux/once_lite.h>
153
154#include "dev.h"
155#include "net-sysfs.h"
156
157
158static DEFINE_SPINLOCK(ptype_lock);
159struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
160struct list_head ptype_all __read_mostly; /* Taps */
161
162static int netif_rx_internal(struct sk_buff *skb);
163static int call_netdevice_notifiers_info(unsigned long val,
164 struct netdev_notifier_info *info);
165static int call_netdevice_notifiers_extack(unsigned long val,
166 struct net_device *dev,
167 struct netlink_ext_ack *extack);
168static struct napi_struct *napi_by_id(unsigned int napi_id);
169
170/*
171 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
172 * semaphore.
173 *
174 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
175 *
176 * Writers must hold the rtnl semaphore while they loop through the
177 * dev_base_head list, and hold dev_base_lock for writing when they do the
178 * actual updates. This allows pure readers to access the list even
179 * while a writer is preparing to update it.
180 *
181 * To put it another way, dev_base_lock is held for writing only to
182 * protect against pure readers; the rtnl semaphore provides the
183 * protection against other writers.
184 *
185 * See, for example usages, register_netdevice() and
186 * unregister_netdevice(), which must be called with the rtnl
187 * semaphore held.
188 */
189DEFINE_RWLOCK(dev_base_lock);
190EXPORT_SYMBOL(dev_base_lock);
191
192static DEFINE_MUTEX(ifalias_mutex);
193
194/* protects napi_hash addition/deletion and napi_gen_id */
195static DEFINE_SPINLOCK(napi_hash_lock);
196
197static unsigned int napi_gen_id = NR_CPUS;
198static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
199
200static DECLARE_RWSEM(devnet_rename_sem);
201
202static inline void dev_base_seq_inc(struct net *net)
203{
204 while (++net->dev_base_seq == 0)
205 ;
206}
207
208static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
209{
210 unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ));
211
212 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
213}
214
215static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
216{
217 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
218}
219
220static inline void rps_lock_irqsave(struct softnet_data *sd,
221 unsigned long *flags)
222{
223 if (IS_ENABLED(CONFIG_RPS))
224 spin_lock_irqsave(&sd->input_pkt_queue.lock, *flags);
225 else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
226 local_irq_save(*flags);
227}
228
229static inline void rps_lock_irq_disable(struct softnet_data *sd)
230{
231 if (IS_ENABLED(CONFIG_RPS))
232 spin_lock_irq(&sd->input_pkt_queue.lock);
233 else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
234 local_irq_disable();
235}
236
237static inline void rps_unlock_irq_restore(struct softnet_data *sd,
238 unsigned long *flags)
239{
240 if (IS_ENABLED(CONFIG_RPS))
241 spin_unlock_irqrestore(&sd->input_pkt_queue.lock, *flags);
242 else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
243 local_irq_restore(*flags);
244}
245
246static inline void rps_unlock_irq_enable(struct softnet_data *sd)
247{
248 if (IS_ENABLED(CONFIG_RPS))
249 spin_unlock_irq(&sd->input_pkt_queue.lock);
250 else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
251 local_irq_enable();
252}
253
254static struct netdev_name_node *netdev_name_node_alloc(struct net_device *dev,
255 const char *name)
256{
257 struct netdev_name_node *name_node;
258
259 name_node = kmalloc(sizeof(*name_node), GFP_KERNEL);
260 if (!name_node)
261 return NULL;
262 INIT_HLIST_NODE(&name_node->hlist);
263 name_node->dev = dev;
264 name_node->name = name;
265 return name_node;
266}
267
268static struct netdev_name_node *
269netdev_name_node_head_alloc(struct net_device *dev)
270{
271 struct netdev_name_node *name_node;
272
273 name_node = netdev_name_node_alloc(dev, dev->name);
274 if (!name_node)
275 return NULL;
276 INIT_LIST_HEAD(&name_node->list);
277 return name_node;
278}
279
280static void netdev_name_node_free(struct netdev_name_node *name_node)
281{
282 kfree(name_node);
283}
284
285static void netdev_name_node_add(struct net *net,
286 struct netdev_name_node *name_node)
287{
288 hlist_add_head_rcu(&name_node->hlist,
289 dev_name_hash(net, name_node->name));
290}
291
292static void netdev_name_node_del(struct netdev_name_node *name_node)
293{
294 hlist_del_rcu(&name_node->hlist);
295}
296
297static struct netdev_name_node *netdev_name_node_lookup(struct net *net,
298 const char *name)
299{
300 struct hlist_head *head = dev_name_hash(net, name);
301 struct netdev_name_node *name_node;
302
303 hlist_for_each_entry(name_node, head, hlist)
304 if (!strcmp(name_node->name, name))
305 return name_node;
306 return NULL;
307}
308
309static struct netdev_name_node *netdev_name_node_lookup_rcu(struct net *net,
310 const char *name)
311{
312 struct hlist_head *head = dev_name_hash(net, name);
313 struct netdev_name_node *name_node;
314
315 hlist_for_each_entry_rcu(name_node, head, hlist)
316 if (!strcmp(name_node->name, name))
317 return name_node;
318 return NULL;
319}
320
321bool netdev_name_in_use(struct net *net, const char *name)
322{
323 return netdev_name_node_lookup(net, name);
324}
325EXPORT_SYMBOL(netdev_name_in_use);
326
327int netdev_name_node_alt_create(struct net_device *dev, const char *name)
328{
329 struct netdev_name_node *name_node;
330 struct net *net = dev_net(dev);
331
332 name_node = netdev_name_node_lookup(net, name);
333 if (name_node)
334 return -EEXIST;
335 name_node = netdev_name_node_alloc(dev, name);
336 if (!name_node)
337 return -ENOMEM;
338 netdev_name_node_add(net, name_node);
339 /* The node that holds dev->name acts as a head of per-device list. */
340 list_add_tail(&name_node->list, &dev->name_node->list);
341
342 return 0;
343}
344
345static void __netdev_name_node_alt_destroy(struct netdev_name_node *name_node)
346{
347 list_del(&name_node->list);
348 netdev_name_node_del(name_node);
349 kfree(name_node->name);
350 netdev_name_node_free(name_node);
351}
352
353int netdev_name_node_alt_destroy(struct net_device *dev, const char *name)
354{
355 struct netdev_name_node *name_node;
356 struct net *net = dev_net(dev);
357
358 name_node = netdev_name_node_lookup(net, name);
359 if (!name_node)
360 return -ENOENT;
361 /* lookup might have found our primary name or a name belonging
362 * to another device.
363 */
364 if (name_node == dev->name_node || name_node->dev != dev)
365 return -EINVAL;
366
367 __netdev_name_node_alt_destroy(name_node);
368
369 return 0;
370}
371
372static void netdev_name_node_alt_flush(struct net_device *dev)
373{
374 struct netdev_name_node *name_node, *tmp;
375
376 list_for_each_entry_safe(name_node, tmp, &dev->name_node->list, list)
377 __netdev_name_node_alt_destroy(name_node);
378}
379
380/* Device list insertion */
381static void list_netdevice(struct net_device *dev)
382{
383 struct net *net = dev_net(dev);
384
385 ASSERT_RTNL();
386
387 write_lock(&dev_base_lock);
388 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
389 netdev_name_node_add(net, dev->name_node);
390 hlist_add_head_rcu(&dev->index_hlist,
391 dev_index_hash(net, dev->ifindex));
392 write_unlock(&dev_base_lock);
393
394 dev_base_seq_inc(net);
395}
396
397/* Device list removal
398 * caller must respect a RCU grace period before freeing/reusing dev
399 */
400static void unlist_netdevice(struct net_device *dev, bool lock)
401{
402 ASSERT_RTNL();
403
404 /* Unlink dev from the device chain */
405 if (lock)
406 write_lock(&dev_base_lock);
407 list_del_rcu(&dev->dev_list);
408 netdev_name_node_del(dev->name_node);
409 hlist_del_rcu(&dev->index_hlist);
410 if (lock)
411 write_unlock(&dev_base_lock);
412
413 dev_base_seq_inc(dev_net(dev));
414}
415
416/*
417 * Our notifier list
418 */
419
420static RAW_NOTIFIER_HEAD(netdev_chain);
421
422/*
423 * Device drivers call our routines to queue packets here. We empty the
424 * queue in the local softnet handler.
425 */
426
427DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
428EXPORT_PER_CPU_SYMBOL(softnet_data);
429
430#ifdef CONFIG_LOCKDEP
431/*
432 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
433 * according to dev->type
434 */
435static const unsigned short netdev_lock_type[] = {
436 ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
437 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
438 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
439 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
440 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
441 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
442 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
443 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
444 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
445 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
446 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
447 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
448 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
449 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
450 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
451
452static const char *const netdev_lock_name[] = {
453 "_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
454 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
455 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
456 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
457 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
458 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
459 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
460 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
461 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
462 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
463 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
464 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
465 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
466 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
467 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
468
469static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
470static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
471
472static inline unsigned short netdev_lock_pos(unsigned short dev_type)
473{
474 int i;
475
476 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
477 if (netdev_lock_type[i] == dev_type)
478 return i;
479 /* the last key is used by default */
480 return ARRAY_SIZE(netdev_lock_type) - 1;
481}
482
483static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
484 unsigned short dev_type)
485{
486 int i;
487
488 i = netdev_lock_pos(dev_type);
489 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
490 netdev_lock_name[i]);
491}
492
493static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
494{
495 int i;
496
497 i = netdev_lock_pos(dev->type);
498 lockdep_set_class_and_name(&dev->addr_list_lock,
499 &netdev_addr_lock_key[i],
500 netdev_lock_name[i]);
501}
502#else
503static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
504 unsigned short dev_type)
505{
506}
507
508static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
509{
510}
511#endif
512
513/*******************************************************************************
514 *
515 * Protocol management and registration routines
516 *
517 *******************************************************************************/
518
519
520/*
521 * Add a protocol ID to the list. Now that the input handler is
522 * smarter we can dispense with all the messy stuff that used to be
523 * here.
524 *
525 * BEWARE!!! Protocol handlers, mangling input packets,
526 * MUST BE last in hash buckets and checking protocol handlers
527 * MUST start from promiscuous ptype_all chain in net_bh.
528 * It is true now, do not change it.
529 * Explanation follows: if protocol handler, mangling packet, will
530 * be the first on list, it is not able to sense, that packet
531 * is cloned and should be copied-on-write, so that it will
532 * change it and subsequent readers will get broken packet.
533 * --ANK (980803)
534 */
535
536static inline struct list_head *ptype_head(const struct packet_type *pt)
537{
538 if (pt->type == htons(ETH_P_ALL))
539 return pt->dev ? &pt->dev->ptype_all : &ptype_all;
540 else
541 return pt->dev ? &pt->dev->ptype_specific :
542 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
543}
544
545/**
546 * dev_add_pack - add packet handler
547 * @pt: packet type declaration
548 *
549 * Add a protocol handler to the networking stack. The passed &packet_type
550 * is linked into kernel lists and may not be freed until it has been
551 * removed from the kernel lists.
552 *
553 * This call does not sleep therefore it can not
554 * guarantee all CPU's that are in middle of receiving packets
555 * will see the new packet type (until the next received packet).
556 */
557
558void dev_add_pack(struct packet_type *pt)
559{
560 struct list_head *head = ptype_head(pt);
561
562 spin_lock(&ptype_lock);
563 list_add_rcu(&pt->list, head);
564 spin_unlock(&ptype_lock);
565}
566EXPORT_SYMBOL(dev_add_pack);
567
568/**
569 * __dev_remove_pack - remove packet handler
570 * @pt: packet type declaration
571 *
572 * Remove a protocol handler that was previously added to the kernel
573 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
574 * from the kernel lists and can be freed or reused once this function
575 * returns.
576 *
577 * The packet type might still be in use by receivers
578 * and must not be freed until after all the CPU's have gone
579 * through a quiescent state.
580 */
581void __dev_remove_pack(struct packet_type *pt)
582{
583 struct list_head *head = ptype_head(pt);
584 struct packet_type *pt1;
585
586 spin_lock(&ptype_lock);
587
588 list_for_each_entry(pt1, head, list) {
589 if (pt == pt1) {
590 list_del_rcu(&pt->list);
591 goto out;
592 }
593 }
594
595 pr_warn("dev_remove_pack: %p not found\n", pt);
596out:
597 spin_unlock(&ptype_lock);
598}
599EXPORT_SYMBOL(__dev_remove_pack);
600
601/**
602 * dev_remove_pack - remove packet handler
603 * @pt: packet type declaration
604 *
605 * Remove a protocol handler that was previously added to the kernel
606 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
607 * from the kernel lists and can be freed or reused once this function
608 * returns.
609 *
610 * This call sleeps to guarantee that no CPU is looking at the packet
611 * type after return.
612 */
613void dev_remove_pack(struct packet_type *pt)
614{
615 __dev_remove_pack(pt);
616
617 synchronize_net();
618}
619EXPORT_SYMBOL(dev_remove_pack);
620
621
622/*******************************************************************************
623 *
624 * Device Interface Subroutines
625 *
626 *******************************************************************************/
627
628/**
629 * dev_get_iflink - get 'iflink' value of a interface
630 * @dev: targeted interface
631 *
632 * Indicates the ifindex the interface is linked to.
633 * Physical interfaces have the same 'ifindex' and 'iflink' values.
634 */
635
636int dev_get_iflink(const struct net_device *dev)
637{
638 if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
639 return dev->netdev_ops->ndo_get_iflink(dev);
640
641 return dev->ifindex;
642}
643EXPORT_SYMBOL(dev_get_iflink);
644
645/**
646 * dev_fill_metadata_dst - Retrieve tunnel egress information.
647 * @dev: targeted interface
648 * @skb: The packet.
649 *
650 * For better visibility of tunnel traffic OVS needs to retrieve
651 * egress tunnel information for a packet. Following API allows
652 * user to get this info.
653 */
654int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
655{
656 struct ip_tunnel_info *info;
657
658 if (!dev->netdev_ops || !dev->netdev_ops->ndo_fill_metadata_dst)
659 return -EINVAL;
660
661 info = skb_tunnel_info_unclone(skb);
662 if (!info)
663 return -ENOMEM;
664 if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
665 return -EINVAL;
666
667 return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
668}
669EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
670
671static struct net_device_path *dev_fwd_path(struct net_device_path_stack *stack)
672{
673 int k = stack->num_paths++;
674
675 if (WARN_ON_ONCE(k >= NET_DEVICE_PATH_STACK_MAX))
676 return NULL;
677
678 return &stack->path[k];
679}
680
681int dev_fill_forward_path(const struct net_device *dev, const u8 *daddr,
682 struct net_device_path_stack *stack)
683{
684 const struct net_device *last_dev;
685 struct net_device_path_ctx ctx = {
686 .dev = dev,
687 };
688 struct net_device_path *path;
689 int ret = 0;
690
691 memcpy(ctx.daddr, daddr, sizeof(ctx.daddr));
692 stack->num_paths = 0;
693 while (ctx.dev && ctx.dev->netdev_ops->ndo_fill_forward_path) {
694 last_dev = ctx.dev;
695 path = dev_fwd_path(stack);
696 if (!path)
697 return -1;
698
699 memset(path, 0, sizeof(struct net_device_path));
700 ret = ctx.dev->netdev_ops->ndo_fill_forward_path(&ctx, path);
701 if (ret < 0)
702 return -1;
703
704 if (WARN_ON_ONCE(last_dev == ctx.dev))
705 return -1;
706 }
707
708 if (!ctx.dev)
709 return ret;
710
711 path = dev_fwd_path(stack);
712 if (!path)
713 return -1;
714 path->type = DEV_PATH_ETHERNET;
715 path->dev = ctx.dev;
716
717 return ret;
718}
719EXPORT_SYMBOL_GPL(dev_fill_forward_path);
720
721/**
722 * __dev_get_by_name - find a device by its name
723 * @net: the applicable net namespace
724 * @name: name to find
725 *
726 * Find an interface by name. Must be called under RTNL semaphore
727 * or @dev_base_lock. If the name is found a pointer to the device
728 * is returned. If the name is not found then %NULL is returned. The
729 * reference counters are not incremented so the caller must be
730 * careful with locks.
731 */
732
733struct net_device *__dev_get_by_name(struct net *net, const char *name)
734{
735 struct netdev_name_node *node_name;
736
737 node_name = netdev_name_node_lookup(net, name);
738 return node_name ? node_name->dev : NULL;
739}
740EXPORT_SYMBOL(__dev_get_by_name);
741
742/**
743 * dev_get_by_name_rcu - find a device by its name
744 * @net: the applicable net namespace
745 * @name: name to find
746 *
747 * Find an interface by name.
748 * If the name is found a pointer to the device is returned.
749 * If the name is not found then %NULL is returned.
750 * The reference counters are not incremented so the caller must be
751 * careful with locks. The caller must hold RCU lock.
752 */
753
754struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
755{
756 struct netdev_name_node *node_name;
757
758 node_name = netdev_name_node_lookup_rcu(net, name);
759 return node_name ? node_name->dev : NULL;
760}
761EXPORT_SYMBOL(dev_get_by_name_rcu);
762
763/**
764 * dev_get_by_name - find a device by its name
765 * @net: the applicable net namespace
766 * @name: name to find
767 *
768 * Find an interface by name. This can be called from any
769 * context and does its own locking. The returned handle has
770 * the usage count incremented and the caller must use dev_put() to
771 * release it when it is no longer needed. %NULL is returned if no
772 * matching device is found.
773 */
774
775struct net_device *dev_get_by_name(struct net *net, const char *name)
776{
777 struct net_device *dev;
778
779 rcu_read_lock();
780 dev = dev_get_by_name_rcu(net, name);
781 dev_hold(dev);
782 rcu_read_unlock();
783 return dev;
784}
785EXPORT_SYMBOL(dev_get_by_name);
786
787/**
788 * __dev_get_by_index - find a device by its ifindex
789 * @net: the applicable net namespace
790 * @ifindex: index of device
791 *
792 * Search for an interface by index. Returns %NULL if the device
793 * is not found or a pointer to the device. The device has not
794 * had its reference counter increased so the caller must be careful
795 * about locking. The caller must hold either the RTNL semaphore
796 * or @dev_base_lock.
797 */
798
799struct net_device *__dev_get_by_index(struct net *net, int ifindex)
800{
801 struct net_device *dev;
802 struct hlist_head *head = dev_index_hash(net, ifindex);
803
804 hlist_for_each_entry(dev, head, index_hlist)
805 if (dev->ifindex == ifindex)
806 return dev;
807
808 return NULL;
809}
810EXPORT_SYMBOL(__dev_get_by_index);
811
812/**
813 * dev_get_by_index_rcu - find a device by its ifindex
814 * @net: the applicable net namespace
815 * @ifindex: index of device
816 *
817 * Search for an interface by index. Returns %NULL if the device
818 * is not found or a pointer to the device. The device has not
819 * had its reference counter increased so the caller must be careful
820 * about locking. The caller must hold RCU lock.
821 */
822
823struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
824{
825 struct net_device *dev;
826 struct hlist_head *head = dev_index_hash(net, ifindex);
827
828 hlist_for_each_entry_rcu(dev, head, index_hlist)
829 if (dev->ifindex == ifindex)
830 return dev;
831
832 return NULL;
833}
834EXPORT_SYMBOL(dev_get_by_index_rcu);
835
836
837/**
838 * dev_get_by_index - find a device by its ifindex
839 * @net: the applicable net namespace
840 * @ifindex: index of device
841 *
842 * Search for an interface by index. Returns NULL if the device
843 * is not found or a pointer to the device. The device returned has
844 * had a reference added and the pointer is safe until the user calls
845 * dev_put to indicate they have finished with it.
846 */
847
848struct net_device *dev_get_by_index(struct net *net, int ifindex)
849{
850 struct net_device *dev;
851
852 rcu_read_lock();
853 dev = dev_get_by_index_rcu(net, ifindex);
854 dev_hold(dev);
855 rcu_read_unlock();
856 return dev;
857}
858EXPORT_SYMBOL(dev_get_by_index);
859
860/**
861 * dev_get_by_napi_id - find a device by napi_id
862 * @napi_id: ID of the NAPI struct
863 *
864 * Search for an interface by NAPI ID. Returns %NULL if the device
865 * is not found or a pointer to the device. The device has not had
866 * its reference counter increased so the caller must be careful
867 * about locking. The caller must hold RCU lock.
868 */
869
870struct net_device *dev_get_by_napi_id(unsigned int napi_id)
871{
872 struct napi_struct *napi;
873
874 WARN_ON_ONCE(!rcu_read_lock_held());
875
876 if (napi_id < MIN_NAPI_ID)
877 return NULL;
878
879 napi = napi_by_id(napi_id);
880
881 return napi ? napi->dev : NULL;
882}
883EXPORT_SYMBOL(dev_get_by_napi_id);
884
885/**
886 * netdev_get_name - get a netdevice name, knowing its ifindex.
887 * @net: network namespace
888 * @name: a pointer to the buffer where the name will be stored.
889 * @ifindex: the ifindex of the interface to get the name from.
890 */
891int netdev_get_name(struct net *net, char *name, int ifindex)
892{
893 struct net_device *dev;
894 int ret;
895
896 down_read(&devnet_rename_sem);
897 rcu_read_lock();
898
899 dev = dev_get_by_index_rcu(net, ifindex);
900 if (!dev) {
901 ret = -ENODEV;
902 goto out;
903 }
904
905 strcpy(name, dev->name);
906
907 ret = 0;
908out:
909 rcu_read_unlock();
910 up_read(&devnet_rename_sem);
911 return ret;
912}
913
914/**
915 * dev_getbyhwaddr_rcu - find a device by its hardware address
916 * @net: the applicable net namespace
917 * @type: media type of device
918 * @ha: hardware address
919 *
920 * Search for an interface by MAC address. Returns NULL if the device
921 * is not found or a pointer to the device.
922 * The caller must hold RCU or RTNL.
923 * The returned device has not had its ref count increased
924 * and the caller must therefore be careful about locking
925 *
926 */
927
928struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
929 const char *ha)
930{
931 struct net_device *dev;
932
933 for_each_netdev_rcu(net, dev)
934 if (dev->type == type &&
935 !memcmp(dev->dev_addr, ha, dev->addr_len))
936 return dev;
937
938 return NULL;
939}
940EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
941
942struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
943{
944 struct net_device *dev, *ret = NULL;
945
946 rcu_read_lock();
947 for_each_netdev_rcu(net, dev)
948 if (dev->type == type) {
949 dev_hold(dev);
950 ret = dev;
951 break;
952 }
953 rcu_read_unlock();
954 return ret;
955}
956EXPORT_SYMBOL(dev_getfirstbyhwtype);
957
958/**
959 * __dev_get_by_flags - find any device with given flags
960 * @net: the applicable net namespace
961 * @if_flags: IFF_* values
962 * @mask: bitmask of bits in if_flags to check
963 *
964 * Search for any interface with the given flags. Returns NULL if a device
965 * is not found or a pointer to the device. Must be called inside
966 * rtnl_lock(), and result refcount is unchanged.
967 */
968
969struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
970 unsigned short mask)
971{
972 struct net_device *dev, *ret;
973
974 ASSERT_RTNL();
975
976 ret = NULL;
977 for_each_netdev(net, dev) {
978 if (((dev->flags ^ if_flags) & mask) == 0) {
979 ret = dev;
980 break;
981 }
982 }
983 return ret;
984}
985EXPORT_SYMBOL(__dev_get_by_flags);
986
987/**
988 * dev_valid_name - check if name is okay for network device
989 * @name: name string
990 *
991 * Network device names need to be valid file names to
992 * allow sysfs to work. We also disallow any kind of
993 * whitespace.
994 */
995bool dev_valid_name(const char *name)
996{
997 if (*name == '\0')
998 return false;
999 if (strnlen(name, IFNAMSIZ) == IFNAMSIZ)
1000 return false;
1001 if (!strcmp(name, ".") || !strcmp(name, ".."))
1002 return false;
1003
1004 while (*name) {
1005 if (*name == '/' || *name == ':' || isspace(*name))
1006 return false;
1007 name++;
1008 }
1009 return true;
1010}
1011EXPORT_SYMBOL(dev_valid_name);
1012
1013/**
1014 * __dev_alloc_name - allocate a name for a device
1015 * @net: network namespace to allocate the device name in
1016 * @name: name format string
1017 * @buf: scratch buffer and result name string
1018 *
1019 * Passed a format string - eg "lt%d" it will try and find a suitable
1020 * id. It scans list of devices to build up a free map, then chooses
1021 * the first empty slot. The caller must hold the dev_base or rtnl lock
1022 * while allocating the name and adding the device in order to avoid
1023 * duplicates.
1024 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1025 * Returns the number of the unit assigned or a negative errno code.
1026 */
1027
1028static int __dev_alloc_name(struct net *net, const char *name, char *buf)
1029{
1030 int i = 0;
1031 const char *p;
1032 const int max_netdevices = 8*PAGE_SIZE;
1033 unsigned long *inuse;
1034 struct net_device *d;
1035
1036 if (!dev_valid_name(name))
1037 return -EINVAL;
1038
1039 p = strchr(name, '%');
1040 if (p) {
1041 /*
1042 * Verify the string as this thing may have come from
1043 * the user. There must be either one "%d" and no other "%"
1044 * characters.
1045 */
1046 if (p[1] != 'd' || strchr(p + 2, '%'))
1047 return -EINVAL;
1048
1049 /* Use one page as a bit array of possible slots */
1050 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
1051 if (!inuse)
1052 return -ENOMEM;
1053
1054 for_each_netdev(net, d) {
1055 struct netdev_name_node *name_node;
1056 list_for_each_entry(name_node, &d->name_node->list, list) {
1057 if (!sscanf(name_node->name, name, &i))
1058 continue;
1059 if (i < 0 || i >= max_netdevices)
1060 continue;
1061
1062 /* avoid cases where sscanf is not exact inverse of printf */
1063 snprintf(buf, IFNAMSIZ, name, i);
1064 if (!strncmp(buf, name_node->name, IFNAMSIZ))
1065 __set_bit(i, inuse);
1066 }
1067 if (!sscanf(d->name, name, &i))
1068 continue;
1069 if (i < 0 || i >= max_netdevices)
1070 continue;
1071
1072 /* avoid cases where sscanf is not exact inverse of printf */
1073 snprintf(buf, IFNAMSIZ, name, i);
1074 if (!strncmp(buf, d->name, IFNAMSIZ))
1075 __set_bit(i, inuse);
1076 }
1077
1078 i = find_first_zero_bit(inuse, max_netdevices);
1079 free_page((unsigned long) inuse);
1080 }
1081
1082 snprintf(buf, IFNAMSIZ, name, i);
1083 if (!netdev_name_in_use(net, buf))
1084 return i;
1085
1086 /* It is possible to run out of possible slots
1087 * when the name is long and there isn't enough space left
1088 * for the digits, or if all bits are used.
1089 */
1090 return -ENFILE;
1091}
1092
1093static int dev_alloc_name_ns(struct net *net,
1094 struct net_device *dev,
1095 const char *name)
1096{
1097 char buf[IFNAMSIZ];
1098 int ret;
1099
1100 BUG_ON(!net);
1101 ret = __dev_alloc_name(net, name, buf);
1102 if (ret >= 0)
1103 strlcpy(dev->name, buf, IFNAMSIZ);
1104 return ret;
1105}
1106
1107/**
1108 * dev_alloc_name - allocate a name for a device
1109 * @dev: device
1110 * @name: name format string
1111 *
1112 * Passed a format string - eg "lt%d" it will try and find a suitable
1113 * id. It scans list of devices to build up a free map, then chooses
1114 * the first empty slot. The caller must hold the dev_base or rtnl lock
1115 * while allocating the name and adding the device in order to avoid
1116 * duplicates.
1117 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1118 * Returns the number of the unit assigned or a negative errno code.
1119 */
1120
1121int dev_alloc_name(struct net_device *dev, const char *name)
1122{
1123 return dev_alloc_name_ns(dev_net(dev), dev, name);
1124}
1125EXPORT_SYMBOL(dev_alloc_name);
1126
1127static int dev_get_valid_name(struct net *net, struct net_device *dev,
1128 const char *name)
1129{
1130 BUG_ON(!net);
1131
1132 if (!dev_valid_name(name))
1133 return -EINVAL;
1134
1135 if (strchr(name, '%'))
1136 return dev_alloc_name_ns(net, dev, name);
1137 else if (netdev_name_in_use(net, name))
1138 return -EEXIST;
1139 else if (dev->name != name)
1140 strlcpy(dev->name, name, IFNAMSIZ);
1141
1142 return 0;
1143}
1144
1145/**
1146 * dev_change_name - change name of a device
1147 * @dev: device
1148 * @newname: name (or format string) must be at least IFNAMSIZ
1149 *
1150 * Change name of a device, can pass format strings "eth%d".
1151 * for wildcarding.
1152 */
1153int dev_change_name(struct net_device *dev, const char *newname)
1154{
1155 unsigned char old_assign_type;
1156 char oldname[IFNAMSIZ];
1157 int err = 0;
1158 int ret;
1159 struct net *net;
1160
1161 ASSERT_RTNL();
1162 BUG_ON(!dev_net(dev));
1163
1164 net = dev_net(dev);
1165
1166 /* Some auto-enslaved devices e.g. failover slaves are
1167 * special, as userspace might rename the device after
1168 * the interface had been brought up and running since
1169 * the point kernel initiated auto-enslavement. Allow
1170 * live name change even when these slave devices are
1171 * up and running.
1172 *
1173 * Typically, users of these auto-enslaving devices
1174 * don't actually care about slave name change, as
1175 * they are supposed to operate on master interface
1176 * directly.
1177 */
1178 if (dev->flags & IFF_UP &&
1179 likely(!(dev->priv_flags & IFF_LIVE_RENAME_OK)))
1180 return -EBUSY;
1181
1182 down_write(&devnet_rename_sem);
1183
1184 if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
1185 up_write(&devnet_rename_sem);
1186 return 0;
1187 }
1188
1189 memcpy(oldname, dev->name, IFNAMSIZ);
1190
1191 err = dev_get_valid_name(net, dev, newname);
1192 if (err < 0) {
1193 up_write(&devnet_rename_sem);
1194 return err;
1195 }
1196
1197 if (oldname[0] && !strchr(oldname, '%'))
1198 netdev_info(dev, "renamed from %s\n", oldname);
1199
1200 old_assign_type = dev->name_assign_type;
1201 dev->name_assign_type = NET_NAME_RENAMED;
1202
1203rollback:
1204 ret = device_rename(&dev->dev, dev->name);
1205 if (ret) {
1206 memcpy(dev->name, oldname, IFNAMSIZ);
1207 dev->name_assign_type = old_assign_type;
1208 up_write(&devnet_rename_sem);
1209 return ret;
1210 }
1211
1212 up_write(&devnet_rename_sem);
1213
1214 netdev_adjacent_rename_links(dev, oldname);
1215
1216 write_lock(&dev_base_lock);
1217 netdev_name_node_del(dev->name_node);
1218 write_unlock(&dev_base_lock);
1219
1220 synchronize_rcu();
1221
1222 write_lock(&dev_base_lock);
1223 netdev_name_node_add(net, dev->name_node);
1224 write_unlock(&dev_base_lock);
1225
1226 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1227 ret = notifier_to_errno(ret);
1228
1229 if (ret) {
1230 /* err >= 0 after dev_alloc_name() or stores the first errno */
1231 if (err >= 0) {
1232 err = ret;
1233 down_write(&devnet_rename_sem);
1234 memcpy(dev->name, oldname, IFNAMSIZ);
1235 memcpy(oldname, newname, IFNAMSIZ);
1236 dev->name_assign_type = old_assign_type;
1237 old_assign_type = NET_NAME_RENAMED;
1238 goto rollback;
1239 } else {
1240 netdev_err(dev, "name change rollback failed: %d\n",
1241 ret);
1242 }
1243 }
1244
1245 return err;
1246}
1247
1248/**
1249 * dev_set_alias - change ifalias of a device
1250 * @dev: device
1251 * @alias: name up to IFALIASZ
1252 * @len: limit of bytes to copy from info
1253 *
1254 * Set ifalias for a device,
1255 */
1256int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1257{
1258 struct dev_ifalias *new_alias = NULL;
1259
1260 if (len >= IFALIASZ)
1261 return -EINVAL;
1262
1263 if (len) {
1264 new_alias = kmalloc(sizeof(*new_alias) + len + 1, GFP_KERNEL);
1265 if (!new_alias)
1266 return -ENOMEM;
1267
1268 memcpy(new_alias->ifalias, alias, len);
1269 new_alias->ifalias[len] = 0;
1270 }
1271
1272 mutex_lock(&ifalias_mutex);
1273 new_alias = rcu_replace_pointer(dev->ifalias, new_alias,
1274 mutex_is_locked(&ifalias_mutex));
1275 mutex_unlock(&ifalias_mutex);
1276
1277 if (new_alias)
1278 kfree_rcu(new_alias, rcuhead);
1279
1280 return len;
1281}
1282EXPORT_SYMBOL(dev_set_alias);
1283
1284/**
1285 * dev_get_alias - get ifalias of a device
1286 * @dev: device
1287 * @name: buffer to store name of ifalias
1288 * @len: size of buffer
1289 *
1290 * get ifalias for a device. Caller must make sure dev cannot go
1291 * away, e.g. rcu read lock or own a reference count to device.
1292 */
1293int dev_get_alias(const struct net_device *dev, char *name, size_t len)
1294{
1295 const struct dev_ifalias *alias;
1296 int ret = 0;
1297
1298 rcu_read_lock();
1299 alias = rcu_dereference(dev->ifalias);
1300 if (alias)
1301 ret = snprintf(name, len, "%s", alias->ifalias);
1302 rcu_read_unlock();
1303
1304 return ret;
1305}
1306
1307/**
1308 * netdev_features_change - device changes features
1309 * @dev: device to cause notification
1310 *
1311 * Called to indicate a device has changed features.
1312 */
1313void netdev_features_change(struct net_device *dev)
1314{
1315 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1316}
1317EXPORT_SYMBOL(netdev_features_change);
1318
1319/**
1320 * netdev_state_change - device changes state
1321 * @dev: device to cause notification
1322 *
1323 * Called to indicate a device has changed state. This function calls
1324 * the notifier chains for netdev_chain and sends a NEWLINK message
1325 * to the routing socket.
1326 */
1327void netdev_state_change(struct net_device *dev)
1328{
1329 if (dev->flags & IFF_UP) {
1330 struct netdev_notifier_change_info change_info = {
1331 .info.dev = dev,
1332 };
1333
1334 call_netdevice_notifiers_info(NETDEV_CHANGE,
1335 &change_info.info);
1336 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL);
1337 }
1338}
1339EXPORT_SYMBOL(netdev_state_change);
1340
1341/**
1342 * __netdev_notify_peers - notify network peers about existence of @dev,
1343 * to be called when rtnl lock is already held.
1344 * @dev: network device
1345 *
1346 * Generate traffic such that interested network peers are aware of
1347 * @dev, such as by generating a gratuitous ARP. This may be used when
1348 * a device wants to inform the rest of the network about some sort of
1349 * reconfiguration such as a failover event or virtual machine
1350 * migration.
1351 */
1352void __netdev_notify_peers(struct net_device *dev)
1353{
1354 ASSERT_RTNL();
1355 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1356 call_netdevice_notifiers(NETDEV_RESEND_IGMP, dev);
1357}
1358EXPORT_SYMBOL(__netdev_notify_peers);
1359
1360/**
1361 * netdev_notify_peers - notify network peers about existence of @dev
1362 * @dev: network device
1363 *
1364 * Generate traffic such that interested network peers are aware of
1365 * @dev, such as by generating a gratuitous ARP. This may be used when
1366 * a device wants to inform the rest of the network about some sort of
1367 * reconfiguration such as a failover event or virtual machine
1368 * migration.
1369 */
1370void netdev_notify_peers(struct net_device *dev)
1371{
1372 rtnl_lock();
1373 __netdev_notify_peers(dev);
1374 rtnl_unlock();
1375}
1376EXPORT_SYMBOL(netdev_notify_peers);
1377
1378static int napi_threaded_poll(void *data);
1379
1380static int napi_kthread_create(struct napi_struct *n)
1381{
1382 int err = 0;
1383
1384 /* Create and wake up the kthread once to put it in
1385 * TASK_INTERRUPTIBLE mode to avoid the blocked task
1386 * warning and work with loadavg.
1387 */
1388 n->thread = kthread_run(napi_threaded_poll, n, "napi/%s-%d",
1389 n->dev->name, n->napi_id);
1390 if (IS_ERR(n->thread)) {
1391 err = PTR_ERR(n->thread);
1392 pr_err("kthread_run failed with err %d\n", err);
1393 n->thread = NULL;
1394 }
1395
1396 return err;
1397}
1398
1399static int __dev_open(struct net_device *dev, struct netlink_ext_ack *extack)
1400{
1401 const struct net_device_ops *ops = dev->netdev_ops;
1402 int ret;
1403
1404 ASSERT_RTNL();
1405 dev_addr_check(dev);
1406
1407 if (!netif_device_present(dev)) {
1408 /* may be detached because parent is runtime-suspended */
1409 if (dev->dev.parent)
1410 pm_runtime_resume(dev->dev.parent);
1411 if (!netif_device_present(dev))
1412 return -ENODEV;
1413 }
1414
1415 /* Block netpoll from trying to do any rx path servicing.
1416 * If we don't do this there is a chance ndo_poll_controller
1417 * or ndo_poll may be running while we open the device
1418 */
1419 netpoll_poll_disable(dev);
1420
1421 ret = call_netdevice_notifiers_extack(NETDEV_PRE_UP, dev, extack);
1422 ret = notifier_to_errno(ret);
1423 if (ret)
1424 return ret;
1425
1426 set_bit(__LINK_STATE_START, &dev->state);
1427
1428 if (ops->ndo_validate_addr)
1429 ret = ops->ndo_validate_addr(dev);
1430
1431 if (!ret && ops->ndo_open)
1432 ret = ops->ndo_open(dev);
1433
1434 netpoll_poll_enable(dev);
1435
1436 if (ret)
1437 clear_bit(__LINK_STATE_START, &dev->state);
1438 else {
1439 dev->flags |= IFF_UP;
1440 dev_set_rx_mode(dev);
1441 dev_activate(dev);
1442 add_device_randomness(dev->dev_addr, dev->addr_len);
1443 }
1444
1445 return ret;
1446}
1447
1448/**
1449 * dev_open - prepare an interface for use.
1450 * @dev: device to open
1451 * @extack: netlink extended ack
1452 *
1453 * Takes a device from down to up state. The device's private open
1454 * function is invoked and then the multicast lists are loaded. Finally
1455 * the device is moved into the up state and a %NETDEV_UP message is
1456 * sent to the netdev notifier chain.
1457 *
1458 * Calling this function on an active interface is a nop. On a failure
1459 * a negative errno code is returned.
1460 */
1461int dev_open(struct net_device *dev, struct netlink_ext_ack *extack)
1462{
1463 int ret;
1464
1465 if (dev->flags & IFF_UP)
1466 return 0;
1467
1468 ret = __dev_open(dev, extack);
1469 if (ret < 0)
1470 return ret;
1471
1472 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1473 call_netdevice_notifiers(NETDEV_UP, dev);
1474
1475 return ret;
1476}
1477EXPORT_SYMBOL(dev_open);
1478
1479static void __dev_close_many(struct list_head *head)
1480{
1481 struct net_device *dev;
1482
1483 ASSERT_RTNL();
1484 might_sleep();
1485
1486 list_for_each_entry(dev, head, close_list) {
1487 /* Temporarily disable netpoll until the interface is down */
1488 netpoll_poll_disable(dev);
1489
1490 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1491
1492 clear_bit(__LINK_STATE_START, &dev->state);
1493
1494 /* Synchronize to scheduled poll. We cannot touch poll list, it
1495 * can be even on different cpu. So just clear netif_running().
1496 *
1497 * dev->stop() will invoke napi_disable() on all of it's
1498 * napi_struct instances on this device.
1499 */
1500 smp_mb__after_atomic(); /* Commit netif_running(). */
1501 }
1502
1503 dev_deactivate_many(head);
1504
1505 list_for_each_entry(dev, head, close_list) {
1506 const struct net_device_ops *ops = dev->netdev_ops;
1507
1508 /*
1509 * Call the device specific close. This cannot fail.
1510 * Only if device is UP
1511 *
1512 * We allow it to be called even after a DETACH hot-plug
1513 * event.
1514 */
1515 if (ops->ndo_stop)
1516 ops->ndo_stop(dev);
1517
1518 dev->flags &= ~IFF_UP;
1519 netpoll_poll_enable(dev);
1520 }
1521}
1522
1523static void __dev_close(struct net_device *dev)
1524{
1525 LIST_HEAD(single);
1526
1527 list_add(&dev->close_list, &single);
1528 __dev_close_many(&single);
1529 list_del(&single);
1530}
1531
1532void dev_close_many(struct list_head *head, bool unlink)
1533{
1534 struct net_device *dev, *tmp;
1535
1536 /* Remove the devices that don't need to be closed */
1537 list_for_each_entry_safe(dev, tmp, head, close_list)
1538 if (!(dev->flags & IFF_UP))
1539 list_del_init(&dev->close_list);
1540
1541 __dev_close_many(head);
1542
1543 list_for_each_entry_safe(dev, tmp, head, close_list) {
1544 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1545 call_netdevice_notifiers(NETDEV_DOWN, dev);
1546 if (unlink)
1547 list_del_init(&dev->close_list);
1548 }
1549}
1550EXPORT_SYMBOL(dev_close_many);
1551
1552/**
1553 * dev_close - shutdown an interface.
1554 * @dev: device to shutdown
1555 *
1556 * This function moves an active device into down state. A
1557 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1558 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1559 * chain.
1560 */
1561void dev_close(struct net_device *dev)
1562{
1563 if (dev->flags & IFF_UP) {
1564 LIST_HEAD(single);
1565
1566 list_add(&dev->close_list, &single);
1567 dev_close_many(&single, true);
1568 list_del(&single);
1569 }
1570}
1571EXPORT_SYMBOL(dev_close);
1572
1573
1574/**
1575 * dev_disable_lro - disable Large Receive Offload on a device
1576 * @dev: device
1577 *
1578 * Disable Large Receive Offload (LRO) on a net device. Must be
1579 * called under RTNL. This is needed if received packets may be
1580 * forwarded to another interface.
1581 */
1582void dev_disable_lro(struct net_device *dev)
1583{
1584 struct net_device *lower_dev;
1585 struct list_head *iter;
1586
1587 dev->wanted_features &= ~NETIF_F_LRO;
1588 netdev_update_features(dev);
1589
1590 if (unlikely(dev->features & NETIF_F_LRO))
1591 netdev_WARN(dev, "failed to disable LRO!\n");
1592
1593 netdev_for_each_lower_dev(dev, lower_dev, iter)
1594 dev_disable_lro(lower_dev);
1595}
1596EXPORT_SYMBOL(dev_disable_lro);
1597
1598/**
1599 * dev_disable_gro_hw - disable HW Generic Receive Offload on a device
1600 * @dev: device
1601 *
1602 * Disable HW Generic Receive Offload (GRO_HW) on a net device. Must be
1603 * called under RTNL. This is needed if Generic XDP is installed on
1604 * the device.
1605 */
1606static void dev_disable_gro_hw(struct net_device *dev)
1607{
1608 dev->wanted_features &= ~NETIF_F_GRO_HW;
1609 netdev_update_features(dev);
1610
1611 if (unlikely(dev->features & NETIF_F_GRO_HW))
1612 netdev_WARN(dev, "failed to disable GRO_HW!\n");
1613}
1614
1615const char *netdev_cmd_to_name(enum netdev_cmd cmd)
1616{
1617#define N(val) \
1618 case NETDEV_##val: \
1619 return "NETDEV_" __stringify(val);
1620 switch (cmd) {
1621 N(UP) N(DOWN) N(REBOOT) N(CHANGE) N(REGISTER) N(UNREGISTER)
1622 N(CHANGEMTU) N(CHANGEADDR) N(GOING_DOWN) N(CHANGENAME) N(FEAT_CHANGE)
1623 N(BONDING_FAILOVER) N(PRE_UP) N(PRE_TYPE_CHANGE) N(POST_TYPE_CHANGE)
1624 N(POST_INIT) N(RELEASE) N(NOTIFY_PEERS) N(JOIN) N(CHANGEUPPER)
1625 N(RESEND_IGMP) N(PRECHANGEMTU) N(CHANGEINFODATA) N(BONDING_INFO)
1626 N(PRECHANGEUPPER) N(CHANGELOWERSTATE) N(UDP_TUNNEL_PUSH_INFO)
1627 N(UDP_TUNNEL_DROP_INFO) N(CHANGE_TX_QUEUE_LEN)
1628 N(CVLAN_FILTER_PUSH_INFO) N(CVLAN_FILTER_DROP_INFO)
1629 N(SVLAN_FILTER_PUSH_INFO) N(SVLAN_FILTER_DROP_INFO)
1630 N(PRE_CHANGEADDR) N(OFFLOAD_XSTATS_ENABLE) N(OFFLOAD_XSTATS_DISABLE)
1631 N(OFFLOAD_XSTATS_REPORT_USED) N(OFFLOAD_XSTATS_REPORT_DELTA)
1632 }
1633#undef N
1634 return "UNKNOWN_NETDEV_EVENT";
1635}
1636EXPORT_SYMBOL_GPL(netdev_cmd_to_name);
1637
1638static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1639 struct net_device *dev)
1640{
1641 struct netdev_notifier_info info = {
1642 .dev = dev,
1643 };
1644
1645 return nb->notifier_call(nb, val, &info);
1646}
1647
1648static int call_netdevice_register_notifiers(struct notifier_block *nb,
1649 struct net_device *dev)
1650{
1651 int err;
1652
1653 err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
1654 err = notifier_to_errno(err);
1655 if (err)
1656 return err;
1657
1658 if (!(dev->flags & IFF_UP))
1659 return 0;
1660
1661 call_netdevice_notifier(nb, NETDEV_UP, dev);
1662 return 0;
1663}
1664
1665static void call_netdevice_unregister_notifiers(struct notifier_block *nb,
1666 struct net_device *dev)
1667{
1668 if (dev->flags & IFF_UP) {
1669 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1670 dev);
1671 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1672 }
1673 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1674}
1675
1676static int call_netdevice_register_net_notifiers(struct notifier_block *nb,
1677 struct net *net)
1678{
1679 struct net_device *dev;
1680 int err;
1681
1682 for_each_netdev(net, dev) {
1683 err = call_netdevice_register_notifiers(nb, dev);
1684 if (err)
1685 goto rollback;
1686 }
1687 return 0;
1688
1689rollback:
1690 for_each_netdev_continue_reverse(net, dev)
1691 call_netdevice_unregister_notifiers(nb, dev);
1692 return err;
1693}
1694
1695static void call_netdevice_unregister_net_notifiers(struct notifier_block *nb,
1696 struct net *net)
1697{
1698 struct net_device *dev;
1699
1700 for_each_netdev(net, dev)
1701 call_netdevice_unregister_notifiers(nb, dev);
1702}
1703
1704static int dev_boot_phase = 1;
1705
1706/**
1707 * register_netdevice_notifier - register a network notifier block
1708 * @nb: notifier
1709 *
1710 * Register a notifier to be called when network device events occur.
1711 * The notifier passed is linked into the kernel structures and must
1712 * not be reused until it has been unregistered. A negative errno code
1713 * is returned on a failure.
1714 *
1715 * When registered all registration and up events are replayed
1716 * to the new notifier to allow device to have a race free
1717 * view of the network device list.
1718 */
1719
1720int register_netdevice_notifier(struct notifier_block *nb)
1721{
1722 struct net *net;
1723 int err;
1724
1725 /* Close race with setup_net() and cleanup_net() */
1726 down_write(&pernet_ops_rwsem);
1727 rtnl_lock();
1728 err = raw_notifier_chain_register(&netdev_chain, nb);
1729 if (err)
1730 goto unlock;
1731 if (dev_boot_phase)
1732 goto unlock;
1733 for_each_net(net) {
1734 err = call_netdevice_register_net_notifiers(nb, net);
1735 if (err)
1736 goto rollback;
1737 }
1738
1739unlock:
1740 rtnl_unlock();
1741 up_write(&pernet_ops_rwsem);
1742 return err;
1743
1744rollback:
1745 for_each_net_continue_reverse(net)
1746 call_netdevice_unregister_net_notifiers(nb, net);
1747
1748 raw_notifier_chain_unregister(&netdev_chain, nb);
1749 goto unlock;
1750}
1751EXPORT_SYMBOL(register_netdevice_notifier);
1752
1753/**
1754 * unregister_netdevice_notifier - unregister a network notifier block
1755 * @nb: notifier
1756 *
1757 * Unregister a notifier previously registered by
1758 * register_netdevice_notifier(). The notifier is unlinked into the
1759 * kernel structures and may then be reused. A negative errno code
1760 * is returned on a failure.
1761 *
1762 * After unregistering unregister and down device events are synthesized
1763 * for all devices on the device list to the removed notifier to remove
1764 * the need for special case cleanup code.
1765 */
1766
1767int unregister_netdevice_notifier(struct notifier_block *nb)
1768{
1769 struct net *net;
1770 int err;
1771
1772 /* Close race with setup_net() and cleanup_net() */
1773 down_write(&pernet_ops_rwsem);
1774 rtnl_lock();
1775 err = raw_notifier_chain_unregister(&netdev_chain, nb);
1776 if (err)
1777 goto unlock;
1778
1779 for_each_net(net)
1780 call_netdevice_unregister_net_notifiers(nb, net);
1781
1782unlock:
1783 rtnl_unlock();
1784 up_write(&pernet_ops_rwsem);
1785 return err;
1786}
1787EXPORT_SYMBOL(unregister_netdevice_notifier);
1788
1789static int __register_netdevice_notifier_net(struct net *net,
1790 struct notifier_block *nb,
1791 bool ignore_call_fail)
1792{
1793 int err;
1794
1795 err = raw_notifier_chain_register(&net->netdev_chain, nb);
1796 if (err)
1797 return err;
1798 if (dev_boot_phase)
1799 return 0;
1800
1801 err = call_netdevice_register_net_notifiers(nb, net);
1802 if (err && !ignore_call_fail)
1803 goto chain_unregister;
1804
1805 return 0;
1806
1807chain_unregister:
1808 raw_notifier_chain_unregister(&net->netdev_chain, nb);
1809 return err;
1810}
1811
1812static int __unregister_netdevice_notifier_net(struct net *net,
1813 struct notifier_block *nb)
1814{
1815 int err;
1816
1817 err = raw_notifier_chain_unregister(&net->netdev_chain, nb);
1818 if (err)
1819 return err;
1820
1821 call_netdevice_unregister_net_notifiers(nb, net);
1822 return 0;
1823}
1824
1825/**
1826 * register_netdevice_notifier_net - register a per-netns network notifier block
1827 * @net: network namespace
1828 * @nb: notifier
1829 *
1830 * Register a notifier to be called when network device events occur.
1831 * The notifier passed is linked into the kernel structures and must
1832 * not be reused until it has been unregistered. A negative errno code
1833 * is returned on a failure.
1834 *
1835 * When registered all registration and up events are replayed
1836 * to the new notifier to allow device to have a race free
1837 * view of the network device list.
1838 */
1839
1840int register_netdevice_notifier_net(struct net *net, struct notifier_block *nb)
1841{
1842 int err;
1843
1844 rtnl_lock();
1845 err = __register_netdevice_notifier_net(net, nb, false);
1846 rtnl_unlock();
1847 return err;
1848}
1849EXPORT_SYMBOL(register_netdevice_notifier_net);
1850
1851/**
1852 * unregister_netdevice_notifier_net - unregister a per-netns
1853 * network notifier block
1854 * @net: network namespace
1855 * @nb: notifier
1856 *
1857 * Unregister a notifier previously registered by
1858 * register_netdevice_notifier(). The notifier is unlinked into the
1859 * kernel structures and may then be reused. A negative errno code
1860 * is returned on a failure.
1861 *
1862 * After unregistering unregister and down device events are synthesized
1863 * for all devices on the device list to the removed notifier to remove
1864 * the need for special case cleanup code.
1865 */
1866
1867int unregister_netdevice_notifier_net(struct net *net,
1868 struct notifier_block *nb)
1869{
1870 int err;
1871
1872 rtnl_lock();
1873 err = __unregister_netdevice_notifier_net(net, nb);
1874 rtnl_unlock();
1875 return err;
1876}
1877EXPORT_SYMBOL(unregister_netdevice_notifier_net);
1878
1879int register_netdevice_notifier_dev_net(struct net_device *dev,
1880 struct notifier_block *nb,
1881 struct netdev_net_notifier *nn)
1882{
1883 int err;
1884
1885 rtnl_lock();
1886 err = __register_netdevice_notifier_net(dev_net(dev), nb, false);
1887 if (!err) {
1888 nn->nb = nb;
1889 list_add(&nn->list, &dev->net_notifier_list);
1890 }
1891 rtnl_unlock();
1892 return err;
1893}
1894EXPORT_SYMBOL(register_netdevice_notifier_dev_net);
1895
1896int unregister_netdevice_notifier_dev_net(struct net_device *dev,
1897 struct notifier_block *nb,
1898 struct netdev_net_notifier *nn)
1899{
1900 int err;
1901
1902 rtnl_lock();
1903 list_del(&nn->list);
1904 err = __unregister_netdevice_notifier_net(dev_net(dev), nb);
1905 rtnl_unlock();
1906 return err;
1907}
1908EXPORT_SYMBOL(unregister_netdevice_notifier_dev_net);
1909
1910static void move_netdevice_notifiers_dev_net(struct net_device *dev,
1911 struct net *net)
1912{
1913 struct netdev_net_notifier *nn;
1914
1915 list_for_each_entry(nn, &dev->net_notifier_list, list) {
1916 __unregister_netdevice_notifier_net(dev_net(dev), nn->nb);
1917 __register_netdevice_notifier_net(net, nn->nb, true);
1918 }
1919}
1920
1921/**
1922 * call_netdevice_notifiers_info - call all network notifier blocks
1923 * @val: value passed unmodified to notifier function
1924 * @info: notifier information data
1925 *
1926 * Call all network notifier blocks. Parameters and return value
1927 * are as for raw_notifier_call_chain().
1928 */
1929
1930static int call_netdevice_notifiers_info(unsigned long val,
1931 struct netdev_notifier_info *info)
1932{
1933 struct net *net = dev_net(info->dev);
1934 int ret;
1935
1936 ASSERT_RTNL();
1937
1938 /* Run per-netns notifier block chain first, then run the global one.
1939 * Hopefully, one day, the global one is going to be removed after
1940 * all notifier block registrators get converted to be per-netns.
1941 */
1942 ret = raw_notifier_call_chain(&net->netdev_chain, val, info);
1943 if (ret & NOTIFY_STOP_MASK)
1944 return ret;
1945 return raw_notifier_call_chain(&netdev_chain, val, info);
1946}
1947
1948/**
1949 * call_netdevice_notifiers_info_robust - call per-netns notifier blocks
1950 * for and rollback on error
1951 * @val_up: value passed unmodified to notifier function
1952 * @val_down: value passed unmodified to the notifier function when
1953 * recovering from an error on @val_up
1954 * @info: notifier information data
1955 *
1956 * Call all per-netns network notifier blocks, but not notifier blocks on
1957 * the global notifier chain. Parameters and return value are as for
1958 * raw_notifier_call_chain_robust().
1959 */
1960
1961static int
1962call_netdevice_notifiers_info_robust(unsigned long val_up,
1963 unsigned long val_down,
1964 struct netdev_notifier_info *info)
1965{
1966 struct net *net = dev_net(info->dev);
1967
1968 ASSERT_RTNL();
1969
1970 return raw_notifier_call_chain_robust(&net->netdev_chain,
1971 val_up, val_down, info);
1972}
1973
1974static int call_netdevice_notifiers_extack(unsigned long val,
1975 struct net_device *dev,
1976 struct netlink_ext_ack *extack)
1977{
1978 struct netdev_notifier_info info = {
1979 .dev = dev,
1980 .extack = extack,
1981 };
1982
1983 return call_netdevice_notifiers_info(val, &info);
1984}
1985
1986/**
1987 * call_netdevice_notifiers - call all network notifier blocks
1988 * @val: value passed unmodified to notifier function
1989 * @dev: net_device pointer passed unmodified to notifier function
1990 *
1991 * Call all network notifier blocks. Parameters and return value
1992 * are as for raw_notifier_call_chain().
1993 */
1994
1995int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
1996{
1997 return call_netdevice_notifiers_extack(val, dev, NULL);
1998}
1999EXPORT_SYMBOL(call_netdevice_notifiers);
2000
2001/**
2002 * call_netdevice_notifiers_mtu - call all network notifier blocks
2003 * @val: value passed unmodified to notifier function
2004 * @dev: net_device pointer passed unmodified to notifier function
2005 * @arg: additional u32 argument passed to the notifier function
2006 *
2007 * Call all network notifier blocks. Parameters and return value
2008 * are as for raw_notifier_call_chain().
2009 */
2010static int call_netdevice_notifiers_mtu(unsigned long val,
2011 struct net_device *dev, u32 arg)
2012{
2013 struct netdev_notifier_info_ext info = {
2014 .info.dev = dev,
2015 .ext.mtu = arg,
2016 };
2017
2018 BUILD_BUG_ON(offsetof(struct netdev_notifier_info_ext, info) != 0);
2019
2020 return call_netdevice_notifiers_info(val, &info.info);
2021}
2022
2023#ifdef CONFIG_NET_INGRESS
2024static DEFINE_STATIC_KEY_FALSE(ingress_needed_key);
2025
2026void net_inc_ingress_queue(void)
2027{
2028 static_branch_inc(&ingress_needed_key);
2029}
2030EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
2031
2032void net_dec_ingress_queue(void)
2033{
2034 static_branch_dec(&ingress_needed_key);
2035}
2036EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
2037#endif
2038
2039#ifdef CONFIG_NET_EGRESS
2040static DEFINE_STATIC_KEY_FALSE(egress_needed_key);
2041
2042void net_inc_egress_queue(void)
2043{
2044 static_branch_inc(&egress_needed_key);
2045}
2046EXPORT_SYMBOL_GPL(net_inc_egress_queue);
2047
2048void net_dec_egress_queue(void)
2049{
2050 static_branch_dec(&egress_needed_key);
2051}
2052EXPORT_SYMBOL_GPL(net_dec_egress_queue);
2053#endif
2054
2055DEFINE_STATIC_KEY_FALSE(netstamp_needed_key);
2056EXPORT_SYMBOL(netstamp_needed_key);
2057#ifdef CONFIG_JUMP_LABEL
2058static atomic_t netstamp_needed_deferred;
2059static atomic_t netstamp_wanted;
2060static void netstamp_clear(struct work_struct *work)
2061{
2062 int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
2063 int wanted;
2064
2065 wanted = atomic_add_return(deferred, &netstamp_wanted);
2066 if (wanted > 0)
2067 static_branch_enable(&netstamp_needed_key);
2068 else
2069 static_branch_disable(&netstamp_needed_key);
2070}
2071static DECLARE_WORK(netstamp_work, netstamp_clear);
2072#endif
2073
2074void net_enable_timestamp(void)
2075{
2076#ifdef CONFIG_JUMP_LABEL
2077 int wanted;
2078
2079 while (1) {
2080 wanted = atomic_read(&netstamp_wanted);
2081 if (wanted <= 0)
2082 break;
2083 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted + 1) == wanted)
2084 return;
2085 }
2086 atomic_inc(&netstamp_needed_deferred);
2087 schedule_work(&netstamp_work);
2088#else
2089 static_branch_inc(&netstamp_needed_key);
2090#endif
2091}
2092EXPORT_SYMBOL(net_enable_timestamp);
2093
2094void net_disable_timestamp(void)
2095{
2096#ifdef CONFIG_JUMP_LABEL
2097 int wanted;
2098
2099 while (1) {
2100 wanted = atomic_read(&netstamp_wanted);
2101 if (wanted <= 1)
2102 break;
2103 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted - 1) == wanted)
2104 return;
2105 }
2106 atomic_dec(&netstamp_needed_deferred);
2107 schedule_work(&netstamp_work);
2108#else
2109 static_branch_dec(&netstamp_needed_key);
2110#endif
2111}
2112EXPORT_SYMBOL(net_disable_timestamp);
2113
2114static inline void net_timestamp_set(struct sk_buff *skb)
2115{
2116 skb->tstamp = 0;
2117 skb->mono_delivery_time = 0;
2118 if (static_branch_unlikely(&netstamp_needed_key))
2119 skb->tstamp = ktime_get_real();
2120}
2121
2122#define net_timestamp_check(COND, SKB) \
2123 if (static_branch_unlikely(&netstamp_needed_key)) { \
2124 if ((COND) && !(SKB)->tstamp) \
2125 (SKB)->tstamp = ktime_get_real(); \
2126 } \
2127
2128bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb)
2129{
2130 return __is_skb_forwardable(dev, skb, true);
2131}
2132EXPORT_SYMBOL_GPL(is_skb_forwardable);
2133
2134static int __dev_forward_skb2(struct net_device *dev, struct sk_buff *skb,
2135 bool check_mtu)
2136{
2137 int ret = ____dev_forward_skb(dev, skb, check_mtu);
2138
2139 if (likely(!ret)) {
2140 skb->protocol = eth_type_trans(skb, dev);
2141 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
2142 }
2143
2144 return ret;
2145}
2146
2147int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
2148{
2149 return __dev_forward_skb2(dev, skb, true);
2150}
2151EXPORT_SYMBOL_GPL(__dev_forward_skb);
2152
2153/**
2154 * dev_forward_skb - loopback an skb to another netif
2155 *
2156 * @dev: destination network device
2157 * @skb: buffer to forward
2158 *
2159 * return values:
2160 * NET_RX_SUCCESS (no congestion)
2161 * NET_RX_DROP (packet was dropped, but freed)
2162 *
2163 * dev_forward_skb can be used for injecting an skb from the
2164 * start_xmit function of one device into the receive queue
2165 * of another device.
2166 *
2167 * The receiving device may be in another namespace, so
2168 * we have to clear all information in the skb that could
2169 * impact namespace isolation.
2170 */
2171int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
2172{
2173 return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
2174}
2175EXPORT_SYMBOL_GPL(dev_forward_skb);
2176
2177int dev_forward_skb_nomtu(struct net_device *dev, struct sk_buff *skb)
2178{
2179 return __dev_forward_skb2(dev, skb, false) ?: netif_rx_internal(skb);
2180}
2181
2182static inline int deliver_skb(struct sk_buff *skb,
2183 struct packet_type *pt_prev,
2184 struct net_device *orig_dev)
2185{
2186 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
2187 return -ENOMEM;
2188 refcount_inc(&skb->users);
2189 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
2190}
2191
2192static inline void deliver_ptype_list_skb(struct sk_buff *skb,
2193 struct packet_type **pt,
2194 struct net_device *orig_dev,
2195 __be16 type,
2196 struct list_head *ptype_list)
2197{
2198 struct packet_type *ptype, *pt_prev = *pt;
2199
2200 list_for_each_entry_rcu(ptype, ptype_list, list) {
2201 if (ptype->type != type)
2202 continue;
2203 if (pt_prev)
2204 deliver_skb(skb, pt_prev, orig_dev);
2205 pt_prev = ptype;
2206 }
2207 *pt = pt_prev;
2208}
2209
2210static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
2211{
2212 if (!ptype->af_packet_priv || !skb->sk)
2213 return false;
2214
2215 if (ptype->id_match)
2216 return ptype->id_match(ptype, skb->sk);
2217 else if ((struct sock *)ptype->af_packet_priv == skb->sk)
2218 return true;
2219
2220 return false;
2221}
2222
2223/**
2224 * dev_nit_active - return true if any network interface taps are in use
2225 *
2226 * @dev: network device to check for the presence of taps
2227 */
2228bool dev_nit_active(struct net_device *dev)
2229{
2230 return !list_empty(&ptype_all) || !list_empty(&dev->ptype_all);
2231}
2232EXPORT_SYMBOL_GPL(dev_nit_active);
2233
2234/*
2235 * Support routine. Sends outgoing frames to any network
2236 * taps currently in use.
2237 */
2238
2239void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
2240{
2241 struct packet_type *ptype;
2242 struct sk_buff *skb2 = NULL;
2243 struct packet_type *pt_prev = NULL;
2244 struct list_head *ptype_list = &ptype_all;
2245
2246 rcu_read_lock();
2247again:
2248 list_for_each_entry_rcu(ptype, ptype_list, list) {
2249 if (ptype->ignore_outgoing)
2250 continue;
2251
2252 /* Never send packets back to the socket
2253 * they originated from - MvS (miquels@drinkel.ow.org)
2254 */
2255 if (skb_loop_sk(ptype, skb))
2256 continue;
2257
2258 if (pt_prev) {
2259 deliver_skb(skb2, pt_prev, skb->dev);
2260 pt_prev = ptype;
2261 continue;
2262 }
2263
2264 /* need to clone skb, done only once */
2265 skb2 = skb_clone(skb, GFP_ATOMIC);
2266 if (!skb2)
2267 goto out_unlock;
2268
2269 net_timestamp_set(skb2);
2270
2271 /* skb->nh should be correctly
2272 * set by sender, so that the second statement is
2273 * just protection against buggy protocols.
2274 */
2275 skb_reset_mac_header(skb2);
2276
2277 if (skb_network_header(skb2) < skb2->data ||
2278 skb_network_header(skb2) > skb_tail_pointer(skb2)) {
2279 net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
2280 ntohs(skb2->protocol),
2281 dev->name);
2282 skb_reset_network_header(skb2);
2283 }
2284
2285 skb2->transport_header = skb2->network_header;
2286 skb2->pkt_type = PACKET_OUTGOING;
2287 pt_prev = ptype;
2288 }
2289
2290 if (ptype_list == &ptype_all) {
2291 ptype_list = &dev->ptype_all;
2292 goto again;
2293 }
2294out_unlock:
2295 if (pt_prev) {
2296 if (!skb_orphan_frags_rx(skb2, GFP_ATOMIC))
2297 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
2298 else
2299 kfree_skb(skb2);
2300 }
2301 rcu_read_unlock();
2302}
2303EXPORT_SYMBOL_GPL(dev_queue_xmit_nit);
2304
2305/**
2306 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
2307 * @dev: Network device
2308 * @txq: number of queues available
2309 *
2310 * If real_num_tx_queues is changed the tc mappings may no longer be
2311 * valid. To resolve this verify the tc mapping remains valid and if
2312 * not NULL the mapping. With no priorities mapping to this
2313 * offset/count pair it will no longer be used. In the worst case TC0
2314 * is invalid nothing can be done so disable priority mappings. If is
2315 * expected that drivers will fix this mapping if they can before
2316 * calling netif_set_real_num_tx_queues.
2317 */
2318static void netif_setup_tc(struct net_device *dev, unsigned int txq)
2319{
2320 int i;
2321 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2322
2323 /* If TC0 is invalidated disable TC mapping */
2324 if (tc->offset + tc->count > txq) {
2325 netdev_warn(dev, "Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
2326 dev->num_tc = 0;
2327 return;
2328 }
2329
2330 /* Invalidated prio to tc mappings set to TC0 */
2331 for (i = 1; i < TC_BITMASK + 1; i++) {
2332 int q = netdev_get_prio_tc_map(dev, i);
2333
2334 tc = &dev->tc_to_txq[q];
2335 if (tc->offset + tc->count > txq) {
2336 netdev_warn(dev, "Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
2337 i, q);
2338 netdev_set_prio_tc_map(dev, i, 0);
2339 }
2340 }
2341}
2342
2343int netdev_txq_to_tc(struct net_device *dev, unsigned int txq)
2344{
2345 if (dev->num_tc) {
2346 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2347 int i;
2348
2349 /* walk through the TCs and see if it falls into any of them */
2350 for (i = 0; i < TC_MAX_QUEUE; i++, tc++) {
2351 if ((txq - tc->offset) < tc->count)
2352 return i;
2353 }
2354
2355 /* didn't find it, just return -1 to indicate no match */
2356 return -1;
2357 }
2358
2359 return 0;
2360}
2361EXPORT_SYMBOL(netdev_txq_to_tc);
2362
2363#ifdef CONFIG_XPS
2364static struct static_key xps_needed __read_mostly;
2365static struct static_key xps_rxqs_needed __read_mostly;
2366static DEFINE_MUTEX(xps_map_mutex);
2367#define xmap_dereference(P) \
2368 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
2369
2370static bool remove_xps_queue(struct xps_dev_maps *dev_maps,
2371 struct xps_dev_maps *old_maps, int tci, u16 index)
2372{
2373 struct xps_map *map = NULL;
2374 int pos;
2375
2376 if (dev_maps)
2377 map = xmap_dereference(dev_maps->attr_map[tci]);
2378 if (!map)
2379 return false;
2380
2381 for (pos = map->len; pos--;) {
2382 if (map->queues[pos] != index)
2383 continue;
2384
2385 if (map->len > 1) {
2386 map->queues[pos] = map->queues[--map->len];
2387 break;
2388 }
2389
2390 if (old_maps)
2391 RCU_INIT_POINTER(old_maps->attr_map[tci], NULL);
2392 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
2393 kfree_rcu(map, rcu);
2394 return false;
2395 }
2396
2397 return true;
2398}
2399
2400static bool remove_xps_queue_cpu(struct net_device *dev,
2401 struct xps_dev_maps *dev_maps,
2402 int cpu, u16 offset, u16 count)
2403{
2404 int num_tc = dev_maps->num_tc;
2405 bool active = false;
2406 int tci;
2407
2408 for (tci = cpu * num_tc; num_tc--; tci++) {
2409 int i, j;
2410
2411 for (i = count, j = offset; i--; j++) {
2412 if (!remove_xps_queue(dev_maps, NULL, tci, j))
2413 break;
2414 }
2415
2416 active |= i < 0;
2417 }
2418
2419 return active;
2420}
2421
2422static void reset_xps_maps(struct net_device *dev,
2423 struct xps_dev_maps *dev_maps,
2424 enum xps_map_type type)
2425{
2426 static_key_slow_dec_cpuslocked(&xps_needed);
2427 if (type == XPS_RXQS)
2428 static_key_slow_dec_cpuslocked(&xps_rxqs_needed);
2429
2430 RCU_INIT_POINTER(dev->xps_maps[type], NULL);
2431
2432 kfree_rcu(dev_maps, rcu);
2433}
2434
2435static void clean_xps_maps(struct net_device *dev, enum xps_map_type type,
2436 u16 offset, u16 count)
2437{
2438 struct xps_dev_maps *dev_maps;
2439 bool active = false;
2440 int i, j;
2441
2442 dev_maps = xmap_dereference(dev->xps_maps[type]);
2443 if (!dev_maps)
2444 return;
2445
2446 for (j = 0; j < dev_maps->nr_ids; j++)
2447 active |= remove_xps_queue_cpu(dev, dev_maps, j, offset, count);
2448 if (!active)
2449 reset_xps_maps(dev, dev_maps, type);
2450
2451 if (type == XPS_CPUS) {
2452 for (i = offset + (count - 1); count--; i--)
2453 netdev_queue_numa_node_write(
2454 netdev_get_tx_queue(dev, i), NUMA_NO_NODE);
2455 }
2456}
2457
2458static void netif_reset_xps_queues(struct net_device *dev, u16 offset,
2459 u16 count)
2460{
2461 if (!static_key_false(&xps_needed))
2462 return;
2463
2464 cpus_read_lock();
2465 mutex_lock(&xps_map_mutex);
2466
2467 if (static_key_false(&xps_rxqs_needed))
2468 clean_xps_maps(dev, XPS_RXQS, offset, count);
2469
2470 clean_xps_maps(dev, XPS_CPUS, offset, count);
2471
2472 mutex_unlock(&xps_map_mutex);
2473 cpus_read_unlock();
2474}
2475
2476static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
2477{
2478 netif_reset_xps_queues(dev, index, dev->num_tx_queues - index);
2479}
2480
2481static struct xps_map *expand_xps_map(struct xps_map *map, int attr_index,
2482 u16 index, bool is_rxqs_map)
2483{
2484 struct xps_map *new_map;
2485 int alloc_len = XPS_MIN_MAP_ALLOC;
2486 int i, pos;
2487
2488 for (pos = 0; map && pos < map->len; pos++) {
2489 if (map->queues[pos] != index)
2490 continue;
2491 return map;
2492 }
2493
2494 /* Need to add tx-queue to this CPU's/rx-queue's existing map */
2495 if (map) {
2496 if (pos < map->alloc_len)
2497 return map;
2498
2499 alloc_len = map->alloc_len * 2;
2500 }
2501
2502 /* Need to allocate new map to store tx-queue on this CPU's/rx-queue's
2503 * map
2504 */
2505 if (is_rxqs_map)
2506 new_map = kzalloc(XPS_MAP_SIZE(alloc_len), GFP_KERNEL);
2507 else
2508 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
2509 cpu_to_node(attr_index));
2510 if (!new_map)
2511 return NULL;
2512
2513 for (i = 0; i < pos; i++)
2514 new_map->queues[i] = map->queues[i];
2515 new_map->alloc_len = alloc_len;
2516 new_map->len = pos;
2517
2518 return new_map;
2519}
2520
2521/* Copy xps maps at a given index */
2522static void xps_copy_dev_maps(struct xps_dev_maps *dev_maps,
2523 struct xps_dev_maps *new_dev_maps, int index,
2524 int tc, bool skip_tc)
2525{
2526 int i, tci = index * dev_maps->num_tc;
2527 struct xps_map *map;
2528
2529 /* copy maps belonging to foreign traffic classes */
2530 for (i = 0; i < dev_maps->num_tc; i++, tci++) {
2531 if (i == tc && skip_tc)
2532 continue;
2533
2534 /* fill in the new device map from the old device map */
2535 map = xmap_dereference(dev_maps->attr_map[tci]);
2536 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2537 }
2538}
2539
2540/* Must be called under cpus_read_lock */
2541int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask,
2542 u16 index, enum xps_map_type type)
2543{
2544 struct xps_dev_maps *dev_maps, *new_dev_maps = NULL, *old_dev_maps = NULL;
2545 const unsigned long *online_mask = NULL;
2546 bool active = false, copy = false;
2547 int i, j, tci, numa_node_id = -2;
2548 int maps_sz, num_tc = 1, tc = 0;
2549 struct xps_map *map, *new_map;
2550 unsigned int nr_ids;
2551
2552 if (dev->num_tc) {
2553 /* Do not allow XPS on subordinate device directly */
2554 num_tc = dev->num_tc;
2555 if (num_tc < 0)
2556 return -EINVAL;
2557
2558 /* If queue belongs to subordinate dev use its map */
2559 dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev;
2560
2561 tc = netdev_txq_to_tc(dev, index);
2562 if (tc < 0)
2563 return -EINVAL;
2564 }
2565
2566 mutex_lock(&xps_map_mutex);
2567
2568 dev_maps = xmap_dereference(dev->xps_maps[type]);
2569 if (type == XPS_RXQS) {
2570 maps_sz = XPS_RXQ_DEV_MAPS_SIZE(num_tc, dev->num_rx_queues);
2571 nr_ids = dev->num_rx_queues;
2572 } else {
2573 maps_sz = XPS_CPU_DEV_MAPS_SIZE(num_tc);
2574 if (num_possible_cpus() > 1)
2575 online_mask = cpumask_bits(cpu_online_mask);
2576 nr_ids = nr_cpu_ids;
2577 }
2578
2579 if (maps_sz < L1_CACHE_BYTES)
2580 maps_sz = L1_CACHE_BYTES;
2581
2582 /* The old dev_maps could be larger or smaller than the one we're
2583 * setting up now, as dev->num_tc or nr_ids could have been updated in
2584 * between. We could try to be smart, but let's be safe instead and only
2585 * copy foreign traffic classes if the two map sizes match.
2586 */
2587 if (dev_maps &&
2588 dev_maps->num_tc == num_tc && dev_maps->nr_ids == nr_ids)
2589 copy = true;
2590
2591 /* allocate memory for queue storage */
2592 for (j = -1; j = netif_attrmask_next_and(j, online_mask, mask, nr_ids),
2593 j < nr_ids;) {
2594 if (!new_dev_maps) {
2595 new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
2596 if (!new_dev_maps) {
2597 mutex_unlock(&xps_map_mutex);
2598 return -ENOMEM;
2599 }
2600
2601 new_dev_maps->nr_ids = nr_ids;
2602 new_dev_maps->num_tc = num_tc;
2603 }
2604
2605 tci = j * num_tc + tc;
2606 map = copy ? xmap_dereference(dev_maps->attr_map[tci]) : NULL;
2607
2608 map = expand_xps_map(map, j, index, type == XPS_RXQS);
2609 if (!map)
2610 goto error;
2611
2612 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2613 }
2614
2615 if (!new_dev_maps)
2616 goto out_no_new_maps;
2617
2618 if (!dev_maps) {
2619 /* Increment static keys at most once per type */
2620 static_key_slow_inc_cpuslocked(&xps_needed);
2621 if (type == XPS_RXQS)
2622 static_key_slow_inc_cpuslocked(&xps_rxqs_needed);
2623 }
2624
2625 for (j = 0; j < nr_ids; j++) {
2626 bool skip_tc = false;
2627
2628 tci = j * num_tc + tc;
2629 if (netif_attr_test_mask(j, mask, nr_ids) &&
2630 netif_attr_test_online(j, online_mask, nr_ids)) {
2631 /* add tx-queue to CPU/rx-queue maps */
2632 int pos = 0;
2633
2634 skip_tc = true;
2635
2636 map = xmap_dereference(new_dev_maps->attr_map[tci]);
2637 while ((pos < map->len) && (map->queues[pos] != index))
2638 pos++;
2639
2640 if (pos == map->len)
2641 map->queues[map->len++] = index;
2642#ifdef CONFIG_NUMA
2643 if (type == XPS_CPUS) {
2644 if (numa_node_id == -2)
2645 numa_node_id = cpu_to_node(j);
2646 else if (numa_node_id != cpu_to_node(j))
2647 numa_node_id = -1;
2648 }
2649#endif
2650 }
2651
2652 if (copy)
2653 xps_copy_dev_maps(dev_maps, new_dev_maps, j, tc,
2654 skip_tc);
2655 }
2656
2657 rcu_assign_pointer(dev->xps_maps[type], new_dev_maps);
2658
2659 /* Cleanup old maps */
2660 if (!dev_maps)
2661 goto out_no_old_maps;
2662
2663 for (j = 0; j < dev_maps->nr_ids; j++) {
2664 for (i = num_tc, tci = j * dev_maps->num_tc; i--; tci++) {
2665 map = xmap_dereference(dev_maps->attr_map[tci]);
2666 if (!map)
2667 continue;
2668
2669 if (copy) {
2670 new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
2671 if (map == new_map)
2672 continue;
2673 }
2674
2675 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
2676 kfree_rcu(map, rcu);
2677 }
2678 }
2679
2680 old_dev_maps = dev_maps;
2681
2682out_no_old_maps:
2683 dev_maps = new_dev_maps;
2684 active = true;
2685
2686out_no_new_maps:
2687 if (type == XPS_CPUS)
2688 /* update Tx queue numa node */
2689 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2690 (numa_node_id >= 0) ?
2691 numa_node_id : NUMA_NO_NODE);
2692
2693 if (!dev_maps)
2694 goto out_no_maps;
2695
2696 /* removes tx-queue from unused CPUs/rx-queues */
2697 for (j = 0; j < dev_maps->nr_ids; j++) {
2698 tci = j * dev_maps->num_tc;
2699
2700 for (i = 0; i < dev_maps->num_tc; i++, tci++) {
2701 if (i == tc &&
2702 netif_attr_test_mask(j, mask, dev_maps->nr_ids) &&
2703 netif_attr_test_online(j, online_mask, dev_maps->nr_ids))
2704 continue;
2705
2706 active |= remove_xps_queue(dev_maps,
2707 copy ? old_dev_maps : NULL,
2708 tci, index);
2709 }
2710 }
2711
2712 if (old_dev_maps)
2713 kfree_rcu(old_dev_maps, rcu);
2714
2715 /* free map if not active */
2716 if (!active)
2717 reset_xps_maps(dev, dev_maps, type);
2718
2719out_no_maps:
2720 mutex_unlock(&xps_map_mutex);
2721
2722 return 0;
2723error:
2724 /* remove any maps that we added */
2725 for (j = 0; j < nr_ids; j++) {
2726 for (i = num_tc, tci = j * num_tc; i--; tci++) {
2727 new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
2728 map = copy ?
2729 xmap_dereference(dev_maps->attr_map[tci]) :
2730 NULL;
2731 if (new_map && new_map != map)
2732 kfree(new_map);
2733 }
2734 }
2735
2736 mutex_unlock(&xps_map_mutex);
2737
2738 kfree(new_dev_maps);
2739 return -ENOMEM;
2740}
2741EXPORT_SYMBOL_GPL(__netif_set_xps_queue);
2742
2743int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2744 u16 index)
2745{
2746 int ret;
2747
2748 cpus_read_lock();
2749 ret = __netif_set_xps_queue(dev, cpumask_bits(mask), index, XPS_CPUS);
2750 cpus_read_unlock();
2751
2752 return ret;
2753}
2754EXPORT_SYMBOL(netif_set_xps_queue);
2755
2756#endif
2757static void netdev_unbind_all_sb_channels(struct net_device *dev)
2758{
2759 struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
2760
2761 /* Unbind any subordinate channels */
2762 while (txq-- != &dev->_tx[0]) {
2763 if (txq->sb_dev)
2764 netdev_unbind_sb_channel(dev, txq->sb_dev);
2765 }
2766}
2767
2768void netdev_reset_tc(struct net_device *dev)
2769{
2770#ifdef CONFIG_XPS
2771 netif_reset_xps_queues_gt(dev, 0);
2772#endif
2773 netdev_unbind_all_sb_channels(dev);
2774
2775 /* Reset TC configuration of device */
2776 dev->num_tc = 0;
2777 memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq));
2778 memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map));
2779}
2780EXPORT_SYMBOL(netdev_reset_tc);
2781
2782int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset)
2783{
2784 if (tc >= dev->num_tc)
2785 return -EINVAL;
2786
2787#ifdef CONFIG_XPS
2788 netif_reset_xps_queues(dev, offset, count);
2789#endif
2790 dev->tc_to_txq[tc].count = count;
2791 dev->tc_to_txq[tc].offset = offset;
2792 return 0;
2793}
2794EXPORT_SYMBOL(netdev_set_tc_queue);
2795
2796int netdev_set_num_tc(struct net_device *dev, u8 num_tc)
2797{
2798 if (num_tc > TC_MAX_QUEUE)
2799 return -EINVAL;
2800
2801#ifdef CONFIG_XPS
2802 netif_reset_xps_queues_gt(dev, 0);
2803#endif
2804 netdev_unbind_all_sb_channels(dev);
2805
2806 dev->num_tc = num_tc;
2807 return 0;
2808}
2809EXPORT_SYMBOL(netdev_set_num_tc);
2810
2811void netdev_unbind_sb_channel(struct net_device *dev,
2812 struct net_device *sb_dev)
2813{
2814 struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
2815
2816#ifdef CONFIG_XPS
2817 netif_reset_xps_queues_gt(sb_dev, 0);
2818#endif
2819 memset(sb_dev->tc_to_txq, 0, sizeof(sb_dev->tc_to_txq));
2820 memset(sb_dev->prio_tc_map, 0, sizeof(sb_dev->prio_tc_map));
2821
2822 while (txq-- != &dev->_tx[0]) {
2823 if (txq->sb_dev == sb_dev)
2824 txq->sb_dev = NULL;
2825 }
2826}
2827EXPORT_SYMBOL(netdev_unbind_sb_channel);
2828
2829int netdev_bind_sb_channel_queue(struct net_device *dev,
2830 struct net_device *sb_dev,
2831 u8 tc, u16 count, u16 offset)
2832{
2833 /* Make certain the sb_dev and dev are already configured */
2834 if (sb_dev->num_tc >= 0 || tc >= dev->num_tc)
2835 return -EINVAL;
2836
2837 /* We cannot hand out queues we don't have */
2838 if ((offset + count) > dev->real_num_tx_queues)
2839 return -EINVAL;
2840
2841 /* Record the mapping */
2842 sb_dev->tc_to_txq[tc].count = count;
2843 sb_dev->tc_to_txq[tc].offset = offset;
2844
2845 /* Provide a way for Tx queue to find the tc_to_txq map or
2846 * XPS map for itself.
2847 */
2848 while (count--)
2849 netdev_get_tx_queue(dev, count + offset)->sb_dev = sb_dev;
2850
2851 return 0;
2852}
2853EXPORT_SYMBOL(netdev_bind_sb_channel_queue);
2854
2855int netdev_set_sb_channel(struct net_device *dev, u16 channel)
2856{
2857 /* Do not use a multiqueue device to represent a subordinate channel */
2858 if (netif_is_multiqueue(dev))
2859 return -ENODEV;
2860
2861 /* We allow channels 1 - 32767 to be used for subordinate channels.
2862 * Channel 0 is meant to be "native" mode and used only to represent
2863 * the main root device. We allow writing 0 to reset the device back
2864 * to normal mode after being used as a subordinate channel.
2865 */
2866 if (channel > S16_MAX)
2867 return -EINVAL;
2868
2869 dev->num_tc = -channel;
2870
2871 return 0;
2872}
2873EXPORT_SYMBOL(netdev_set_sb_channel);
2874
2875/*
2876 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2877 * greater than real_num_tx_queues stale skbs on the qdisc must be flushed.
2878 */
2879int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
2880{
2881 bool disabling;
2882 int rc;
2883
2884 disabling = txq < dev->real_num_tx_queues;
2885
2886 if (txq < 1 || txq > dev->num_tx_queues)
2887 return -EINVAL;
2888
2889 if (dev->reg_state == NETREG_REGISTERED ||
2890 dev->reg_state == NETREG_UNREGISTERING) {
2891 ASSERT_RTNL();
2892
2893 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2894 txq);
2895 if (rc)
2896 return rc;
2897
2898 if (dev->num_tc)
2899 netif_setup_tc(dev, txq);
2900
2901 dev_qdisc_change_real_num_tx(dev, txq);
2902
2903 dev->real_num_tx_queues = txq;
2904
2905 if (disabling) {
2906 synchronize_net();
2907 qdisc_reset_all_tx_gt(dev, txq);
2908#ifdef CONFIG_XPS
2909 netif_reset_xps_queues_gt(dev, txq);
2910#endif
2911 }
2912 } else {
2913 dev->real_num_tx_queues = txq;
2914 }
2915
2916 return 0;
2917}
2918EXPORT_SYMBOL(netif_set_real_num_tx_queues);
2919
2920#ifdef CONFIG_SYSFS
2921/**
2922 * netif_set_real_num_rx_queues - set actual number of RX queues used
2923 * @dev: Network device
2924 * @rxq: Actual number of RX queues
2925 *
2926 * This must be called either with the rtnl_lock held or before
2927 * registration of the net device. Returns 0 on success, or a
2928 * negative error code. If called before registration, it always
2929 * succeeds.
2930 */
2931int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
2932{
2933 int rc;
2934
2935 if (rxq < 1 || rxq > dev->num_rx_queues)
2936 return -EINVAL;
2937
2938 if (dev->reg_state == NETREG_REGISTERED) {
2939 ASSERT_RTNL();
2940
2941 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2942 rxq);
2943 if (rc)
2944 return rc;
2945 }
2946
2947 dev->real_num_rx_queues = rxq;
2948 return 0;
2949}
2950EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2951#endif
2952
2953/**
2954 * netif_set_real_num_queues - set actual number of RX and TX queues used
2955 * @dev: Network device
2956 * @txq: Actual number of TX queues
2957 * @rxq: Actual number of RX queues
2958 *
2959 * Set the real number of both TX and RX queues.
2960 * Does nothing if the number of queues is already correct.
2961 */
2962int netif_set_real_num_queues(struct net_device *dev,
2963 unsigned int txq, unsigned int rxq)
2964{
2965 unsigned int old_rxq = dev->real_num_rx_queues;
2966 int err;
2967
2968 if (txq < 1 || txq > dev->num_tx_queues ||
2969 rxq < 1 || rxq > dev->num_rx_queues)
2970 return -EINVAL;
2971
2972 /* Start from increases, so the error path only does decreases -
2973 * decreases can't fail.
2974 */
2975 if (rxq > dev->real_num_rx_queues) {
2976 err = netif_set_real_num_rx_queues(dev, rxq);
2977 if (err)
2978 return err;
2979 }
2980 if (txq > dev->real_num_tx_queues) {
2981 err = netif_set_real_num_tx_queues(dev, txq);
2982 if (err)
2983 goto undo_rx;
2984 }
2985 if (rxq < dev->real_num_rx_queues)
2986 WARN_ON(netif_set_real_num_rx_queues(dev, rxq));
2987 if (txq < dev->real_num_tx_queues)
2988 WARN_ON(netif_set_real_num_tx_queues(dev, txq));
2989
2990 return 0;
2991undo_rx:
2992 WARN_ON(netif_set_real_num_rx_queues(dev, old_rxq));
2993 return err;
2994}
2995EXPORT_SYMBOL(netif_set_real_num_queues);
2996
2997/**
2998 * netif_set_tso_max_size() - set the max size of TSO frames supported
2999 * @dev: netdev to update
3000 * @size: max skb->len of a TSO frame
3001 *
3002 * Set the limit on the size of TSO super-frames the device can handle.
3003 * Unless explicitly set the stack will assume the value of
3004 * %GSO_LEGACY_MAX_SIZE.
3005 */
3006void netif_set_tso_max_size(struct net_device *dev, unsigned int size)
3007{
3008 dev->tso_max_size = min(GSO_MAX_SIZE, size);
3009 if (size < READ_ONCE(dev->gso_max_size))
3010 netif_set_gso_max_size(dev, size);
3011}
3012EXPORT_SYMBOL(netif_set_tso_max_size);
3013
3014/**
3015 * netif_set_tso_max_segs() - set the max number of segs supported for TSO
3016 * @dev: netdev to update
3017 * @segs: max number of TCP segments
3018 *
3019 * Set the limit on the number of TCP segments the device can generate from
3020 * a single TSO super-frame.
3021 * Unless explicitly set the stack will assume the value of %GSO_MAX_SEGS.
3022 */
3023void netif_set_tso_max_segs(struct net_device *dev, unsigned int segs)
3024{
3025 dev->tso_max_segs = segs;
3026 if (segs < READ_ONCE(dev->gso_max_segs))
3027 netif_set_gso_max_segs(dev, segs);
3028}
3029EXPORT_SYMBOL(netif_set_tso_max_segs);
3030
3031/**
3032 * netif_inherit_tso_max() - copy all TSO limits from a lower device to an upper
3033 * @to: netdev to update
3034 * @from: netdev from which to copy the limits
3035 */
3036void netif_inherit_tso_max(struct net_device *to, const struct net_device *from)
3037{
3038 netif_set_tso_max_size(to, from->tso_max_size);
3039 netif_set_tso_max_segs(to, from->tso_max_segs);
3040}
3041EXPORT_SYMBOL(netif_inherit_tso_max);
3042
3043/**
3044 * netif_get_num_default_rss_queues - default number of RSS queues
3045 *
3046 * Default value is the number of physical cores if there are only 1 or 2, or
3047 * divided by 2 if there are more.
3048 */
3049int netif_get_num_default_rss_queues(void)
3050{
3051 cpumask_var_t cpus;
3052 int cpu, count = 0;
3053
3054 if (unlikely(is_kdump_kernel() || !zalloc_cpumask_var(&cpus, GFP_KERNEL)))
3055 return 1;
3056
3057 cpumask_copy(cpus, cpu_online_mask);
3058 for_each_cpu(cpu, cpus) {
3059 ++count;
3060 cpumask_andnot(cpus, cpus, topology_sibling_cpumask(cpu));
3061 }
3062 free_cpumask_var(cpus);
3063
3064 return count > 2 ? DIV_ROUND_UP(count, 2) : count;
3065}
3066EXPORT_SYMBOL(netif_get_num_default_rss_queues);
3067
3068static void __netif_reschedule(struct Qdisc *q)
3069{
3070 struct softnet_data *sd;
3071 unsigned long flags;
3072
3073 local_irq_save(flags);
3074 sd = this_cpu_ptr(&softnet_data);
3075 q->next_sched = NULL;
3076 *sd->output_queue_tailp = q;
3077 sd->output_queue_tailp = &q->next_sched;
3078 raise_softirq_irqoff(NET_TX_SOFTIRQ);
3079 local_irq_restore(flags);
3080}
3081
3082void __netif_schedule(struct Qdisc *q)
3083{
3084 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
3085 __netif_reschedule(q);
3086}
3087EXPORT_SYMBOL(__netif_schedule);
3088
3089struct dev_kfree_skb_cb {
3090 enum skb_free_reason reason;
3091};
3092
3093static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
3094{
3095 return (struct dev_kfree_skb_cb *)skb->cb;
3096}
3097
3098void netif_schedule_queue(struct netdev_queue *txq)
3099{
3100 rcu_read_lock();
3101 if (!netif_xmit_stopped(txq)) {
3102 struct Qdisc *q = rcu_dereference(txq->qdisc);
3103
3104 __netif_schedule(q);
3105 }
3106 rcu_read_unlock();
3107}
3108EXPORT_SYMBOL(netif_schedule_queue);
3109
3110void netif_tx_wake_queue(struct netdev_queue *dev_queue)
3111{
3112 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
3113 struct Qdisc *q;
3114
3115 rcu_read_lock();
3116 q = rcu_dereference(dev_queue->qdisc);
3117 __netif_schedule(q);
3118 rcu_read_unlock();
3119 }
3120}
3121EXPORT_SYMBOL(netif_tx_wake_queue);
3122
3123void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
3124{
3125 unsigned long flags;
3126
3127 if (unlikely(!skb))
3128 return;
3129
3130 if (likely(refcount_read(&skb->users) == 1)) {
3131 smp_rmb();
3132 refcount_set(&skb->users, 0);
3133 } else if (likely(!refcount_dec_and_test(&skb->users))) {
3134 return;
3135 }
3136 get_kfree_skb_cb(skb)->reason = reason;
3137 local_irq_save(flags);
3138 skb->next = __this_cpu_read(softnet_data.completion_queue);
3139 __this_cpu_write(softnet_data.completion_queue, skb);
3140 raise_softirq_irqoff(NET_TX_SOFTIRQ);
3141 local_irq_restore(flags);
3142}
3143EXPORT_SYMBOL(__dev_kfree_skb_irq);
3144
3145void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason)
3146{
3147 if (in_hardirq() || irqs_disabled())
3148 __dev_kfree_skb_irq(skb, reason);
3149 else
3150 dev_kfree_skb(skb);
3151}
3152EXPORT_SYMBOL(__dev_kfree_skb_any);
3153
3154
3155/**
3156 * netif_device_detach - mark device as removed
3157 * @dev: network device
3158 *
3159 * Mark device as removed from system and therefore no longer available.
3160 */
3161void netif_device_detach(struct net_device *dev)
3162{
3163 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
3164 netif_running(dev)) {
3165 netif_tx_stop_all_queues(dev);
3166 }
3167}
3168EXPORT_SYMBOL(netif_device_detach);
3169
3170/**
3171 * netif_device_attach - mark device as attached
3172 * @dev: network device
3173 *
3174 * Mark device as attached from system and restart if needed.
3175 */
3176void netif_device_attach(struct net_device *dev)
3177{
3178 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
3179 netif_running(dev)) {
3180 netif_tx_wake_all_queues(dev);
3181 __netdev_watchdog_up(dev);
3182 }
3183}
3184EXPORT_SYMBOL(netif_device_attach);
3185
3186/*
3187 * Returns a Tx hash based on the given packet descriptor a Tx queues' number
3188 * to be used as a distribution range.
3189 */
3190static u16 skb_tx_hash(const struct net_device *dev,
3191 const struct net_device *sb_dev,
3192 struct sk_buff *skb)
3193{
3194 u32 hash;
3195 u16 qoffset = 0;
3196 u16 qcount = dev->real_num_tx_queues;
3197
3198 if (dev->num_tc) {
3199 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
3200
3201 qoffset = sb_dev->tc_to_txq[tc].offset;
3202 qcount = sb_dev->tc_to_txq[tc].count;
3203 if (unlikely(!qcount)) {
3204 net_warn_ratelimited("%s: invalid qcount, qoffset %u for tc %u\n",
3205 sb_dev->name, qoffset, tc);
3206 qoffset = 0;
3207 qcount = dev->real_num_tx_queues;
3208 }
3209 }
3210
3211 if (skb_rx_queue_recorded(skb)) {
3212 hash = skb_get_rx_queue(skb);
3213 if (hash >= qoffset)
3214 hash -= qoffset;
3215 while (unlikely(hash >= qcount))
3216 hash -= qcount;
3217 return hash + qoffset;
3218 }
3219
3220 return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
3221}
3222
3223static void skb_warn_bad_offload(const struct sk_buff *skb)
3224{
3225 static const netdev_features_t null_features;
3226 struct net_device *dev = skb->dev;
3227 const char *name = "";
3228
3229 if (!net_ratelimit())
3230 return;
3231
3232 if (dev) {
3233 if (dev->dev.parent)
3234 name = dev_driver_string(dev->dev.parent);
3235 else
3236 name = netdev_name(dev);
3237 }
3238 skb_dump(KERN_WARNING, skb, false);
3239 WARN(1, "%s: caps=(%pNF, %pNF)\n",
3240 name, dev ? &dev->features : &null_features,
3241 skb->sk ? &skb->sk->sk_route_caps : &null_features);
3242}
3243
3244/*
3245 * Invalidate hardware checksum when packet is to be mangled, and
3246 * complete checksum manually on outgoing path.
3247 */
3248int skb_checksum_help(struct sk_buff *skb)
3249{
3250 __wsum csum;
3251 int ret = 0, offset;
3252
3253 if (skb->ip_summed == CHECKSUM_COMPLETE)
3254 goto out_set_summed;
3255
3256 if (unlikely(skb_is_gso(skb))) {
3257 skb_warn_bad_offload(skb);
3258 return -EINVAL;
3259 }
3260
3261 /* Before computing a checksum, we should make sure no frag could
3262 * be modified by an external entity : checksum could be wrong.
3263 */
3264 if (skb_has_shared_frag(skb)) {
3265 ret = __skb_linearize(skb);
3266 if (ret)
3267 goto out;
3268 }
3269
3270 offset = skb_checksum_start_offset(skb);
3271 ret = -EINVAL;
3272 if (WARN_ON_ONCE(offset >= skb_headlen(skb))) {
3273 DO_ONCE_LITE(skb_dump, KERN_ERR, skb, false);
3274 goto out;
3275 }
3276 csum = skb_checksum(skb, offset, skb->len - offset, 0);
3277
3278 offset += skb->csum_offset;
3279 if (WARN_ON_ONCE(offset + sizeof(__sum16) > skb_headlen(skb))) {
3280 DO_ONCE_LITE(skb_dump, KERN_ERR, skb, false);
3281 goto out;
3282 }
3283 ret = skb_ensure_writable(skb, offset + sizeof(__sum16));
3284 if (ret)
3285 goto out;
3286
3287 *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
3288out_set_summed:
3289 skb->ip_summed = CHECKSUM_NONE;
3290out:
3291 return ret;
3292}
3293EXPORT_SYMBOL(skb_checksum_help);
3294
3295int skb_crc32c_csum_help(struct sk_buff *skb)
3296{
3297 __le32 crc32c_csum;
3298 int ret = 0, offset, start;
3299
3300 if (skb->ip_summed != CHECKSUM_PARTIAL)
3301 goto out;
3302
3303 if (unlikely(skb_is_gso(skb)))
3304 goto out;
3305
3306 /* Before computing a checksum, we should make sure no frag could
3307 * be modified by an external entity : checksum could be wrong.
3308 */
3309 if (unlikely(skb_has_shared_frag(skb))) {
3310 ret = __skb_linearize(skb);
3311 if (ret)
3312 goto out;
3313 }
3314 start = skb_checksum_start_offset(skb);
3315 offset = start + offsetof(struct sctphdr, checksum);
3316 if (WARN_ON_ONCE(offset >= skb_headlen(skb))) {
3317 ret = -EINVAL;
3318 goto out;
3319 }
3320
3321 ret = skb_ensure_writable(skb, offset + sizeof(__le32));
3322 if (ret)
3323 goto out;
3324
3325 crc32c_csum = cpu_to_le32(~__skb_checksum(skb, start,
3326 skb->len - start, ~(__u32)0,
3327 crc32c_csum_stub));
3328 *(__le32 *)(skb->data + offset) = crc32c_csum;
3329 skb->ip_summed = CHECKSUM_NONE;
3330 skb->csum_not_inet = 0;
3331out:
3332 return ret;
3333}
3334
3335__be16 skb_network_protocol(struct sk_buff *skb, int *depth)
3336{
3337 __be16 type = skb->protocol;
3338
3339 /* Tunnel gso handlers can set protocol to ethernet. */
3340 if (type == htons(ETH_P_TEB)) {
3341 struct ethhdr *eth;
3342
3343 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
3344 return 0;
3345
3346 eth = (struct ethhdr *)skb->data;
3347 type = eth->h_proto;
3348 }
3349
3350 return __vlan_get_protocol(skb, type, depth);
3351}
3352
3353/* openvswitch calls this on rx path, so we need a different check.
3354 */
3355static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
3356{
3357 if (tx_path)
3358 return skb->ip_summed != CHECKSUM_PARTIAL &&
3359 skb->ip_summed != CHECKSUM_UNNECESSARY;
3360
3361 return skb->ip_summed == CHECKSUM_NONE;
3362}
3363
3364/**
3365 * __skb_gso_segment - Perform segmentation on skb.
3366 * @skb: buffer to segment
3367 * @features: features for the output path (see dev->features)
3368 * @tx_path: whether it is called in TX path
3369 *
3370 * This function segments the given skb and returns a list of segments.
3371 *
3372 * It may return NULL if the skb requires no segmentation. This is
3373 * only possible when GSO is used for verifying header integrity.
3374 *
3375 * Segmentation preserves SKB_GSO_CB_OFFSET bytes of previous skb cb.
3376 */
3377struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
3378 netdev_features_t features, bool tx_path)
3379{
3380 struct sk_buff *segs;
3381
3382 if (unlikely(skb_needs_check(skb, tx_path))) {
3383 int err;
3384
3385 /* We're going to init ->check field in TCP or UDP header */
3386 err = skb_cow_head(skb, 0);
3387 if (err < 0)
3388 return ERR_PTR(err);
3389 }
3390
3391 /* Only report GSO partial support if it will enable us to
3392 * support segmentation on this frame without needing additional
3393 * work.
3394 */
3395 if (features & NETIF_F_GSO_PARTIAL) {
3396 netdev_features_t partial_features = NETIF_F_GSO_ROBUST;
3397 struct net_device *dev = skb->dev;
3398
3399 partial_features |= dev->features & dev->gso_partial_features;
3400 if (!skb_gso_ok(skb, features | partial_features))
3401 features &= ~NETIF_F_GSO_PARTIAL;
3402 }
3403
3404 BUILD_BUG_ON(SKB_GSO_CB_OFFSET +
3405 sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb));
3406
3407 SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
3408 SKB_GSO_CB(skb)->encap_level = 0;
3409
3410 skb_reset_mac_header(skb);
3411 skb_reset_mac_len(skb);
3412
3413 segs = skb_mac_gso_segment(skb, features);
3414
3415 if (segs != skb && unlikely(skb_needs_check(skb, tx_path) && !IS_ERR(segs)))
3416 skb_warn_bad_offload(skb);
3417
3418 return segs;
3419}
3420EXPORT_SYMBOL(__skb_gso_segment);
3421
3422/* Take action when hardware reception checksum errors are detected. */
3423#ifdef CONFIG_BUG
3424static void do_netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb)
3425{
3426 netdev_err(dev, "hw csum failure\n");
3427 skb_dump(KERN_ERR, skb, true);
3428 dump_stack();
3429}
3430
3431void netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb)
3432{
3433 DO_ONCE_LITE(do_netdev_rx_csum_fault, dev, skb);
3434}
3435EXPORT_SYMBOL(netdev_rx_csum_fault);
3436#endif
3437
3438/* XXX: check that highmem exists at all on the given machine. */
3439static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
3440{
3441#ifdef CONFIG_HIGHMEM
3442 int i;
3443
3444 if (!(dev->features & NETIF_F_HIGHDMA)) {
3445 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3446 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3447
3448 if (PageHighMem(skb_frag_page(frag)))
3449 return 1;
3450 }
3451 }
3452#endif
3453 return 0;
3454}
3455
3456/* If MPLS offload request, verify we are testing hardware MPLS features
3457 * instead of standard features for the netdev.
3458 */
3459#if IS_ENABLED(CONFIG_NET_MPLS_GSO)
3460static netdev_features_t net_mpls_features(struct sk_buff *skb,
3461 netdev_features_t features,
3462 __be16 type)
3463{
3464 if (eth_p_mpls(type))
3465 features &= skb->dev->mpls_features;
3466
3467 return features;
3468}
3469#else
3470static netdev_features_t net_mpls_features(struct sk_buff *skb,
3471 netdev_features_t features,
3472 __be16 type)
3473{
3474 return features;
3475}
3476#endif
3477
3478static netdev_features_t harmonize_features(struct sk_buff *skb,
3479 netdev_features_t features)
3480{
3481 __be16 type;
3482
3483 type = skb_network_protocol(skb, NULL);
3484 features = net_mpls_features(skb, features, type);
3485
3486 if (skb->ip_summed != CHECKSUM_NONE &&
3487 !can_checksum_protocol(features, type)) {
3488 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
3489 }
3490 if (illegal_highdma(skb->dev, skb))
3491 features &= ~NETIF_F_SG;
3492
3493 return features;
3494}
3495
3496netdev_features_t passthru_features_check(struct sk_buff *skb,
3497 struct net_device *dev,
3498 netdev_features_t features)
3499{
3500 return features;
3501}
3502EXPORT_SYMBOL(passthru_features_check);
3503
3504static netdev_features_t dflt_features_check(struct sk_buff *skb,
3505 struct net_device *dev,
3506 netdev_features_t features)
3507{
3508 return vlan_features_check(skb, features);
3509}
3510
3511static netdev_features_t gso_features_check(const struct sk_buff *skb,
3512 struct net_device *dev,
3513 netdev_features_t features)
3514{
3515 u16 gso_segs = skb_shinfo(skb)->gso_segs;
3516
3517 if (gso_segs > READ_ONCE(dev->gso_max_segs))
3518 return features & ~NETIF_F_GSO_MASK;
3519
3520 if (!skb_shinfo(skb)->gso_type) {
3521 skb_warn_bad_offload(skb);
3522 return features & ~NETIF_F_GSO_MASK;
3523 }
3524
3525 /* Support for GSO partial features requires software
3526 * intervention before we can actually process the packets
3527 * so we need to strip support for any partial features now
3528 * and we can pull them back in after we have partially
3529 * segmented the frame.
3530 */
3531 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL))
3532 features &= ~dev->gso_partial_features;
3533
3534 /* Make sure to clear the IPv4 ID mangling feature if the
3535 * IPv4 header has the potential to be fragmented.
3536 */
3537 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
3538 struct iphdr *iph = skb->encapsulation ?
3539 inner_ip_hdr(skb) : ip_hdr(skb);
3540
3541 if (!(iph->frag_off & htons(IP_DF)))
3542 features &= ~NETIF_F_TSO_MANGLEID;
3543 }
3544
3545 return features;
3546}
3547
3548netdev_features_t netif_skb_features(struct sk_buff *skb)
3549{
3550 struct net_device *dev = skb->dev;
3551 netdev_features_t features = dev->features;
3552
3553 if (skb_is_gso(skb))
3554 features = gso_features_check(skb, dev, features);
3555
3556 /* If encapsulation offload request, verify we are testing
3557 * hardware encapsulation features instead of standard
3558 * features for the netdev
3559 */
3560 if (skb->encapsulation)
3561 features &= dev->hw_enc_features;
3562
3563 if (skb_vlan_tagged(skb))
3564 features = netdev_intersect_features(features,
3565 dev->vlan_features |
3566 NETIF_F_HW_VLAN_CTAG_TX |
3567 NETIF_F_HW_VLAN_STAG_TX);
3568
3569 if (dev->netdev_ops->ndo_features_check)
3570 features &= dev->netdev_ops->ndo_features_check(skb, dev,
3571 features);
3572 else
3573 features &= dflt_features_check(skb, dev, features);
3574
3575 return harmonize_features(skb, features);
3576}
3577EXPORT_SYMBOL(netif_skb_features);
3578
3579static int xmit_one(struct sk_buff *skb, struct net_device *dev,
3580 struct netdev_queue *txq, bool more)
3581{
3582 unsigned int len;
3583 int rc;
3584
3585 if (dev_nit_active(dev))
3586 dev_queue_xmit_nit(skb, dev);
3587
3588 len = skb->len;
3589 trace_net_dev_start_xmit(skb, dev);
3590 rc = netdev_start_xmit(skb, dev, txq, more);
3591 trace_net_dev_xmit(skb, rc, dev, len);
3592
3593 return rc;
3594}
3595
3596struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
3597 struct netdev_queue *txq, int *ret)
3598{
3599 struct sk_buff *skb = first;
3600 int rc = NETDEV_TX_OK;
3601
3602 while (skb) {
3603 struct sk_buff *next = skb->next;
3604
3605 skb_mark_not_on_list(skb);
3606 rc = xmit_one(skb, dev, txq, next != NULL);
3607 if (unlikely(!dev_xmit_complete(rc))) {
3608 skb->next = next;
3609 goto out;
3610 }
3611
3612 skb = next;
3613 if (netif_tx_queue_stopped(txq) && skb) {
3614 rc = NETDEV_TX_BUSY;
3615 break;
3616 }
3617 }
3618
3619out:
3620 *ret = rc;
3621 return skb;
3622}
3623
3624static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
3625 netdev_features_t features)
3626{
3627 if (skb_vlan_tag_present(skb) &&
3628 !vlan_hw_offload_capable(features, skb->vlan_proto))
3629 skb = __vlan_hwaccel_push_inside(skb);
3630 return skb;
3631}
3632
3633int skb_csum_hwoffload_help(struct sk_buff *skb,
3634 const netdev_features_t features)
3635{
3636 if (unlikely(skb_csum_is_sctp(skb)))
3637 return !!(features & NETIF_F_SCTP_CRC) ? 0 :
3638 skb_crc32c_csum_help(skb);
3639
3640 if (features & NETIF_F_HW_CSUM)
3641 return 0;
3642
3643 if (features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) {
3644 switch (skb->csum_offset) {
3645 case offsetof(struct tcphdr, check):
3646 case offsetof(struct udphdr, check):
3647 return 0;
3648 }
3649 }
3650
3651 return skb_checksum_help(skb);
3652}
3653EXPORT_SYMBOL(skb_csum_hwoffload_help);
3654
3655static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev, bool *again)
3656{
3657 netdev_features_t features;
3658
3659 features = netif_skb_features(skb);
3660 skb = validate_xmit_vlan(skb, features);
3661 if (unlikely(!skb))
3662 goto out_null;
3663
3664 skb = sk_validate_xmit_skb(skb, dev);
3665 if (unlikely(!skb))
3666 goto out_null;
3667
3668 if (netif_needs_gso(skb, features)) {
3669 struct sk_buff *segs;
3670
3671 segs = skb_gso_segment(skb, features);
3672 if (IS_ERR(segs)) {
3673 goto out_kfree_skb;
3674 } else if (segs) {
3675 consume_skb(skb);
3676 skb = segs;
3677 }
3678 } else {
3679 if (skb_needs_linearize(skb, features) &&
3680 __skb_linearize(skb))
3681 goto out_kfree_skb;
3682
3683 /* If packet is not checksummed and device does not
3684 * support checksumming for this protocol, complete
3685 * checksumming here.
3686 */
3687 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3688 if (skb->encapsulation)
3689 skb_set_inner_transport_header(skb,
3690 skb_checksum_start_offset(skb));
3691 else
3692 skb_set_transport_header(skb,
3693 skb_checksum_start_offset(skb));
3694 if (skb_csum_hwoffload_help(skb, features))
3695 goto out_kfree_skb;
3696 }
3697 }
3698
3699 skb = validate_xmit_xfrm(skb, features, again);
3700
3701 return skb;
3702
3703out_kfree_skb:
3704 kfree_skb(skb);
3705out_null:
3706 dev_core_stats_tx_dropped_inc(dev);
3707 return NULL;
3708}
3709
3710struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again)
3711{
3712 struct sk_buff *next, *head = NULL, *tail;
3713
3714 for (; skb != NULL; skb = next) {
3715 next = skb->next;
3716 skb_mark_not_on_list(skb);
3717
3718 /* in case skb wont be segmented, point to itself */
3719 skb->prev = skb;
3720
3721 skb = validate_xmit_skb(skb, dev, again);
3722 if (!skb)
3723 continue;
3724
3725 if (!head)
3726 head = skb;
3727 else
3728 tail->next = skb;
3729 /* If skb was segmented, skb->prev points to
3730 * the last segment. If not, it still contains skb.
3731 */
3732 tail = skb->prev;
3733 }
3734 return head;
3735}
3736EXPORT_SYMBOL_GPL(validate_xmit_skb_list);
3737
3738static void qdisc_pkt_len_init(struct sk_buff *skb)
3739{
3740 const struct skb_shared_info *shinfo = skb_shinfo(skb);
3741
3742 qdisc_skb_cb(skb)->pkt_len = skb->len;
3743
3744 /* To get more precise estimation of bytes sent on wire,
3745 * we add to pkt_len the headers size of all segments
3746 */
3747 if (shinfo->gso_size && skb_transport_header_was_set(skb)) {
3748 unsigned int hdr_len;
3749 u16 gso_segs = shinfo->gso_segs;
3750
3751 /* mac layer + network layer */
3752 hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
3753
3754 /* + transport layer */
3755 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) {
3756 const struct tcphdr *th;
3757 struct tcphdr _tcphdr;
3758
3759 th = skb_header_pointer(skb, skb_transport_offset(skb),
3760 sizeof(_tcphdr), &_tcphdr);
3761 if (likely(th))
3762 hdr_len += __tcp_hdrlen(th);
3763 } else {
3764 struct udphdr _udphdr;
3765
3766 if (skb_header_pointer(skb, skb_transport_offset(skb),
3767 sizeof(_udphdr), &_udphdr))
3768 hdr_len += sizeof(struct udphdr);
3769 }
3770
3771 if (shinfo->gso_type & SKB_GSO_DODGY)
3772 gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
3773 shinfo->gso_size);
3774
3775 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
3776 }
3777}
3778
3779static int dev_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *q,
3780 struct sk_buff **to_free,
3781 struct netdev_queue *txq)
3782{
3783 int rc;
3784
3785 rc = q->enqueue(skb, q, to_free) & NET_XMIT_MASK;
3786 if (rc == NET_XMIT_SUCCESS)
3787 trace_qdisc_enqueue(q, txq, skb);
3788 return rc;
3789}
3790
3791static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
3792 struct net_device *dev,
3793 struct netdev_queue *txq)
3794{
3795 spinlock_t *root_lock = qdisc_lock(q);
3796 struct sk_buff *to_free = NULL;
3797 bool contended;
3798 int rc;
3799
3800 qdisc_calculate_pkt_len(skb, q);
3801
3802 if (q->flags & TCQ_F_NOLOCK) {
3803 if (q->flags & TCQ_F_CAN_BYPASS && nolock_qdisc_is_empty(q) &&
3804 qdisc_run_begin(q)) {
3805 /* Retest nolock_qdisc_is_empty() within the protection
3806 * of q->seqlock to protect from racing with requeuing.
3807 */
3808 if (unlikely(!nolock_qdisc_is_empty(q))) {
3809 rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
3810 __qdisc_run(q);
3811 qdisc_run_end(q);
3812
3813 goto no_lock_out;
3814 }
3815
3816 qdisc_bstats_cpu_update(q, skb);
3817 if (sch_direct_xmit(skb, q, dev, txq, NULL, true) &&
3818 !nolock_qdisc_is_empty(q))
3819 __qdisc_run(q);
3820
3821 qdisc_run_end(q);
3822 return NET_XMIT_SUCCESS;
3823 }
3824
3825 rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
3826 qdisc_run(q);
3827
3828no_lock_out:
3829 if (unlikely(to_free))
3830 kfree_skb_list_reason(to_free,
3831 SKB_DROP_REASON_QDISC_DROP);
3832 return rc;
3833 }
3834
3835 /*
3836 * Heuristic to force contended enqueues to serialize on a
3837 * separate lock before trying to get qdisc main lock.
3838 * This permits qdisc->running owner to get the lock more
3839 * often and dequeue packets faster.
3840 * On PREEMPT_RT it is possible to preempt the qdisc owner during xmit
3841 * and then other tasks will only enqueue packets. The packets will be
3842 * sent after the qdisc owner is scheduled again. To prevent this
3843 * scenario the task always serialize on the lock.
3844 */
3845 contended = qdisc_is_running(q) || IS_ENABLED(CONFIG_PREEMPT_RT);
3846 if (unlikely(contended))
3847 spin_lock(&q->busylock);
3848
3849 spin_lock(root_lock);
3850 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
3851 __qdisc_drop(skb, &to_free);
3852 rc = NET_XMIT_DROP;
3853 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
3854 qdisc_run_begin(q)) {
3855 /*
3856 * This is a work-conserving queue; there are no old skbs
3857 * waiting to be sent out; and the qdisc is not running -
3858 * xmit the skb directly.
3859 */
3860
3861 qdisc_bstats_update(q, skb);
3862
3863 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
3864 if (unlikely(contended)) {
3865 spin_unlock(&q->busylock);
3866 contended = false;
3867 }
3868 __qdisc_run(q);
3869 }
3870
3871 qdisc_run_end(q);
3872 rc = NET_XMIT_SUCCESS;
3873 } else {
3874 rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
3875 if (qdisc_run_begin(q)) {
3876 if (unlikely(contended)) {
3877 spin_unlock(&q->busylock);
3878 contended = false;
3879 }
3880 __qdisc_run(q);
3881 qdisc_run_end(q);
3882 }
3883 }
3884 spin_unlock(root_lock);
3885 if (unlikely(to_free))
3886 kfree_skb_list_reason(to_free, SKB_DROP_REASON_QDISC_DROP);
3887 if (unlikely(contended))
3888 spin_unlock(&q->busylock);
3889 return rc;
3890}
3891
3892#if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
3893static void skb_update_prio(struct sk_buff *skb)
3894{
3895 const struct netprio_map *map;
3896 const struct sock *sk;
3897 unsigned int prioidx;
3898
3899 if (skb->priority)
3900 return;
3901 map = rcu_dereference_bh(skb->dev->priomap);
3902 if (!map)
3903 return;
3904 sk = skb_to_full_sk(skb);
3905 if (!sk)
3906 return;
3907
3908 prioidx = sock_cgroup_prioidx(&sk->sk_cgrp_data);
3909
3910 if (prioidx < map->priomap_len)
3911 skb->priority = map->priomap[prioidx];
3912}
3913#else
3914#define skb_update_prio(skb)
3915#endif
3916
3917/**
3918 * dev_loopback_xmit - loop back @skb
3919 * @net: network namespace this loopback is happening in
3920 * @sk: sk needed to be a netfilter okfn
3921 * @skb: buffer to transmit
3922 */
3923int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
3924{
3925 skb_reset_mac_header(skb);
3926 __skb_pull(skb, skb_network_offset(skb));
3927 skb->pkt_type = PACKET_LOOPBACK;
3928 if (skb->ip_summed == CHECKSUM_NONE)
3929 skb->ip_summed = CHECKSUM_UNNECESSARY;
3930 DEBUG_NET_WARN_ON_ONCE(!skb_dst(skb));
3931 skb_dst_force(skb);
3932 netif_rx(skb);
3933 return 0;
3934}
3935EXPORT_SYMBOL(dev_loopback_xmit);
3936
3937#ifdef CONFIG_NET_EGRESS
3938static struct sk_buff *
3939sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
3940{
3941#ifdef CONFIG_NET_CLS_ACT
3942 struct mini_Qdisc *miniq = rcu_dereference_bh(dev->miniq_egress);
3943 struct tcf_result cl_res;
3944
3945 if (!miniq)
3946 return skb;
3947
3948 /* qdisc_skb_cb(skb)->pkt_len was already set by the caller. */
3949 tc_skb_cb(skb)->mru = 0;
3950 tc_skb_cb(skb)->post_ct = false;
3951 mini_qdisc_bstats_cpu_update(miniq, skb);
3952
3953 switch (tcf_classify(skb, miniq->block, miniq->filter_list, &cl_res, false)) {
3954 case TC_ACT_OK:
3955 case TC_ACT_RECLASSIFY:
3956 skb->tc_index = TC_H_MIN(cl_res.classid);
3957 break;
3958 case TC_ACT_SHOT:
3959 mini_qdisc_qstats_cpu_drop(miniq);
3960 *ret = NET_XMIT_DROP;
3961 kfree_skb_reason(skb, SKB_DROP_REASON_TC_EGRESS);
3962 return NULL;
3963 case TC_ACT_STOLEN:
3964 case TC_ACT_QUEUED:
3965 case TC_ACT_TRAP:
3966 *ret = NET_XMIT_SUCCESS;
3967 consume_skb(skb);
3968 return NULL;
3969 case TC_ACT_REDIRECT:
3970 /* No need to push/pop skb's mac_header here on egress! */
3971 skb_do_redirect(skb);
3972 *ret = NET_XMIT_SUCCESS;
3973 return NULL;
3974 default:
3975 break;
3976 }
3977#endif /* CONFIG_NET_CLS_ACT */
3978
3979 return skb;
3980}
3981
3982static struct netdev_queue *
3983netdev_tx_queue_mapping(struct net_device *dev, struct sk_buff *skb)
3984{
3985 int qm = skb_get_queue_mapping(skb);
3986
3987 return netdev_get_tx_queue(dev, netdev_cap_txqueue(dev, qm));
3988}
3989
3990static bool netdev_xmit_txqueue_skipped(void)
3991{
3992 return __this_cpu_read(softnet_data.xmit.skip_txqueue);
3993}
3994
3995void netdev_xmit_skip_txqueue(bool skip)
3996{
3997 __this_cpu_write(softnet_data.xmit.skip_txqueue, skip);
3998}
3999EXPORT_SYMBOL_GPL(netdev_xmit_skip_txqueue);
4000#endif /* CONFIG_NET_EGRESS */
4001
4002#ifdef CONFIG_XPS
4003static int __get_xps_queue_idx(struct net_device *dev, struct sk_buff *skb,
4004 struct xps_dev_maps *dev_maps, unsigned int tci)
4005{
4006 int tc = netdev_get_prio_tc_map(dev, skb->priority);
4007 struct xps_map *map;
4008 int queue_index = -1;
4009
4010 if (tc >= dev_maps->num_tc || tci >= dev_maps->nr_ids)
4011 return queue_index;
4012
4013 tci *= dev_maps->num_tc;
4014 tci += tc;
4015
4016 map = rcu_dereference(dev_maps->attr_map[tci]);
4017 if (map) {
4018 if (map->len == 1)
4019 queue_index = map->queues[0];
4020 else
4021 queue_index = map->queues[reciprocal_scale(
4022 skb_get_hash(skb), map->len)];
4023 if (unlikely(queue_index >= dev->real_num_tx_queues))
4024 queue_index = -1;
4025 }
4026 return queue_index;
4027}
4028#endif
4029
4030static int get_xps_queue(struct net_device *dev, struct net_device *sb_dev,
4031 struct sk_buff *skb)
4032{
4033#ifdef CONFIG_XPS
4034 struct xps_dev_maps *dev_maps;
4035 struct sock *sk = skb->sk;
4036 int queue_index = -1;
4037
4038 if (!static_key_false(&xps_needed))
4039 return -1;
4040
4041 rcu_read_lock();
4042 if (!static_key_false(&xps_rxqs_needed))
4043 goto get_cpus_map;
4044
4045 dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_RXQS]);
4046 if (dev_maps) {
4047 int tci = sk_rx_queue_get(sk);
4048
4049 if (tci >= 0)
4050 queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
4051 tci);
4052 }
4053
4054get_cpus_map:
4055 if (queue_index < 0) {
4056 dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_CPUS]);
4057 if (dev_maps) {
4058 unsigned int tci = skb->sender_cpu - 1;
4059
4060 queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
4061 tci);
4062 }
4063 }
4064 rcu_read_unlock();
4065
4066 return queue_index;
4067#else
4068 return -1;
4069#endif
4070}
4071
4072u16 dev_pick_tx_zero(struct net_device *dev, struct sk_buff *skb,
4073 struct net_device *sb_dev)
4074{
4075 return 0;
4076}
4077EXPORT_SYMBOL(dev_pick_tx_zero);
4078
4079u16 dev_pick_tx_cpu_id(struct net_device *dev, struct sk_buff *skb,
4080 struct net_device *sb_dev)
4081{
4082 return (u16)raw_smp_processor_id() % dev->real_num_tx_queues;
4083}
4084EXPORT_SYMBOL(dev_pick_tx_cpu_id);
4085
4086u16 netdev_pick_tx(struct net_device *dev, struct sk_buff *skb,
4087 struct net_device *sb_dev)
4088{
4089 struct sock *sk = skb->sk;
4090 int queue_index = sk_tx_queue_get(sk);
4091
4092 sb_dev = sb_dev ? : dev;
4093
4094 if (queue_index < 0 || skb->ooo_okay ||
4095 queue_index >= dev->real_num_tx_queues) {
4096 int new_index = get_xps_queue(dev, sb_dev, skb);
4097
4098 if (new_index < 0)
4099 new_index = skb_tx_hash(dev, sb_dev, skb);
4100
4101 if (queue_index != new_index && sk &&
4102 sk_fullsock(sk) &&
4103 rcu_access_pointer(sk->sk_dst_cache))
4104 sk_tx_queue_set(sk, new_index);
4105
4106 queue_index = new_index;
4107 }
4108
4109 return queue_index;
4110}
4111EXPORT_SYMBOL(netdev_pick_tx);
4112
4113struct netdev_queue *netdev_core_pick_tx(struct net_device *dev,
4114 struct sk_buff *skb,
4115 struct net_device *sb_dev)
4116{
4117 int queue_index = 0;
4118
4119#ifdef CONFIG_XPS
4120 u32 sender_cpu = skb->sender_cpu - 1;
4121
4122 if (sender_cpu >= (u32)NR_CPUS)
4123 skb->sender_cpu = raw_smp_processor_id() + 1;
4124#endif
4125
4126 if (dev->real_num_tx_queues != 1) {
4127 const struct net_device_ops *ops = dev->netdev_ops;
4128
4129 if (ops->ndo_select_queue)
4130 queue_index = ops->ndo_select_queue(dev, skb, sb_dev);
4131 else
4132 queue_index = netdev_pick_tx(dev, skb, sb_dev);
4133
4134 queue_index = netdev_cap_txqueue(dev, queue_index);
4135 }
4136
4137 skb_set_queue_mapping(skb, queue_index);
4138 return netdev_get_tx_queue(dev, queue_index);
4139}
4140
4141/**
4142 * __dev_queue_xmit() - transmit a buffer
4143 * @skb: buffer to transmit
4144 * @sb_dev: suboordinate device used for L2 forwarding offload
4145 *
4146 * Queue a buffer for transmission to a network device. The caller must
4147 * have set the device and priority and built the buffer before calling
4148 * this function. The function can be called from an interrupt.
4149 *
4150 * When calling this method, interrupts MUST be enabled. This is because
4151 * the BH enable code must have IRQs enabled so that it will not deadlock.
4152 *
4153 * Regardless of the return value, the skb is consumed, so it is currently
4154 * difficult to retry a send to this method. (You can bump the ref count
4155 * before sending to hold a reference for retry if you are careful.)
4156 *
4157 * Return:
4158 * * 0 - buffer successfully transmitted
4159 * * positive qdisc return code - NET_XMIT_DROP etc.
4160 * * negative errno - other errors
4161 */
4162int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev)
4163{
4164 struct net_device *dev = skb->dev;
4165 struct netdev_queue *txq = NULL;
4166 struct Qdisc *q;
4167 int rc = -ENOMEM;
4168 bool again = false;
4169
4170 skb_reset_mac_header(skb);
4171 skb_assert_len(skb);
4172
4173 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
4174 __skb_tstamp_tx(skb, NULL, NULL, skb->sk, SCM_TSTAMP_SCHED);
4175
4176 /* Disable soft irqs for various locks below. Also
4177 * stops preemption for RCU.
4178 */
4179 rcu_read_lock_bh();
4180
4181 skb_update_prio(skb);
4182
4183 qdisc_pkt_len_init(skb);
4184#ifdef CONFIG_NET_CLS_ACT
4185 skb->tc_at_ingress = 0;
4186#endif
4187#ifdef CONFIG_NET_EGRESS
4188 if (static_branch_unlikely(&egress_needed_key)) {
4189 if (nf_hook_egress_active()) {
4190 skb = nf_hook_egress(skb, &rc, dev);
4191 if (!skb)
4192 goto out;
4193 }
4194
4195 netdev_xmit_skip_txqueue(false);
4196
4197 nf_skip_egress(skb, true);
4198 skb = sch_handle_egress(skb, &rc, dev);
4199 if (!skb)
4200 goto out;
4201 nf_skip_egress(skb, false);
4202
4203 if (netdev_xmit_txqueue_skipped())
4204 txq = netdev_tx_queue_mapping(dev, skb);
4205 }
4206#endif
4207 /* If device/qdisc don't need skb->dst, release it right now while
4208 * its hot in this cpu cache.
4209 */
4210 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
4211 skb_dst_drop(skb);
4212 else
4213 skb_dst_force(skb);
4214
4215 if (!txq)
4216 txq = netdev_core_pick_tx(dev, skb, sb_dev);
4217
4218 q = rcu_dereference_bh(txq->qdisc);
4219
4220 trace_net_dev_queue(skb);
4221 if (q->enqueue) {
4222 rc = __dev_xmit_skb(skb, q, dev, txq);
4223 goto out;
4224 }
4225
4226 /* The device has no queue. Common case for software devices:
4227 * loopback, all the sorts of tunnels...
4228
4229 * Really, it is unlikely that netif_tx_lock protection is necessary
4230 * here. (f.e. loopback and IP tunnels are clean ignoring statistics
4231 * counters.)
4232 * However, it is possible, that they rely on protection
4233 * made by us here.
4234
4235 * Check this and shot the lock. It is not prone from deadlocks.
4236 *Either shot noqueue qdisc, it is even simpler 8)
4237 */
4238 if (dev->flags & IFF_UP) {
4239 int cpu = smp_processor_id(); /* ok because BHs are off */
4240
4241 /* Other cpus might concurrently change txq->xmit_lock_owner
4242 * to -1 or to their cpu id, but not to our id.
4243 */
4244 if (READ_ONCE(txq->xmit_lock_owner) != cpu) {
4245 if (dev_xmit_recursion())
4246 goto recursion_alert;
4247
4248 skb = validate_xmit_skb(skb, dev, &again);
4249 if (!skb)
4250 goto out;
4251
4252 HARD_TX_LOCK(dev, txq, cpu);
4253
4254 if (!netif_xmit_stopped(txq)) {
4255 dev_xmit_recursion_inc();
4256 skb = dev_hard_start_xmit(skb, dev, txq, &rc);
4257 dev_xmit_recursion_dec();
4258 if (dev_xmit_complete(rc)) {
4259 HARD_TX_UNLOCK(dev, txq);
4260 goto out;
4261 }
4262 }
4263 HARD_TX_UNLOCK(dev, txq);
4264 net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
4265 dev->name);
4266 } else {
4267 /* Recursion is detected! It is possible,
4268 * unfortunately
4269 */
4270recursion_alert:
4271 net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
4272 dev->name);
4273 }
4274 }
4275
4276 rc = -ENETDOWN;
4277 rcu_read_unlock_bh();
4278
4279 dev_core_stats_tx_dropped_inc(dev);
4280 kfree_skb_list(skb);
4281 return rc;
4282out:
4283 rcu_read_unlock_bh();
4284 return rc;
4285}
4286EXPORT_SYMBOL(__dev_queue_xmit);
4287
4288int __dev_direct_xmit(struct sk_buff *skb, u16 queue_id)
4289{
4290 struct net_device *dev = skb->dev;
4291 struct sk_buff *orig_skb = skb;
4292 struct netdev_queue *txq;
4293 int ret = NETDEV_TX_BUSY;
4294 bool again = false;
4295
4296 if (unlikely(!netif_running(dev) ||
4297 !netif_carrier_ok(dev)))
4298 goto drop;
4299
4300 skb = validate_xmit_skb_list(skb, dev, &again);
4301 if (skb != orig_skb)
4302 goto drop;
4303
4304 skb_set_queue_mapping(skb, queue_id);
4305 txq = skb_get_tx_queue(dev, skb);
4306
4307 local_bh_disable();
4308
4309 dev_xmit_recursion_inc();
4310 HARD_TX_LOCK(dev, txq, smp_processor_id());
4311 if (!netif_xmit_frozen_or_drv_stopped(txq))
4312 ret = netdev_start_xmit(skb, dev, txq, false);
4313 HARD_TX_UNLOCK(dev, txq);
4314 dev_xmit_recursion_dec();
4315
4316 local_bh_enable();
4317 return ret;
4318drop:
4319 dev_core_stats_tx_dropped_inc(dev);
4320 kfree_skb_list(skb);
4321 return NET_XMIT_DROP;
4322}
4323EXPORT_SYMBOL(__dev_direct_xmit);
4324
4325/*************************************************************************
4326 * Receiver routines
4327 *************************************************************************/
4328
4329int netdev_max_backlog __read_mostly = 1000;
4330EXPORT_SYMBOL(netdev_max_backlog);
4331
4332int netdev_tstamp_prequeue __read_mostly = 1;
4333unsigned int sysctl_skb_defer_max __read_mostly = 64;
4334int netdev_budget __read_mostly = 300;
4335/* Must be at least 2 jiffes to guarantee 1 jiffy timeout */
4336unsigned int __read_mostly netdev_budget_usecs = 2 * USEC_PER_SEC / HZ;
4337int weight_p __read_mostly = 64; /* old backlog weight */
4338int dev_weight_rx_bias __read_mostly = 1; /* bias for backlog weight */
4339int dev_weight_tx_bias __read_mostly = 1; /* bias for output_queue quota */
4340int dev_rx_weight __read_mostly = 64;
4341int dev_tx_weight __read_mostly = 64;
4342
4343/* Called with irq disabled */
4344static inline void ____napi_schedule(struct softnet_data *sd,
4345 struct napi_struct *napi)
4346{
4347 struct task_struct *thread;
4348
4349 lockdep_assert_irqs_disabled();
4350
4351 if (test_bit(NAPI_STATE_THREADED, &napi->state)) {
4352 /* Paired with smp_mb__before_atomic() in
4353 * napi_enable()/dev_set_threaded().
4354 * Use READ_ONCE() to guarantee a complete
4355 * read on napi->thread. Only call
4356 * wake_up_process() when it's not NULL.
4357 */
4358 thread = READ_ONCE(napi->thread);
4359 if (thread) {
4360 /* Avoid doing set_bit() if the thread is in
4361 * INTERRUPTIBLE state, cause napi_thread_wait()
4362 * makes sure to proceed with napi polling
4363 * if the thread is explicitly woken from here.
4364 */
4365 if (READ_ONCE(thread->__state) != TASK_INTERRUPTIBLE)
4366 set_bit(NAPI_STATE_SCHED_THREADED, &napi->state);
4367 wake_up_process(thread);
4368 return;
4369 }
4370 }
4371
4372 list_add_tail(&napi->poll_list, &sd->poll_list);
4373 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4374}
4375
4376#ifdef CONFIG_RPS
4377
4378/* One global table that all flow-based protocols share. */
4379struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
4380EXPORT_SYMBOL(rps_sock_flow_table);
4381u32 rps_cpu_mask __read_mostly;
4382EXPORT_SYMBOL(rps_cpu_mask);
4383
4384struct static_key_false rps_needed __read_mostly;
4385EXPORT_SYMBOL(rps_needed);
4386struct static_key_false rfs_needed __read_mostly;
4387EXPORT_SYMBOL(rfs_needed);
4388
4389static struct rps_dev_flow *
4390set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
4391 struct rps_dev_flow *rflow, u16 next_cpu)
4392{
4393 if (next_cpu < nr_cpu_ids) {
4394#ifdef CONFIG_RFS_ACCEL
4395 struct netdev_rx_queue *rxqueue;
4396 struct rps_dev_flow_table *flow_table;
4397 struct rps_dev_flow *old_rflow;
4398 u32 flow_id;
4399 u16 rxq_index;
4400 int rc;
4401
4402 /* Should we steer this flow to a different hardware queue? */
4403 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
4404 !(dev->features & NETIF_F_NTUPLE))
4405 goto out;
4406 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
4407 if (rxq_index == skb_get_rx_queue(skb))
4408 goto out;
4409
4410 rxqueue = dev->_rx + rxq_index;
4411 flow_table = rcu_dereference(rxqueue->rps_flow_table);
4412 if (!flow_table)
4413 goto out;
4414 flow_id = skb_get_hash(skb) & flow_table->mask;
4415 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
4416 rxq_index, flow_id);
4417 if (rc < 0)
4418 goto out;
4419 old_rflow = rflow;
4420 rflow = &flow_table->flows[flow_id];
4421 rflow->filter = rc;
4422 if (old_rflow->filter == rflow->filter)
4423 old_rflow->filter = RPS_NO_FILTER;
4424 out:
4425#endif
4426 rflow->last_qtail =
4427 per_cpu(softnet_data, next_cpu).input_queue_head;
4428 }
4429
4430 rflow->cpu = next_cpu;
4431 return rflow;
4432}
4433
4434/*
4435 * get_rps_cpu is called from netif_receive_skb and returns the target
4436 * CPU from the RPS map of the receiving queue for a given skb.
4437 * rcu_read_lock must be held on entry.
4438 */
4439static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
4440 struct rps_dev_flow **rflowp)
4441{
4442 const struct rps_sock_flow_table *sock_flow_table;
4443 struct netdev_rx_queue *rxqueue = dev->_rx;
4444 struct rps_dev_flow_table *flow_table;
4445 struct rps_map *map;
4446 int cpu = -1;
4447 u32 tcpu;
4448 u32 hash;
4449
4450 if (skb_rx_queue_recorded(skb)) {
4451 u16 index = skb_get_rx_queue(skb);
4452
4453 if (unlikely(index >= dev->real_num_rx_queues)) {
4454 WARN_ONCE(dev->real_num_rx_queues > 1,
4455 "%s received packet on queue %u, but number "
4456 "of RX queues is %u\n",
4457 dev->name, index, dev->real_num_rx_queues);
4458 goto done;
4459 }
4460 rxqueue += index;
4461 }
4462
4463 /* Avoid computing hash if RFS/RPS is not active for this rxqueue */
4464
4465 flow_table = rcu_dereference(rxqueue->rps_flow_table);
4466 map = rcu_dereference(rxqueue->rps_map);
4467 if (!flow_table && !map)
4468 goto done;
4469
4470 skb_reset_network_header(skb);
4471 hash = skb_get_hash(skb);
4472 if (!hash)
4473 goto done;
4474
4475 sock_flow_table = rcu_dereference(rps_sock_flow_table);
4476 if (flow_table && sock_flow_table) {
4477 struct rps_dev_flow *rflow;
4478 u32 next_cpu;
4479 u32 ident;
4480
4481 /* First check into global flow table if there is a match */
4482 ident = sock_flow_table->ents[hash & sock_flow_table->mask];
4483 if ((ident ^ hash) & ~rps_cpu_mask)
4484 goto try_rps;
4485
4486 next_cpu = ident & rps_cpu_mask;
4487
4488 /* OK, now we know there is a match,
4489 * we can look at the local (per receive queue) flow table
4490 */
4491 rflow = &flow_table->flows[hash & flow_table->mask];
4492 tcpu = rflow->cpu;
4493
4494 /*
4495 * If the desired CPU (where last recvmsg was done) is
4496 * different from current CPU (one in the rx-queue flow
4497 * table entry), switch if one of the following holds:
4498 * - Current CPU is unset (>= nr_cpu_ids).
4499 * - Current CPU is offline.
4500 * - The current CPU's queue tail has advanced beyond the
4501 * last packet that was enqueued using this table entry.
4502 * This guarantees that all previous packets for the flow
4503 * have been dequeued, thus preserving in order delivery.
4504 */
4505 if (unlikely(tcpu != next_cpu) &&
4506 (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
4507 ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
4508 rflow->last_qtail)) >= 0)) {
4509 tcpu = next_cpu;
4510 rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
4511 }
4512
4513 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
4514 *rflowp = rflow;
4515 cpu = tcpu;
4516 goto done;
4517 }
4518 }
4519
4520try_rps:
4521
4522 if (map) {
4523 tcpu = map->cpus[reciprocal_scale(hash, map->len)];
4524 if (cpu_online(tcpu)) {
4525 cpu = tcpu;
4526 goto done;
4527 }
4528 }
4529
4530done:
4531 return cpu;
4532}
4533
4534#ifdef CONFIG_RFS_ACCEL
4535
4536/**
4537 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
4538 * @dev: Device on which the filter was set
4539 * @rxq_index: RX queue index
4540 * @flow_id: Flow ID passed to ndo_rx_flow_steer()
4541 * @filter_id: Filter ID returned by ndo_rx_flow_steer()
4542 *
4543 * Drivers that implement ndo_rx_flow_steer() should periodically call
4544 * this function for each installed filter and remove the filters for
4545 * which it returns %true.
4546 */
4547bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
4548 u32 flow_id, u16 filter_id)
4549{
4550 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
4551 struct rps_dev_flow_table *flow_table;
4552 struct rps_dev_flow *rflow;
4553 bool expire = true;
4554 unsigned int cpu;
4555
4556 rcu_read_lock();
4557 flow_table = rcu_dereference(rxqueue->rps_flow_table);
4558 if (flow_table && flow_id <= flow_table->mask) {
4559 rflow = &flow_table->flows[flow_id];
4560 cpu = READ_ONCE(rflow->cpu);
4561 if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
4562 ((int)(per_cpu(softnet_data, cpu).input_queue_head -
4563 rflow->last_qtail) <
4564 (int)(10 * flow_table->mask)))
4565 expire = false;
4566 }
4567 rcu_read_unlock();
4568 return expire;
4569}
4570EXPORT_SYMBOL(rps_may_expire_flow);
4571
4572#endif /* CONFIG_RFS_ACCEL */
4573
4574/* Called from hardirq (IPI) context */
4575static void rps_trigger_softirq(void *data)
4576{
4577 struct softnet_data *sd = data;
4578
4579 ____napi_schedule(sd, &sd->backlog);
4580 sd->received_rps++;
4581}
4582
4583#endif /* CONFIG_RPS */
4584
4585/* Called from hardirq (IPI) context */
4586static void trigger_rx_softirq(void *data)
4587{
4588 struct softnet_data *sd = data;
4589
4590 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4591 smp_store_release(&sd->defer_ipi_scheduled, 0);
4592}
4593
4594/*
4595 * Check if this softnet_data structure is another cpu one
4596 * If yes, queue it to our IPI list and return 1
4597 * If no, return 0
4598 */
4599static int napi_schedule_rps(struct softnet_data *sd)
4600{
4601 struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
4602
4603#ifdef CONFIG_RPS
4604 if (sd != mysd) {
4605 sd->rps_ipi_next = mysd->rps_ipi_list;
4606 mysd->rps_ipi_list = sd;
4607
4608 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4609 return 1;
4610 }
4611#endif /* CONFIG_RPS */
4612 __napi_schedule_irqoff(&mysd->backlog);
4613 return 0;
4614}
4615
4616#ifdef CONFIG_NET_FLOW_LIMIT
4617int netdev_flow_limit_table_len __read_mostly = (1 << 12);
4618#endif
4619
4620static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
4621{
4622#ifdef CONFIG_NET_FLOW_LIMIT
4623 struct sd_flow_limit *fl;
4624 struct softnet_data *sd;
4625 unsigned int old_flow, new_flow;
4626
4627 if (qlen < (netdev_max_backlog >> 1))
4628 return false;
4629
4630 sd = this_cpu_ptr(&softnet_data);
4631
4632 rcu_read_lock();
4633 fl = rcu_dereference(sd->flow_limit);
4634 if (fl) {
4635 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
4636 old_flow = fl->history[fl->history_head];
4637 fl->history[fl->history_head] = new_flow;
4638
4639 fl->history_head++;
4640 fl->history_head &= FLOW_LIMIT_HISTORY - 1;
4641
4642 if (likely(fl->buckets[old_flow]))
4643 fl->buckets[old_flow]--;
4644
4645 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
4646 fl->count++;
4647 rcu_read_unlock();
4648 return true;
4649 }
4650 }
4651 rcu_read_unlock();
4652#endif
4653 return false;
4654}
4655
4656/*
4657 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
4658 * queue (may be a remote CPU queue).
4659 */
4660static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
4661 unsigned int *qtail)
4662{
4663 enum skb_drop_reason reason;
4664 struct softnet_data *sd;
4665 unsigned long flags;
4666 unsigned int qlen;
4667
4668 reason = SKB_DROP_REASON_NOT_SPECIFIED;
4669 sd = &per_cpu(softnet_data, cpu);
4670
4671 rps_lock_irqsave(sd, &flags);
4672 if (!netif_running(skb->dev))
4673 goto drop;
4674 qlen = skb_queue_len(&sd->input_pkt_queue);
4675 if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) {
4676 if (qlen) {
4677enqueue:
4678 __skb_queue_tail(&sd->input_pkt_queue, skb);
4679 input_queue_tail_incr_save(sd, qtail);
4680 rps_unlock_irq_restore(sd, &flags);
4681 return NET_RX_SUCCESS;
4682 }
4683
4684 /* Schedule NAPI for backlog device
4685 * We can use non atomic operation since we own the queue lock
4686 */
4687 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state))
4688 napi_schedule_rps(sd);
4689 goto enqueue;
4690 }
4691 reason = SKB_DROP_REASON_CPU_BACKLOG;
4692
4693drop:
4694 sd->dropped++;
4695 rps_unlock_irq_restore(sd, &flags);
4696
4697 dev_core_stats_rx_dropped_inc(skb->dev);
4698 kfree_skb_reason(skb, reason);
4699 return NET_RX_DROP;
4700}
4701
4702static struct netdev_rx_queue *netif_get_rxqueue(struct sk_buff *skb)
4703{
4704 struct net_device *dev = skb->dev;
4705 struct netdev_rx_queue *rxqueue;
4706
4707 rxqueue = dev->_rx;
4708
4709 if (skb_rx_queue_recorded(skb)) {
4710 u16 index = skb_get_rx_queue(skb);
4711
4712 if (unlikely(index >= dev->real_num_rx_queues)) {
4713 WARN_ONCE(dev->real_num_rx_queues > 1,
4714 "%s received packet on queue %u, but number "
4715 "of RX queues is %u\n",
4716 dev->name, index, dev->real_num_rx_queues);
4717
4718 return rxqueue; /* Return first rxqueue */
4719 }
4720 rxqueue += index;
4721 }
4722 return rxqueue;
4723}
4724
4725u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp,
4726 struct bpf_prog *xdp_prog)
4727{
4728 void *orig_data, *orig_data_end, *hard_start;
4729 struct netdev_rx_queue *rxqueue;
4730 bool orig_bcast, orig_host;
4731 u32 mac_len, frame_sz;
4732 __be16 orig_eth_type;
4733 struct ethhdr *eth;
4734 u32 metalen, act;
4735 int off;
4736
4737 /* The XDP program wants to see the packet starting at the MAC
4738 * header.
4739 */
4740 mac_len = skb->data - skb_mac_header(skb);
4741 hard_start = skb->data - skb_headroom(skb);
4742
4743 /* SKB "head" area always have tailroom for skb_shared_info */
4744 frame_sz = (void *)skb_end_pointer(skb) - hard_start;
4745 frame_sz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
4746
4747 rxqueue = netif_get_rxqueue(skb);
4748 xdp_init_buff(xdp, frame_sz, &rxqueue->xdp_rxq);
4749 xdp_prepare_buff(xdp, hard_start, skb_headroom(skb) - mac_len,
4750 skb_headlen(skb) + mac_len, true);
4751
4752 orig_data_end = xdp->data_end;
4753 orig_data = xdp->data;
4754 eth = (struct ethhdr *)xdp->data;
4755 orig_host = ether_addr_equal_64bits(eth->h_dest, skb->dev->dev_addr);
4756 orig_bcast = is_multicast_ether_addr_64bits(eth->h_dest);
4757 orig_eth_type = eth->h_proto;
4758
4759 act = bpf_prog_run_xdp(xdp_prog, xdp);
4760
4761 /* check if bpf_xdp_adjust_head was used */
4762 off = xdp->data - orig_data;
4763 if (off) {
4764 if (off > 0)
4765 __skb_pull(skb, off);
4766 else if (off < 0)
4767 __skb_push(skb, -off);
4768
4769 skb->mac_header += off;
4770 skb_reset_network_header(skb);
4771 }
4772
4773 /* check if bpf_xdp_adjust_tail was used */
4774 off = xdp->data_end - orig_data_end;
4775 if (off != 0) {
4776 skb_set_tail_pointer(skb, xdp->data_end - xdp->data);
4777 skb->len += off; /* positive on grow, negative on shrink */
4778 }
4779
4780 /* check if XDP changed eth hdr such SKB needs update */
4781 eth = (struct ethhdr *)xdp->data;
4782 if ((orig_eth_type != eth->h_proto) ||
4783 (orig_host != ether_addr_equal_64bits(eth->h_dest,
4784 skb->dev->dev_addr)) ||
4785 (orig_bcast != is_multicast_ether_addr_64bits(eth->h_dest))) {
4786 __skb_push(skb, ETH_HLEN);
4787 skb->pkt_type = PACKET_HOST;
4788 skb->protocol = eth_type_trans(skb, skb->dev);
4789 }
4790
4791 /* Redirect/Tx gives L2 packet, code that will reuse skb must __skb_pull
4792 * before calling us again on redirect path. We do not call do_redirect
4793 * as we leave that up to the caller.
4794 *
4795 * Caller is responsible for managing lifetime of skb (i.e. calling
4796 * kfree_skb in response to actions it cannot handle/XDP_DROP).
4797 */
4798 switch (act) {
4799 case XDP_REDIRECT:
4800 case XDP_TX:
4801 __skb_push(skb, mac_len);
4802 break;
4803 case XDP_PASS:
4804 metalen = xdp->data - xdp->data_meta;
4805 if (metalen)
4806 skb_metadata_set(skb, metalen);
4807 break;
4808 }
4809
4810 return act;
4811}
4812
4813static u32 netif_receive_generic_xdp(struct sk_buff *skb,
4814 struct xdp_buff *xdp,
4815 struct bpf_prog *xdp_prog)
4816{
4817 u32 act = XDP_DROP;
4818
4819 /* Reinjected packets coming from act_mirred or similar should
4820 * not get XDP generic processing.
4821 */
4822 if (skb_is_redirected(skb))
4823 return XDP_PASS;
4824
4825 /* XDP packets must be linear and must have sufficient headroom
4826 * of XDP_PACKET_HEADROOM bytes. This is the guarantee that also
4827 * native XDP provides, thus we need to do it here as well.
4828 */
4829 if (skb_cloned(skb) || skb_is_nonlinear(skb) ||
4830 skb_headroom(skb) < XDP_PACKET_HEADROOM) {
4831 int hroom = XDP_PACKET_HEADROOM - skb_headroom(skb);
4832 int troom = skb->tail + skb->data_len - skb->end;
4833
4834 /* In case we have to go down the path and also linearize,
4835 * then lets do the pskb_expand_head() work just once here.
4836 */
4837 if (pskb_expand_head(skb,
4838 hroom > 0 ? ALIGN(hroom, NET_SKB_PAD) : 0,
4839 troom > 0 ? troom + 128 : 0, GFP_ATOMIC))
4840 goto do_drop;
4841 if (skb_linearize(skb))
4842 goto do_drop;
4843 }
4844
4845 act = bpf_prog_run_generic_xdp(skb, xdp, xdp_prog);
4846 switch (act) {
4847 case XDP_REDIRECT:
4848 case XDP_TX:
4849 case XDP_PASS:
4850 break;
4851 default:
4852 bpf_warn_invalid_xdp_action(skb->dev, xdp_prog, act);
4853 fallthrough;
4854 case XDP_ABORTED:
4855 trace_xdp_exception(skb->dev, xdp_prog, act);
4856 fallthrough;
4857 case XDP_DROP:
4858 do_drop:
4859 kfree_skb(skb);
4860 break;
4861 }
4862
4863 return act;
4864}
4865
4866/* When doing generic XDP we have to bypass the qdisc layer and the
4867 * network taps in order to match in-driver-XDP behavior. This also means
4868 * that XDP packets are able to starve other packets going through a qdisc,
4869 * and DDOS attacks will be more effective. In-driver-XDP use dedicated TX
4870 * queues, so they do not have this starvation issue.
4871 */
4872void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog)
4873{
4874 struct net_device *dev = skb->dev;
4875 struct netdev_queue *txq;
4876 bool free_skb = true;
4877 int cpu, rc;
4878
4879 txq = netdev_core_pick_tx(dev, skb, NULL);
4880 cpu = smp_processor_id();
4881 HARD_TX_LOCK(dev, txq, cpu);
4882 if (!netif_xmit_frozen_or_drv_stopped(txq)) {
4883 rc = netdev_start_xmit(skb, dev, txq, 0);
4884 if (dev_xmit_complete(rc))
4885 free_skb = false;
4886 }
4887 HARD_TX_UNLOCK(dev, txq);
4888 if (free_skb) {
4889 trace_xdp_exception(dev, xdp_prog, XDP_TX);
4890 dev_core_stats_tx_dropped_inc(dev);
4891 kfree_skb(skb);
4892 }
4893}
4894
4895static DEFINE_STATIC_KEY_FALSE(generic_xdp_needed_key);
4896
4897int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb)
4898{
4899 if (xdp_prog) {
4900 struct xdp_buff xdp;
4901 u32 act;
4902 int err;
4903
4904 act = netif_receive_generic_xdp(skb, &xdp, xdp_prog);
4905 if (act != XDP_PASS) {
4906 switch (act) {
4907 case XDP_REDIRECT:
4908 err = xdp_do_generic_redirect(skb->dev, skb,
4909 &xdp, xdp_prog);
4910 if (err)
4911 goto out_redir;
4912 break;
4913 case XDP_TX:
4914 generic_xdp_tx(skb, xdp_prog);
4915 break;
4916 }
4917 return XDP_DROP;
4918 }
4919 }
4920 return XDP_PASS;
4921out_redir:
4922 kfree_skb_reason(skb, SKB_DROP_REASON_XDP);
4923 return XDP_DROP;
4924}
4925EXPORT_SYMBOL_GPL(do_xdp_generic);
4926
4927static int netif_rx_internal(struct sk_buff *skb)
4928{
4929 int ret;
4930
4931 net_timestamp_check(netdev_tstamp_prequeue, skb);
4932
4933 trace_netif_rx(skb);
4934
4935#ifdef CONFIG_RPS
4936 if (static_branch_unlikely(&rps_needed)) {
4937 struct rps_dev_flow voidflow, *rflow = &voidflow;
4938 int cpu;
4939
4940 rcu_read_lock();
4941
4942 cpu = get_rps_cpu(skb->dev, skb, &rflow);
4943 if (cpu < 0)
4944 cpu = smp_processor_id();
4945
4946 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
4947
4948 rcu_read_unlock();
4949 } else
4950#endif
4951 {
4952 unsigned int qtail;
4953
4954 ret = enqueue_to_backlog(skb, smp_processor_id(), &qtail);
4955 }
4956 return ret;
4957}
4958
4959/**
4960 * __netif_rx - Slightly optimized version of netif_rx
4961 * @skb: buffer to post
4962 *
4963 * This behaves as netif_rx except that it does not disable bottom halves.
4964 * As a result this function may only be invoked from the interrupt context
4965 * (either hard or soft interrupt).
4966 */
4967int __netif_rx(struct sk_buff *skb)
4968{
4969 int ret;
4970
4971 lockdep_assert_once(hardirq_count() | softirq_count());
4972
4973 trace_netif_rx_entry(skb);
4974 ret = netif_rx_internal(skb);
4975 trace_netif_rx_exit(ret);
4976 return ret;
4977}
4978EXPORT_SYMBOL(__netif_rx);
4979
4980/**
4981 * netif_rx - post buffer to the network code
4982 * @skb: buffer to post
4983 *
4984 * This function receives a packet from a device driver and queues it for
4985 * the upper (protocol) levels to process via the backlog NAPI device. It
4986 * always succeeds. The buffer may be dropped during processing for
4987 * congestion control or by the protocol layers.
4988 * The network buffer is passed via the backlog NAPI device. Modern NIC
4989 * driver should use NAPI and GRO.
4990 * This function can used from interrupt and from process context. The
4991 * caller from process context must not disable interrupts before invoking
4992 * this function.
4993 *
4994 * return values:
4995 * NET_RX_SUCCESS (no congestion)
4996 * NET_RX_DROP (packet was dropped)
4997 *
4998 */
4999int netif_rx(struct sk_buff *skb)
5000{
5001 bool need_bh_off = !(hardirq_count() | softirq_count());
5002 int ret;
5003
5004 if (need_bh_off)
5005 local_bh_disable();
5006 trace_netif_rx_entry(skb);
5007 ret = netif_rx_internal(skb);
5008 trace_netif_rx_exit(ret);
5009 if (need_bh_off)
5010 local_bh_enable();
5011 return ret;
5012}
5013EXPORT_SYMBOL(netif_rx);
5014
5015static __latent_entropy void net_tx_action(struct softirq_action *h)
5016{
5017 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
5018
5019 if (sd->completion_queue) {
5020 struct sk_buff *clist;
5021
5022 local_irq_disable();
5023 clist = sd->completion_queue;
5024 sd->completion_queue = NULL;
5025 local_irq_enable();
5026
5027 while (clist) {
5028 struct sk_buff *skb = clist;
5029
5030 clist = clist->next;
5031
5032 WARN_ON(refcount_read(&skb->users));
5033 if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED))
5034 trace_consume_skb(skb);
5035 else
5036 trace_kfree_skb(skb, net_tx_action,
5037 SKB_DROP_REASON_NOT_SPECIFIED);
5038
5039 if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
5040 __kfree_skb(skb);
5041 else
5042 __kfree_skb_defer(skb);
5043 }
5044 }
5045
5046 if (sd->output_queue) {
5047 struct Qdisc *head;
5048
5049 local_irq_disable();
5050 head = sd->output_queue;
5051 sd->output_queue = NULL;
5052 sd->output_queue_tailp = &sd->output_queue;
5053 local_irq_enable();
5054
5055 rcu_read_lock();
5056
5057 while (head) {
5058 struct Qdisc *q = head;
5059 spinlock_t *root_lock = NULL;
5060
5061 head = head->next_sched;
5062
5063 /* We need to make sure head->next_sched is read
5064 * before clearing __QDISC_STATE_SCHED
5065 */
5066 smp_mb__before_atomic();
5067
5068 if (!(q->flags & TCQ_F_NOLOCK)) {
5069 root_lock =