1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* |
3 | * net/sched/sch_generic.c Generic packet scheduler routines. |
4 | * |
5 | * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru> |
6 | * Jamal Hadi Salim, <hadi@cyberus.ca> 990601 |
7 | * - Ingress support |
8 | */ |
9 | |
10 | #include <linux/bitops.h> |
11 | #include <linux/module.h> |
12 | #include <linux/types.h> |
13 | #include <linux/kernel.h> |
14 | #include <linux/sched.h> |
15 | #include <linux/string.h> |
16 | #include <linux/errno.h> |
17 | #include <linux/netdevice.h> |
18 | #include <linux/skbuff.h> |
19 | #include <linux/rtnetlink.h> |
20 | #include <linux/init.h> |
21 | #include <linux/rcupdate.h> |
22 | #include <linux/list.h> |
23 | #include <linux/slab.h> |
24 | #include <linux/if_vlan.h> |
25 | #include <linux/skb_array.h> |
26 | #include <linux/if_macvlan.h> |
27 | #include <net/sch_generic.h> |
28 | #include <net/pkt_sched.h> |
29 | #include <net/dst.h> |
30 | #include <net/hotdata.h> |
31 | #include <trace/events/qdisc.h> |
32 | #include <trace/events/net.h> |
33 | #include <net/xfrm.h> |
34 | |
35 | /* Qdisc to use by default */ |
36 | const struct Qdisc_ops *default_qdisc_ops = &pfifo_fast_ops; |
37 | EXPORT_SYMBOL(default_qdisc_ops); |
38 | |
39 | static void qdisc_maybe_clear_missed(struct Qdisc *q, |
40 | const struct netdev_queue *txq) |
41 | { |
42 | clear_bit(nr: __QDISC_STATE_MISSED, addr: &q->state); |
43 | |
44 | /* Make sure the below netif_xmit_frozen_or_stopped() |
45 | * checking happens after clearing STATE_MISSED. |
46 | */ |
47 | smp_mb__after_atomic(); |
48 | |
49 | /* Checking netif_xmit_frozen_or_stopped() again to |
50 | * make sure STATE_MISSED is set if the STATE_MISSED |
51 | * set by netif_tx_wake_queue()'s rescheduling of |
52 | * net_tx_action() is cleared by the above clear_bit(). |
53 | */ |
54 | if (!netif_xmit_frozen_or_stopped(dev_queue: txq)) |
55 | set_bit(nr: __QDISC_STATE_MISSED, addr: &q->state); |
56 | else |
57 | set_bit(nr: __QDISC_STATE_DRAINING, addr: &q->state); |
58 | } |
59 | |
60 | /* Main transmission queue. */ |
61 | |
62 | /* Modifications to data participating in scheduling must be protected with |
63 | * qdisc_lock(qdisc) spinlock. |
64 | * |
65 | * The idea is the following: |
66 | * - enqueue, dequeue are serialized via qdisc root lock |
67 | * - ingress filtering is also serialized via qdisc root lock |
68 | * - updates to tree and tree walking are only done under the rtnl mutex. |
69 | */ |
70 | |
71 | #define SKB_XOFF_MAGIC ((struct sk_buff *)1UL) |
72 | |
73 | static inline struct sk_buff *__skb_dequeue_bad_txq(struct Qdisc *q) |
74 | { |
75 | const struct netdev_queue *txq = q->dev_queue; |
76 | spinlock_t *lock = NULL; |
77 | struct sk_buff *skb; |
78 | |
79 | if (q->flags & TCQ_F_NOLOCK) { |
80 | lock = qdisc_lock(qdisc: q); |
81 | spin_lock(lock); |
82 | } |
83 | |
84 | skb = skb_peek(list_: &q->skb_bad_txq); |
85 | if (skb) { |
86 | /* check the reason of requeuing without tx lock first */ |
87 | txq = skb_get_tx_queue(dev: txq->dev, skb); |
88 | if (!netif_xmit_frozen_or_stopped(dev_queue: txq)) { |
89 | skb = __skb_dequeue(list: &q->skb_bad_txq); |
90 | if (qdisc_is_percpu_stats(q)) { |
91 | qdisc_qstats_cpu_backlog_dec(sch: q, skb); |
92 | qdisc_qstats_cpu_qlen_dec(sch: q); |
93 | } else { |
94 | qdisc_qstats_backlog_dec(sch: q, skb); |
95 | q->q.qlen--; |
96 | } |
97 | } else { |
98 | skb = SKB_XOFF_MAGIC; |
99 | qdisc_maybe_clear_missed(q, txq); |
100 | } |
101 | } |
102 | |
103 | if (lock) |
104 | spin_unlock(lock); |
105 | |
106 | return skb; |
107 | } |
108 | |
109 | static inline struct sk_buff *qdisc_dequeue_skb_bad_txq(struct Qdisc *q) |
110 | { |
111 | struct sk_buff *skb = skb_peek(list_: &q->skb_bad_txq); |
112 | |
113 | if (unlikely(skb)) |
114 | skb = __skb_dequeue_bad_txq(q); |
115 | |
116 | return skb; |
117 | } |
118 | |
119 | static inline void qdisc_enqueue_skb_bad_txq(struct Qdisc *q, |
120 | struct sk_buff *skb) |
121 | { |
122 | spinlock_t *lock = NULL; |
123 | |
124 | if (q->flags & TCQ_F_NOLOCK) { |
125 | lock = qdisc_lock(qdisc: q); |
126 | spin_lock(lock); |
127 | } |
128 | |
129 | __skb_queue_tail(list: &q->skb_bad_txq, newsk: skb); |
130 | |
131 | if (qdisc_is_percpu_stats(q)) { |
132 | qdisc_qstats_cpu_backlog_inc(sch: q, skb); |
133 | qdisc_qstats_cpu_qlen_inc(sch: q); |
134 | } else { |
135 | qdisc_qstats_backlog_inc(sch: q, skb); |
136 | q->q.qlen++; |
137 | } |
138 | |
139 | if (lock) |
140 | spin_unlock(lock); |
141 | } |
142 | |
143 | static inline void dev_requeue_skb(struct sk_buff *skb, struct Qdisc *q) |
144 | { |
145 | spinlock_t *lock = NULL; |
146 | |
147 | if (q->flags & TCQ_F_NOLOCK) { |
148 | lock = qdisc_lock(qdisc: q); |
149 | spin_lock(lock); |
150 | } |
151 | |
152 | while (skb) { |
153 | struct sk_buff *next = skb->next; |
154 | |
155 | __skb_queue_tail(list: &q->gso_skb, newsk: skb); |
156 | |
157 | /* it's still part of the queue */ |
158 | if (qdisc_is_percpu_stats(q)) { |
159 | qdisc_qstats_cpu_requeues_inc(sch: q); |
160 | qdisc_qstats_cpu_backlog_inc(sch: q, skb); |
161 | qdisc_qstats_cpu_qlen_inc(sch: q); |
162 | } else { |
163 | q->qstats.requeues++; |
164 | qdisc_qstats_backlog_inc(sch: q, skb); |
165 | q->q.qlen++; |
166 | } |
167 | |
168 | skb = next; |
169 | } |
170 | |
171 | if (lock) { |
172 | spin_unlock(lock); |
173 | set_bit(nr: __QDISC_STATE_MISSED, addr: &q->state); |
174 | } else { |
175 | __netif_schedule(q); |
176 | } |
177 | } |
178 | |
179 | static void try_bulk_dequeue_skb(struct Qdisc *q, |
180 | struct sk_buff *skb, |
181 | const struct netdev_queue *txq, |
182 | int *packets) |
183 | { |
184 | int bytelimit = qdisc_avail_bulklimit(txq) - skb->len; |
185 | |
186 | while (bytelimit > 0) { |
187 | struct sk_buff *nskb = q->dequeue(q); |
188 | |
189 | if (!nskb) |
190 | break; |
191 | |
192 | bytelimit -= nskb->len; /* covers GSO len */ |
193 | skb->next = nskb; |
194 | skb = nskb; |
195 | (*packets)++; /* GSO counts as one pkt */ |
196 | } |
197 | skb_mark_not_on_list(skb); |
198 | } |
199 | |
200 | /* This variant of try_bulk_dequeue_skb() makes sure |
201 | * all skbs in the chain are for the same txq |
202 | */ |
203 | static void try_bulk_dequeue_skb_slow(struct Qdisc *q, |
204 | struct sk_buff *skb, |
205 | int *packets) |
206 | { |
207 | int mapping = skb_get_queue_mapping(skb); |
208 | struct sk_buff *nskb; |
209 | int cnt = 0; |
210 | |
211 | do { |
212 | nskb = q->dequeue(q); |
213 | if (!nskb) |
214 | break; |
215 | if (unlikely(skb_get_queue_mapping(nskb) != mapping)) { |
216 | qdisc_enqueue_skb_bad_txq(q, skb: nskb); |
217 | break; |
218 | } |
219 | skb->next = nskb; |
220 | skb = nskb; |
221 | } while (++cnt < 8); |
222 | (*packets) += cnt; |
223 | skb_mark_not_on_list(skb); |
224 | } |
225 | |
226 | /* Note that dequeue_skb can possibly return a SKB list (via skb->next). |
227 | * A requeued skb (via q->gso_skb) can also be a SKB list. |
228 | */ |
229 | static struct sk_buff *dequeue_skb(struct Qdisc *q, bool *validate, |
230 | int *packets) |
231 | { |
232 | const struct netdev_queue *txq = q->dev_queue; |
233 | struct sk_buff *skb = NULL; |
234 | |
235 | *packets = 1; |
236 | if (unlikely(!skb_queue_empty(&q->gso_skb))) { |
237 | spinlock_t *lock = NULL; |
238 | |
239 | if (q->flags & TCQ_F_NOLOCK) { |
240 | lock = qdisc_lock(qdisc: q); |
241 | spin_lock(lock); |
242 | } |
243 | |
244 | skb = skb_peek(list_: &q->gso_skb); |
245 | |
246 | /* skb may be null if another cpu pulls gso_skb off in between |
247 | * empty check and lock. |
248 | */ |
249 | if (!skb) { |
250 | if (lock) |
251 | spin_unlock(lock); |
252 | goto validate; |
253 | } |
254 | |
255 | /* skb in gso_skb were already validated */ |
256 | *validate = false; |
257 | if (xfrm_offload(skb)) |
258 | *validate = true; |
259 | /* check the reason of requeuing without tx lock first */ |
260 | txq = skb_get_tx_queue(dev: txq->dev, skb); |
261 | if (!netif_xmit_frozen_or_stopped(dev_queue: txq)) { |
262 | skb = __skb_dequeue(list: &q->gso_skb); |
263 | if (qdisc_is_percpu_stats(q)) { |
264 | qdisc_qstats_cpu_backlog_dec(sch: q, skb); |
265 | qdisc_qstats_cpu_qlen_dec(sch: q); |
266 | } else { |
267 | qdisc_qstats_backlog_dec(sch: q, skb); |
268 | q->q.qlen--; |
269 | } |
270 | } else { |
271 | skb = NULL; |
272 | qdisc_maybe_clear_missed(q, txq); |
273 | } |
274 | if (lock) |
275 | spin_unlock(lock); |
276 | goto trace; |
277 | } |
278 | validate: |
279 | *validate = true; |
280 | |
281 | if ((q->flags & TCQ_F_ONETXQUEUE) && |
282 | netif_xmit_frozen_or_stopped(dev_queue: txq)) { |
283 | qdisc_maybe_clear_missed(q, txq); |
284 | return skb; |
285 | } |
286 | |
287 | skb = qdisc_dequeue_skb_bad_txq(q); |
288 | if (unlikely(skb)) { |
289 | if (skb == SKB_XOFF_MAGIC) |
290 | return NULL; |
291 | goto bulk; |
292 | } |
293 | skb = q->dequeue(q); |
294 | if (skb) { |
295 | bulk: |
296 | if (qdisc_may_bulk(qdisc: q)) |
297 | try_bulk_dequeue_skb(q, skb, txq, packets); |
298 | else |
299 | try_bulk_dequeue_skb_slow(q, skb, packets); |
300 | } |
301 | trace: |
302 | trace_qdisc_dequeue(qdisc: q, txq, packets: *packets, skb); |
303 | return skb; |
304 | } |
305 | |
306 | /* |
307 | * Transmit possibly several skbs, and handle the return status as |
308 | * required. Owning qdisc running bit guarantees that only one CPU |
309 | * can execute this function. |
310 | * |
311 | * Returns to the caller: |
312 | * false - hardware queue frozen backoff |
313 | * true - feel free to send more pkts |
314 | */ |
315 | bool sch_direct_xmit(struct sk_buff *skb, struct Qdisc *q, |
316 | struct net_device *dev, struct netdev_queue *txq, |
317 | spinlock_t *root_lock, bool validate) |
318 | { |
319 | int ret = NETDEV_TX_BUSY; |
320 | bool again = false; |
321 | |
322 | /* And release qdisc */ |
323 | if (root_lock) |
324 | spin_unlock(lock: root_lock); |
325 | |
326 | /* Note that we validate skb (GSO, checksum, ...) outside of locks */ |
327 | if (validate) |
328 | skb = validate_xmit_skb_list(skb, dev, again: &again); |
329 | |
330 | #ifdef CONFIG_XFRM_OFFLOAD |
331 | if (unlikely(again)) { |
332 | if (root_lock) |
333 | spin_lock(lock: root_lock); |
334 | |
335 | dev_requeue_skb(skb, q); |
336 | return false; |
337 | } |
338 | #endif |
339 | |
340 | if (likely(skb)) { |
341 | HARD_TX_LOCK(dev, txq, smp_processor_id()); |
342 | if (!netif_xmit_frozen_or_stopped(dev_queue: txq)) |
343 | skb = dev_hard_start_xmit(skb, dev, txq, ret: &ret); |
344 | else |
345 | qdisc_maybe_clear_missed(q, txq); |
346 | |
347 | HARD_TX_UNLOCK(dev, txq); |
348 | } else { |
349 | if (root_lock) |
350 | spin_lock(lock: root_lock); |
351 | return true; |
352 | } |
353 | |
354 | if (root_lock) |
355 | spin_lock(lock: root_lock); |
356 | |
357 | if (!dev_xmit_complete(rc: ret)) { |
358 | /* Driver returned NETDEV_TX_BUSY - requeue skb */ |
359 | if (unlikely(ret != NETDEV_TX_BUSY)) |
360 | net_warn_ratelimited("BUG %s code %d qlen %d\n" , |
361 | dev->name, ret, q->q.qlen); |
362 | |
363 | dev_requeue_skb(skb, q); |
364 | return false; |
365 | } |
366 | |
367 | return true; |
368 | } |
369 | |
370 | /* |
371 | * NOTE: Called under qdisc_lock(q) with locally disabled BH. |
372 | * |
373 | * running seqcount guarantees only one CPU can process |
374 | * this qdisc at a time. qdisc_lock(q) serializes queue accesses for |
375 | * this queue. |
376 | * |
377 | * netif_tx_lock serializes accesses to device driver. |
378 | * |
379 | * qdisc_lock(q) and netif_tx_lock are mutually exclusive, |
380 | * if one is grabbed, another must be free. |
381 | * |
382 | * Note, that this procedure can be called by a watchdog timer |
383 | * |
384 | * Returns to the caller: |
385 | * 0 - queue is empty or throttled. |
386 | * >0 - queue is not empty. |
387 | * |
388 | */ |
389 | static inline bool qdisc_restart(struct Qdisc *q, int *packets) |
390 | { |
391 | spinlock_t *root_lock = NULL; |
392 | struct netdev_queue *txq; |
393 | struct net_device *dev; |
394 | struct sk_buff *skb; |
395 | bool validate; |
396 | |
397 | /* Dequeue packet */ |
398 | skb = dequeue_skb(q, validate: &validate, packets); |
399 | if (unlikely(!skb)) |
400 | return false; |
401 | |
402 | if (!(q->flags & TCQ_F_NOLOCK)) |
403 | root_lock = qdisc_lock(qdisc: q); |
404 | |
405 | dev = qdisc_dev(qdisc: q); |
406 | txq = skb_get_tx_queue(dev, skb); |
407 | |
408 | return sch_direct_xmit(skb, q, dev, txq, root_lock, validate); |
409 | } |
410 | |
411 | void __qdisc_run(struct Qdisc *q) |
412 | { |
413 | int quota = READ_ONCE(net_hotdata.dev_tx_weight); |
414 | int packets; |
415 | |
416 | while (qdisc_restart(q, packets: &packets)) { |
417 | quota -= packets; |
418 | if (quota <= 0) { |
419 | if (q->flags & TCQ_F_NOLOCK) |
420 | set_bit(nr: __QDISC_STATE_MISSED, addr: &q->state); |
421 | else |
422 | __netif_schedule(q); |
423 | |
424 | break; |
425 | } |
426 | } |
427 | } |
428 | |
429 | unsigned long dev_trans_start(struct net_device *dev) |
430 | { |
431 | unsigned long res = READ_ONCE(netdev_get_tx_queue(dev, 0)->trans_start); |
432 | unsigned long val; |
433 | unsigned int i; |
434 | |
435 | for (i = 1; i < dev->num_tx_queues; i++) { |
436 | val = READ_ONCE(netdev_get_tx_queue(dev, i)->trans_start); |
437 | if (val && time_after(val, res)) |
438 | res = val; |
439 | } |
440 | |
441 | return res; |
442 | } |
443 | EXPORT_SYMBOL(dev_trans_start); |
444 | |
445 | static void netif_freeze_queues(struct net_device *dev) |
446 | { |
447 | unsigned int i; |
448 | int cpu; |
449 | |
450 | cpu = smp_processor_id(); |
451 | for (i = 0; i < dev->num_tx_queues; i++) { |
452 | struct netdev_queue *txq = netdev_get_tx_queue(dev, index: i); |
453 | |
454 | /* We are the only thread of execution doing a |
455 | * freeze, but we have to grab the _xmit_lock in |
456 | * order to synchronize with threads which are in |
457 | * the ->hard_start_xmit() handler and already |
458 | * checked the frozen bit. |
459 | */ |
460 | __netif_tx_lock(txq, cpu); |
461 | set_bit(nr: __QUEUE_STATE_FROZEN, addr: &txq->state); |
462 | __netif_tx_unlock(txq); |
463 | } |
464 | } |
465 | |
466 | void netif_tx_lock(struct net_device *dev) |
467 | { |
468 | spin_lock(lock: &dev->tx_global_lock); |
469 | netif_freeze_queues(dev); |
470 | } |
471 | EXPORT_SYMBOL(netif_tx_lock); |
472 | |
473 | static void netif_unfreeze_queues(struct net_device *dev) |
474 | { |
475 | unsigned int i; |
476 | |
477 | for (i = 0; i < dev->num_tx_queues; i++) { |
478 | struct netdev_queue *txq = netdev_get_tx_queue(dev, index: i); |
479 | |
480 | /* No need to grab the _xmit_lock here. If the |
481 | * queue is not stopped for another reason, we |
482 | * force a schedule. |
483 | */ |
484 | clear_bit(nr: __QUEUE_STATE_FROZEN, addr: &txq->state); |
485 | netif_schedule_queue(txq); |
486 | } |
487 | } |
488 | |
489 | void netif_tx_unlock(struct net_device *dev) |
490 | { |
491 | netif_unfreeze_queues(dev); |
492 | spin_unlock(lock: &dev->tx_global_lock); |
493 | } |
494 | EXPORT_SYMBOL(netif_tx_unlock); |
495 | |
496 | static void dev_watchdog(struct timer_list *t) |
497 | { |
498 | struct net_device *dev = from_timer(dev, t, watchdog_timer); |
499 | bool release = true; |
500 | |
501 | spin_lock(lock: &dev->tx_global_lock); |
502 | if (!qdisc_tx_is_noop(dev)) { |
503 | if (netif_device_present(dev) && |
504 | netif_running(dev) && |
505 | netif_carrier_ok(dev)) { |
506 | unsigned int timedout_ms = 0; |
507 | unsigned int i; |
508 | unsigned long trans_start; |
509 | |
510 | for (i = 0; i < dev->num_tx_queues; i++) { |
511 | struct netdev_queue *txq; |
512 | |
513 | txq = netdev_get_tx_queue(dev, index: i); |
514 | trans_start = READ_ONCE(txq->trans_start); |
515 | if (netif_xmit_stopped(dev_queue: txq) && |
516 | time_after(jiffies, (trans_start + |
517 | dev->watchdog_timeo))) { |
518 | timedout_ms = jiffies_to_msecs(j: jiffies - trans_start); |
519 | atomic_long_inc(v: &txq->trans_timeout); |
520 | break; |
521 | } |
522 | } |
523 | |
524 | if (unlikely(timedout_ms)) { |
525 | trace_net_dev_xmit_timeout(dev, queue_index: i); |
526 | netdev_crit(dev, format: "NETDEV WATCHDOG: CPU: %d: transmit queue %u timed out %u ms\n" , |
527 | raw_smp_processor_id(), |
528 | i, timedout_ms); |
529 | netif_freeze_queues(dev); |
530 | dev->netdev_ops->ndo_tx_timeout(dev, i); |
531 | netif_unfreeze_queues(dev); |
532 | } |
533 | if (!mod_timer(timer: &dev->watchdog_timer, |
534 | expires: round_jiffies(j: jiffies + |
535 | dev->watchdog_timeo))) |
536 | release = false; |
537 | } |
538 | } |
539 | spin_unlock(lock: &dev->tx_global_lock); |
540 | |
541 | if (release) |
542 | netdev_put(dev, tracker: &dev->watchdog_dev_tracker); |
543 | } |
544 | |
545 | void __netdev_watchdog_up(struct net_device *dev) |
546 | { |
547 | if (dev->netdev_ops->ndo_tx_timeout) { |
548 | if (dev->watchdog_timeo <= 0) |
549 | dev->watchdog_timeo = 5*HZ; |
550 | if (!mod_timer(timer: &dev->watchdog_timer, |
551 | expires: round_jiffies(j: jiffies + dev->watchdog_timeo))) |
552 | netdev_hold(dev, tracker: &dev->watchdog_dev_tracker, |
553 | GFP_ATOMIC); |
554 | } |
555 | } |
556 | EXPORT_SYMBOL_GPL(__netdev_watchdog_up); |
557 | |
558 | static void dev_watchdog_up(struct net_device *dev) |
559 | { |
560 | __netdev_watchdog_up(dev); |
561 | } |
562 | |
563 | static void dev_watchdog_down(struct net_device *dev) |
564 | { |
565 | netif_tx_lock_bh(dev); |
566 | if (del_timer(timer: &dev->watchdog_timer)) |
567 | netdev_put(dev, tracker: &dev->watchdog_dev_tracker); |
568 | netif_tx_unlock_bh(dev); |
569 | } |
570 | |
571 | /** |
572 | * netif_carrier_on - set carrier |
573 | * @dev: network device |
574 | * |
575 | * Device has detected acquisition of carrier. |
576 | */ |
577 | void netif_carrier_on(struct net_device *dev) |
578 | { |
579 | if (test_and_clear_bit(nr: __LINK_STATE_NOCARRIER, addr: &dev->state)) { |
580 | if (dev->reg_state == NETREG_UNINITIALIZED) |
581 | return; |
582 | atomic_inc(v: &dev->carrier_up_count); |
583 | linkwatch_fire_event(dev); |
584 | if (netif_running(dev)) |
585 | __netdev_watchdog_up(dev); |
586 | } |
587 | } |
588 | EXPORT_SYMBOL(netif_carrier_on); |
589 | |
590 | /** |
591 | * netif_carrier_off - clear carrier |
592 | * @dev: network device |
593 | * |
594 | * Device has detected loss of carrier. |
595 | */ |
596 | void netif_carrier_off(struct net_device *dev) |
597 | { |
598 | if (!test_and_set_bit(nr: __LINK_STATE_NOCARRIER, addr: &dev->state)) { |
599 | if (dev->reg_state == NETREG_UNINITIALIZED) |
600 | return; |
601 | atomic_inc(v: &dev->carrier_down_count); |
602 | linkwatch_fire_event(dev); |
603 | } |
604 | } |
605 | EXPORT_SYMBOL(netif_carrier_off); |
606 | |
607 | /** |
608 | * netif_carrier_event - report carrier state event |
609 | * @dev: network device |
610 | * |
611 | * Device has detected a carrier event but the carrier state wasn't changed. |
612 | * Use in drivers when querying carrier state asynchronously, to avoid missing |
613 | * events (link flaps) if link recovers before it's queried. |
614 | */ |
615 | void netif_carrier_event(struct net_device *dev) |
616 | { |
617 | if (dev->reg_state == NETREG_UNINITIALIZED) |
618 | return; |
619 | atomic_inc(v: &dev->carrier_up_count); |
620 | atomic_inc(v: &dev->carrier_down_count); |
621 | linkwatch_fire_event(dev); |
622 | } |
623 | EXPORT_SYMBOL_GPL(netif_carrier_event); |
624 | |
625 | /* "NOOP" scheduler: the best scheduler, recommended for all interfaces |
626 | under all circumstances. It is difficult to invent anything faster or |
627 | cheaper. |
628 | */ |
629 | |
630 | static int noop_enqueue(struct sk_buff *skb, struct Qdisc *qdisc, |
631 | struct sk_buff **to_free) |
632 | { |
633 | __qdisc_drop(skb, to_free); |
634 | return NET_XMIT_CN; |
635 | } |
636 | |
637 | static struct sk_buff *noop_dequeue(struct Qdisc *qdisc) |
638 | { |
639 | return NULL; |
640 | } |
641 | |
642 | struct Qdisc_ops noop_qdisc_ops __read_mostly = { |
643 | .id = "noop" , |
644 | .priv_size = 0, |
645 | .enqueue = noop_enqueue, |
646 | .dequeue = noop_dequeue, |
647 | .peek = noop_dequeue, |
648 | .owner = THIS_MODULE, |
649 | }; |
650 | |
651 | static struct netdev_queue noop_netdev_queue = { |
652 | RCU_POINTER_INITIALIZER(qdisc, &noop_qdisc), |
653 | RCU_POINTER_INITIALIZER(qdisc_sleeping, &noop_qdisc), |
654 | }; |
655 | |
656 | struct Qdisc noop_qdisc = { |
657 | .enqueue = noop_enqueue, |
658 | .dequeue = noop_dequeue, |
659 | .flags = TCQ_F_BUILTIN, |
660 | .ops = &noop_qdisc_ops, |
661 | .q.lock = __SPIN_LOCK_UNLOCKED(noop_qdisc.q.lock), |
662 | .dev_queue = &noop_netdev_queue, |
663 | .busylock = __SPIN_LOCK_UNLOCKED(noop_qdisc.busylock), |
664 | .gso_skb = { |
665 | .next = (struct sk_buff *)&noop_qdisc.gso_skb, |
666 | .prev = (struct sk_buff *)&noop_qdisc.gso_skb, |
667 | .qlen = 0, |
668 | .lock = __SPIN_LOCK_UNLOCKED(noop_qdisc.gso_skb.lock), |
669 | }, |
670 | .skb_bad_txq = { |
671 | .next = (struct sk_buff *)&noop_qdisc.skb_bad_txq, |
672 | .prev = (struct sk_buff *)&noop_qdisc.skb_bad_txq, |
673 | .qlen = 0, |
674 | .lock = __SPIN_LOCK_UNLOCKED(noop_qdisc.skb_bad_txq.lock), |
675 | }, |
676 | }; |
677 | EXPORT_SYMBOL(noop_qdisc); |
678 | |
679 | static int noqueue_init(struct Qdisc *qdisc, struct nlattr *opt, |
680 | struct netlink_ext_ack *extack) |
681 | { |
682 | /* register_qdisc() assigns a default of noop_enqueue if unset, |
683 | * but __dev_queue_xmit() treats noqueue only as such |
684 | * if this is NULL - so clear it here. */ |
685 | qdisc->enqueue = NULL; |
686 | return 0; |
687 | } |
688 | |
689 | struct Qdisc_ops noqueue_qdisc_ops __read_mostly = { |
690 | .id = "noqueue" , |
691 | .priv_size = 0, |
692 | .init = noqueue_init, |
693 | .enqueue = noop_enqueue, |
694 | .dequeue = noop_dequeue, |
695 | .peek = noop_dequeue, |
696 | .owner = THIS_MODULE, |
697 | }; |
698 | |
699 | const u8 sch_default_prio2band[TC_PRIO_MAX + 1] = { |
700 | 1, 2, 2, 2, 1, 2, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1 |
701 | }; |
702 | EXPORT_SYMBOL(sch_default_prio2band); |
703 | |
704 | /* 3-band FIFO queue: old style, but should be a bit faster than |
705 | generic prio+fifo combination. |
706 | */ |
707 | |
708 | #define PFIFO_FAST_BANDS 3 |
709 | |
710 | /* |
711 | * Private data for a pfifo_fast scheduler containing: |
712 | * - rings for priority bands |
713 | */ |
714 | struct pfifo_fast_priv { |
715 | struct skb_array q[PFIFO_FAST_BANDS]; |
716 | }; |
717 | |
718 | static inline struct skb_array *band2list(struct pfifo_fast_priv *priv, |
719 | int band) |
720 | { |
721 | return &priv->q[band]; |
722 | } |
723 | |
724 | static int pfifo_fast_enqueue(struct sk_buff *skb, struct Qdisc *qdisc, |
725 | struct sk_buff **to_free) |
726 | { |
727 | int band = sch_default_prio2band[skb->priority & TC_PRIO_MAX]; |
728 | struct pfifo_fast_priv *priv = qdisc_priv(qdisc); |
729 | struct skb_array *q = band2list(priv, band); |
730 | unsigned int pkt_len = qdisc_pkt_len(skb); |
731 | int err; |
732 | |
733 | err = skb_array_produce(a: q, skb); |
734 | |
735 | if (unlikely(err)) { |
736 | if (qdisc_is_percpu_stats(q: qdisc)) |
737 | return qdisc_drop_cpu(skb, sch: qdisc, to_free); |
738 | else |
739 | return qdisc_drop(skb, sch: qdisc, to_free); |
740 | } |
741 | |
742 | qdisc_update_stats_at_enqueue(sch: qdisc, pkt_len); |
743 | return NET_XMIT_SUCCESS; |
744 | } |
745 | |
746 | static struct sk_buff *pfifo_fast_dequeue(struct Qdisc *qdisc) |
747 | { |
748 | struct pfifo_fast_priv *priv = qdisc_priv(qdisc); |
749 | struct sk_buff *skb = NULL; |
750 | bool need_retry = true; |
751 | int band; |
752 | |
753 | retry: |
754 | for (band = 0; band < PFIFO_FAST_BANDS && !skb; band++) { |
755 | struct skb_array *q = band2list(priv, band); |
756 | |
757 | if (__skb_array_empty(a: q)) |
758 | continue; |
759 | |
760 | skb = __skb_array_consume(a: q); |
761 | } |
762 | if (likely(skb)) { |
763 | qdisc_update_stats_at_dequeue(sch: qdisc, skb); |
764 | } else if (need_retry && |
765 | READ_ONCE(qdisc->state) & QDISC_STATE_NON_EMPTY) { |
766 | /* Delay clearing the STATE_MISSED here to reduce |
767 | * the overhead of the second spin_trylock() in |
768 | * qdisc_run_begin() and __netif_schedule() calling |
769 | * in qdisc_run_end(). |
770 | */ |
771 | clear_bit(nr: __QDISC_STATE_MISSED, addr: &qdisc->state); |
772 | clear_bit(nr: __QDISC_STATE_DRAINING, addr: &qdisc->state); |
773 | |
774 | /* Make sure dequeuing happens after clearing |
775 | * STATE_MISSED. |
776 | */ |
777 | smp_mb__after_atomic(); |
778 | |
779 | need_retry = false; |
780 | |
781 | goto retry; |
782 | } |
783 | |
784 | return skb; |
785 | } |
786 | |
787 | static struct sk_buff *pfifo_fast_peek(struct Qdisc *qdisc) |
788 | { |
789 | struct pfifo_fast_priv *priv = qdisc_priv(qdisc); |
790 | struct sk_buff *skb = NULL; |
791 | int band; |
792 | |
793 | for (band = 0; band < PFIFO_FAST_BANDS && !skb; band++) { |
794 | struct skb_array *q = band2list(priv, band); |
795 | |
796 | skb = __skb_array_peek(a: q); |
797 | } |
798 | |
799 | return skb; |
800 | } |
801 | |
802 | static void pfifo_fast_reset(struct Qdisc *qdisc) |
803 | { |
804 | int i, band; |
805 | struct pfifo_fast_priv *priv = qdisc_priv(qdisc); |
806 | |
807 | for (band = 0; band < PFIFO_FAST_BANDS; band++) { |
808 | struct skb_array *q = band2list(priv, band); |
809 | struct sk_buff *skb; |
810 | |
811 | /* NULL ring is possible if destroy path is due to a failed |
812 | * skb_array_init() in pfifo_fast_init() case. |
813 | */ |
814 | if (!q->ring.queue) |
815 | continue; |
816 | |
817 | while ((skb = __skb_array_consume(a: q)) != NULL) |
818 | kfree_skb(skb); |
819 | } |
820 | |
821 | if (qdisc_is_percpu_stats(q: qdisc)) { |
822 | for_each_possible_cpu(i) { |
823 | struct gnet_stats_queue *q; |
824 | |
825 | q = per_cpu_ptr(qdisc->cpu_qstats, i); |
826 | q->backlog = 0; |
827 | q->qlen = 0; |
828 | } |
829 | } |
830 | } |
831 | |
832 | static int pfifo_fast_dump(struct Qdisc *qdisc, struct sk_buff *skb) |
833 | { |
834 | struct tc_prio_qopt opt = { .bands = PFIFO_FAST_BANDS }; |
835 | |
836 | memcpy(&opt.priomap, sch_default_prio2band, TC_PRIO_MAX + 1); |
837 | if (nla_put(skb, attrtype: TCA_OPTIONS, attrlen: sizeof(opt), data: &opt)) |
838 | goto nla_put_failure; |
839 | return skb->len; |
840 | |
841 | nla_put_failure: |
842 | return -1; |
843 | } |
844 | |
845 | static int pfifo_fast_init(struct Qdisc *qdisc, struct nlattr *opt, |
846 | struct netlink_ext_ack *extack) |
847 | { |
848 | unsigned int qlen = qdisc_dev(qdisc)->tx_queue_len; |
849 | struct pfifo_fast_priv *priv = qdisc_priv(qdisc); |
850 | int prio; |
851 | |
852 | /* guard against zero length rings */ |
853 | if (!qlen) |
854 | return -EINVAL; |
855 | |
856 | for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) { |
857 | struct skb_array *q = band2list(priv, band: prio); |
858 | int err; |
859 | |
860 | err = skb_array_init(a: q, size: qlen, GFP_KERNEL); |
861 | if (err) |
862 | return -ENOMEM; |
863 | } |
864 | |
865 | /* Can by-pass the queue discipline */ |
866 | qdisc->flags |= TCQ_F_CAN_BYPASS; |
867 | return 0; |
868 | } |
869 | |
870 | static void pfifo_fast_destroy(struct Qdisc *sch) |
871 | { |
872 | struct pfifo_fast_priv *priv = qdisc_priv(sch); |
873 | int prio; |
874 | |
875 | for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) { |
876 | struct skb_array *q = band2list(priv, band: prio); |
877 | |
878 | /* NULL ring is possible if destroy path is due to a failed |
879 | * skb_array_init() in pfifo_fast_init() case. |
880 | */ |
881 | if (!q->ring.queue) |
882 | continue; |
883 | /* Destroy ring but no need to kfree_skb because a call to |
884 | * pfifo_fast_reset() has already done that work. |
885 | */ |
886 | ptr_ring_cleanup(r: &q->ring, NULL); |
887 | } |
888 | } |
889 | |
890 | static int pfifo_fast_change_tx_queue_len(struct Qdisc *sch, |
891 | unsigned int new_len) |
892 | { |
893 | struct pfifo_fast_priv *priv = qdisc_priv(sch); |
894 | struct skb_array *bands[PFIFO_FAST_BANDS]; |
895 | int prio; |
896 | |
897 | for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) { |
898 | struct skb_array *q = band2list(priv, band: prio); |
899 | |
900 | bands[prio] = q; |
901 | } |
902 | |
903 | return skb_array_resize_multiple(rings: bands, PFIFO_FAST_BANDS, size: new_len, |
904 | GFP_KERNEL); |
905 | } |
906 | |
907 | struct Qdisc_ops pfifo_fast_ops __read_mostly = { |
908 | .id = "pfifo_fast" , |
909 | .priv_size = sizeof(struct pfifo_fast_priv), |
910 | .enqueue = pfifo_fast_enqueue, |
911 | .dequeue = pfifo_fast_dequeue, |
912 | .peek = pfifo_fast_peek, |
913 | .init = pfifo_fast_init, |
914 | .destroy = pfifo_fast_destroy, |
915 | .reset = pfifo_fast_reset, |
916 | .dump = pfifo_fast_dump, |
917 | .change_tx_queue_len = pfifo_fast_change_tx_queue_len, |
918 | .owner = THIS_MODULE, |
919 | .static_flags = TCQ_F_NOLOCK | TCQ_F_CPUSTATS, |
920 | }; |
921 | EXPORT_SYMBOL(pfifo_fast_ops); |
922 | |
923 | static struct lock_class_key qdisc_tx_busylock; |
924 | |
925 | struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue, |
926 | const struct Qdisc_ops *ops, |
927 | struct netlink_ext_ack *extack) |
928 | { |
929 | struct Qdisc *sch; |
930 | unsigned int size = sizeof(*sch) + ops->priv_size; |
931 | int err = -ENOBUFS; |
932 | struct net_device *dev; |
933 | |
934 | if (!dev_queue) { |
935 | NL_SET_ERR_MSG(extack, "No device queue given" ); |
936 | err = -EINVAL; |
937 | goto errout; |
938 | } |
939 | |
940 | dev = dev_queue->dev; |
941 | sch = kzalloc_node(size, GFP_KERNEL, node: netdev_queue_numa_node_read(q: dev_queue)); |
942 | |
943 | if (!sch) |
944 | goto errout; |
945 | __skb_queue_head_init(list: &sch->gso_skb); |
946 | __skb_queue_head_init(list: &sch->skb_bad_txq); |
947 | gnet_stats_basic_sync_init(b: &sch->bstats); |
948 | spin_lock_init(&sch->q.lock); |
949 | |
950 | if (ops->static_flags & TCQ_F_CPUSTATS) { |
951 | sch->cpu_bstats = |
952 | netdev_alloc_pcpu_stats(struct gnet_stats_basic_sync); |
953 | if (!sch->cpu_bstats) |
954 | goto errout1; |
955 | |
956 | sch->cpu_qstats = alloc_percpu(struct gnet_stats_queue); |
957 | if (!sch->cpu_qstats) { |
958 | free_percpu(pdata: sch->cpu_bstats); |
959 | goto errout1; |
960 | } |
961 | } |
962 | |
963 | spin_lock_init(&sch->busylock); |
964 | lockdep_set_class(&sch->busylock, |
965 | dev->qdisc_tx_busylock ?: &qdisc_tx_busylock); |
966 | |
967 | /* seqlock has the same scope of busylock, for NOLOCK qdisc */ |
968 | spin_lock_init(&sch->seqlock); |
969 | lockdep_set_class(&sch->seqlock, |
970 | dev->qdisc_tx_busylock ?: &qdisc_tx_busylock); |
971 | |
972 | sch->ops = ops; |
973 | sch->flags = ops->static_flags; |
974 | sch->enqueue = ops->enqueue; |
975 | sch->dequeue = ops->dequeue; |
976 | sch->dev_queue = dev_queue; |
977 | sch->owner = -1; |
978 | netdev_hold(dev, tracker: &sch->dev_tracker, GFP_KERNEL); |
979 | refcount_set(r: &sch->refcnt, n: 1); |
980 | |
981 | return sch; |
982 | errout1: |
983 | kfree(objp: sch); |
984 | errout: |
985 | return ERR_PTR(error: err); |
986 | } |
987 | |
988 | struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue, |
989 | const struct Qdisc_ops *ops, |
990 | unsigned int parentid, |
991 | struct netlink_ext_ack *extack) |
992 | { |
993 | struct Qdisc *sch; |
994 | |
995 | if (!try_module_get(module: ops->owner)) { |
996 | NL_SET_ERR_MSG(extack, "Failed to increase module reference counter" ); |
997 | return NULL; |
998 | } |
999 | |
1000 | sch = qdisc_alloc(dev_queue, ops, extack); |
1001 | if (IS_ERR(ptr: sch)) { |
1002 | module_put(module: ops->owner); |
1003 | return NULL; |
1004 | } |
1005 | sch->parent = parentid; |
1006 | |
1007 | if (!ops->init || ops->init(sch, NULL, extack) == 0) { |
1008 | trace_qdisc_create(ops, dev: dev_queue->dev, parent: parentid); |
1009 | return sch; |
1010 | } |
1011 | |
1012 | qdisc_put(qdisc: sch); |
1013 | return NULL; |
1014 | } |
1015 | EXPORT_SYMBOL(qdisc_create_dflt); |
1016 | |
1017 | /* Under qdisc_lock(qdisc) and BH! */ |
1018 | |
1019 | void qdisc_reset(struct Qdisc *qdisc) |
1020 | { |
1021 | const struct Qdisc_ops *ops = qdisc->ops; |
1022 | |
1023 | trace_qdisc_reset(q: qdisc); |
1024 | |
1025 | if (ops->reset) |
1026 | ops->reset(qdisc); |
1027 | |
1028 | __skb_queue_purge(list: &qdisc->gso_skb); |
1029 | __skb_queue_purge(list: &qdisc->skb_bad_txq); |
1030 | |
1031 | qdisc->q.qlen = 0; |
1032 | qdisc->qstats.backlog = 0; |
1033 | } |
1034 | EXPORT_SYMBOL(qdisc_reset); |
1035 | |
1036 | void qdisc_free(struct Qdisc *qdisc) |
1037 | { |
1038 | if (qdisc_is_percpu_stats(q: qdisc)) { |
1039 | free_percpu(pdata: qdisc->cpu_bstats); |
1040 | free_percpu(pdata: qdisc->cpu_qstats); |
1041 | } |
1042 | |
1043 | kfree(objp: qdisc); |
1044 | } |
1045 | |
1046 | static void qdisc_free_cb(struct rcu_head *head) |
1047 | { |
1048 | struct Qdisc *q = container_of(head, struct Qdisc, rcu); |
1049 | |
1050 | qdisc_free(qdisc: q); |
1051 | } |
1052 | |
1053 | static void __qdisc_destroy(struct Qdisc *qdisc) |
1054 | { |
1055 | const struct Qdisc_ops *ops = qdisc->ops; |
1056 | struct net_device *dev = qdisc_dev(qdisc); |
1057 | |
1058 | #ifdef CONFIG_NET_SCHED |
1059 | qdisc_hash_del(q: qdisc); |
1060 | |
1061 | qdisc_put_stab(rtnl_dereference(qdisc->stab)); |
1062 | #endif |
1063 | gen_kill_estimator(ptr: &qdisc->rate_est); |
1064 | |
1065 | qdisc_reset(qdisc); |
1066 | |
1067 | |
1068 | if (ops->destroy) |
1069 | ops->destroy(qdisc); |
1070 | |
1071 | module_put(module: ops->owner); |
1072 | netdev_put(dev, tracker: &qdisc->dev_tracker); |
1073 | |
1074 | trace_qdisc_destroy(q: qdisc); |
1075 | |
1076 | call_rcu(head: &qdisc->rcu, func: qdisc_free_cb); |
1077 | } |
1078 | |
1079 | void qdisc_destroy(struct Qdisc *qdisc) |
1080 | { |
1081 | if (qdisc->flags & TCQ_F_BUILTIN) |
1082 | return; |
1083 | |
1084 | __qdisc_destroy(qdisc); |
1085 | } |
1086 | |
1087 | void qdisc_put(struct Qdisc *qdisc) |
1088 | { |
1089 | if (!qdisc) |
1090 | return; |
1091 | |
1092 | if (qdisc->flags & TCQ_F_BUILTIN || |
1093 | !refcount_dec_and_test(r: &qdisc->refcnt)) |
1094 | return; |
1095 | |
1096 | __qdisc_destroy(qdisc); |
1097 | } |
1098 | EXPORT_SYMBOL(qdisc_put); |
1099 | |
1100 | /* Version of qdisc_put() that is called with rtnl mutex unlocked. |
1101 | * Intended to be used as optimization, this function only takes rtnl lock if |
1102 | * qdisc reference counter reached zero. |
1103 | */ |
1104 | |
1105 | void qdisc_put_unlocked(struct Qdisc *qdisc) |
1106 | { |
1107 | if (qdisc->flags & TCQ_F_BUILTIN || |
1108 | !refcount_dec_and_rtnl_lock(r: &qdisc->refcnt)) |
1109 | return; |
1110 | |
1111 | __qdisc_destroy(qdisc); |
1112 | rtnl_unlock(); |
1113 | } |
1114 | EXPORT_SYMBOL(qdisc_put_unlocked); |
1115 | |
1116 | /* Attach toplevel qdisc to device queue. */ |
1117 | struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue, |
1118 | struct Qdisc *qdisc) |
1119 | { |
1120 | struct Qdisc *oqdisc = rtnl_dereference(dev_queue->qdisc_sleeping); |
1121 | spinlock_t *root_lock; |
1122 | |
1123 | root_lock = qdisc_lock(qdisc: oqdisc); |
1124 | spin_lock_bh(lock: root_lock); |
1125 | |
1126 | /* ... and graft new one */ |
1127 | if (qdisc == NULL) |
1128 | qdisc = &noop_qdisc; |
1129 | rcu_assign_pointer(dev_queue->qdisc_sleeping, qdisc); |
1130 | rcu_assign_pointer(dev_queue->qdisc, &noop_qdisc); |
1131 | |
1132 | spin_unlock_bh(lock: root_lock); |
1133 | |
1134 | return oqdisc; |
1135 | } |
1136 | EXPORT_SYMBOL(dev_graft_qdisc); |
1137 | |
1138 | static void shutdown_scheduler_queue(struct net_device *dev, |
1139 | struct netdev_queue *dev_queue, |
1140 | void *_qdisc_default) |
1141 | { |
1142 | struct Qdisc *qdisc = rtnl_dereference(dev_queue->qdisc_sleeping); |
1143 | struct Qdisc *qdisc_default = _qdisc_default; |
1144 | |
1145 | if (qdisc) { |
1146 | rcu_assign_pointer(dev_queue->qdisc, qdisc_default); |
1147 | rcu_assign_pointer(dev_queue->qdisc_sleeping, qdisc_default); |
1148 | |
1149 | qdisc_put(qdisc); |
1150 | } |
1151 | } |
1152 | |
1153 | static void attach_one_default_qdisc(struct net_device *dev, |
1154 | struct netdev_queue *dev_queue, |
1155 | void *_unused) |
1156 | { |
1157 | struct Qdisc *qdisc; |
1158 | const struct Qdisc_ops *ops = default_qdisc_ops; |
1159 | |
1160 | if (dev->priv_flags & IFF_NO_QUEUE) |
1161 | ops = &noqueue_qdisc_ops; |
1162 | else if(dev->type == ARPHRD_CAN) |
1163 | ops = &pfifo_fast_ops; |
1164 | |
1165 | qdisc = qdisc_create_dflt(dev_queue, ops, TC_H_ROOT, NULL); |
1166 | if (!qdisc) |
1167 | return; |
1168 | |
1169 | if (!netif_is_multiqueue(dev)) |
1170 | qdisc->flags |= TCQ_F_ONETXQUEUE | TCQ_F_NOPARENT; |
1171 | rcu_assign_pointer(dev_queue->qdisc_sleeping, qdisc); |
1172 | } |
1173 | |
1174 | static void attach_default_qdiscs(struct net_device *dev) |
1175 | { |
1176 | struct netdev_queue *txq; |
1177 | struct Qdisc *qdisc; |
1178 | |
1179 | txq = netdev_get_tx_queue(dev, index: 0); |
1180 | |
1181 | if (!netif_is_multiqueue(dev) || |
1182 | dev->priv_flags & IFF_NO_QUEUE) { |
1183 | netdev_for_each_tx_queue(dev, f: attach_one_default_qdisc, NULL); |
1184 | qdisc = rtnl_dereference(txq->qdisc_sleeping); |
1185 | rcu_assign_pointer(dev->qdisc, qdisc); |
1186 | qdisc_refcount_inc(qdisc); |
1187 | } else { |
1188 | qdisc = qdisc_create_dflt(txq, &mq_qdisc_ops, TC_H_ROOT, NULL); |
1189 | if (qdisc) { |
1190 | rcu_assign_pointer(dev->qdisc, qdisc); |
1191 | qdisc->ops->attach(qdisc); |
1192 | } |
1193 | } |
1194 | qdisc = rtnl_dereference(dev->qdisc); |
1195 | |
1196 | /* Detect default qdisc setup/init failed and fallback to "noqueue" */ |
1197 | if (qdisc == &noop_qdisc) { |
1198 | netdev_warn(dev, format: "default qdisc (%s) fail, fallback to %s\n" , |
1199 | default_qdisc_ops->id, noqueue_qdisc_ops.id); |
1200 | netdev_for_each_tx_queue(dev, f: shutdown_scheduler_queue, arg: &noop_qdisc); |
1201 | dev->priv_flags |= IFF_NO_QUEUE; |
1202 | netdev_for_each_tx_queue(dev, f: attach_one_default_qdisc, NULL); |
1203 | qdisc = rtnl_dereference(txq->qdisc_sleeping); |
1204 | rcu_assign_pointer(dev->qdisc, qdisc); |
1205 | qdisc_refcount_inc(qdisc); |
1206 | dev->priv_flags ^= IFF_NO_QUEUE; |
1207 | } |
1208 | |
1209 | #ifdef CONFIG_NET_SCHED |
1210 | if (qdisc != &noop_qdisc) |
1211 | qdisc_hash_add(q: qdisc, invisible: false); |
1212 | #endif |
1213 | } |
1214 | |
1215 | static void transition_one_qdisc(struct net_device *dev, |
1216 | struct netdev_queue *dev_queue, |
1217 | void *_need_watchdog) |
1218 | { |
1219 | struct Qdisc *new_qdisc = rtnl_dereference(dev_queue->qdisc_sleeping); |
1220 | int *need_watchdog_p = _need_watchdog; |
1221 | |
1222 | if (!(new_qdisc->flags & TCQ_F_BUILTIN)) |
1223 | clear_bit(nr: __QDISC_STATE_DEACTIVATED, addr: &new_qdisc->state); |
1224 | |
1225 | rcu_assign_pointer(dev_queue->qdisc, new_qdisc); |
1226 | if (need_watchdog_p) { |
1227 | WRITE_ONCE(dev_queue->trans_start, 0); |
1228 | *need_watchdog_p = 1; |
1229 | } |
1230 | } |
1231 | |
1232 | void dev_activate(struct net_device *dev) |
1233 | { |
1234 | int need_watchdog; |
1235 | |
1236 | /* No queueing discipline is attached to device; |
1237 | * create default one for devices, which need queueing |
1238 | * and noqueue_qdisc for virtual interfaces |
1239 | */ |
1240 | |
1241 | if (rtnl_dereference(dev->qdisc) == &noop_qdisc) |
1242 | attach_default_qdiscs(dev); |
1243 | |
1244 | if (!netif_carrier_ok(dev)) |
1245 | /* Delay activation until next carrier-on event */ |
1246 | return; |
1247 | |
1248 | need_watchdog = 0; |
1249 | netdev_for_each_tx_queue(dev, f: transition_one_qdisc, arg: &need_watchdog); |
1250 | if (dev_ingress_queue(dev)) |
1251 | transition_one_qdisc(dev, dev_queue: dev_ingress_queue(dev), NULL); |
1252 | |
1253 | if (need_watchdog) { |
1254 | netif_trans_update(dev); |
1255 | dev_watchdog_up(dev); |
1256 | } |
1257 | } |
1258 | EXPORT_SYMBOL(dev_activate); |
1259 | |
1260 | static void qdisc_deactivate(struct Qdisc *qdisc) |
1261 | { |
1262 | if (qdisc->flags & TCQ_F_BUILTIN) |
1263 | return; |
1264 | |
1265 | set_bit(nr: __QDISC_STATE_DEACTIVATED, addr: &qdisc->state); |
1266 | } |
1267 | |
1268 | static void dev_deactivate_queue(struct net_device *dev, |
1269 | struct netdev_queue *dev_queue, |
1270 | void *_qdisc_default) |
1271 | { |
1272 | struct Qdisc *qdisc_default = _qdisc_default; |
1273 | struct Qdisc *qdisc; |
1274 | |
1275 | qdisc = rtnl_dereference(dev_queue->qdisc); |
1276 | if (qdisc) { |
1277 | qdisc_deactivate(qdisc); |
1278 | rcu_assign_pointer(dev_queue->qdisc, qdisc_default); |
1279 | } |
1280 | } |
1281 | |
1282 | static void dev_reset_queue(struct net_device *dev, |
1283 | struct netdev_queue *dev_queue, |
1284 | void *_unused) |
1285 | { |
1286 | struct Qdisc *qdisc; |
1287 | bool nolock; |
1288 | |
1289 | qdisc = rtnl_dereference(dev_queue->qdisc_sleeping); |
1290 | if (!qdisc) |
1291 | return; |
1292 | |
1293 | nolock = qdisc->flags & TCQ_F_NOLOCK; |
1294 | |
1295 | if (nolock) |
1296 | spin_lock_bh(lock: &qdisc->seqlock); |
1297 | spin_lock_bh(lock: qdisc_lock(qdisc)); |
1298 | |
1299 | qdisc_reset(qdisc); |
1300 | |
1301 | spin_unlock_bh(lock: qdisc_lock(qdisc)); |
1302 | if (nolock) { |
1303 | clear_bit(nr: __QDISC_STATE_MISSED, addr: &qdisc->state); |
1304 | clear_bit(nr: __QDISC_STATE_DRAINING, addr: &qdisc->state); |
1305 | spin_unlock_bh(lock: &qdisc->seqlock); |
1306 | } |
1307 | } |
1308 | |
1309 | static bool some_qdisc_is_busy(struct net_device *dev) |
1310 | { |
1311 | unsigned int i; |
1312 | |
1313 | for (i = 0; i < dev->num_tx_queues; i++) { |
1314 | struct netdev_queue *dev_queue; |
1315 | spinlock_t *root_lock; |
1316 | struct Qdisc *q; |
1317 | int val; |
1318 | |
1319 | dev_queue = netdev_get_tx_queue(dev, index: i); |
1320 | q = rtnl_dereference(dev_queue->qdisc_sleeping); |
1321 | |
1322 | root_lock = qdisc_lock(qdisc: q); |
1323 | spin_lock_bh(lock: root_lock); |
1324 | |
1325 | val = (qdisc_is_running(qdisc: q) || |
1326 | test_bit(__QDISC_STATE_SCHED, &q->state)); |
1327 | |
1328 | spin_unlock_bh(lock: root_lock); |
1329 | |
1330 | if (val) |
1331 | return true; |
1332 | } |
1333 | return false; |
1334 | } |
1335 | |
1336 | /** |
1337 | * dev_deactivate_many - deactivate transmissions on several devices |
1338 | * @head: list of devices to deactivate |
1339 | * |
1340 | * This function returns only when all outstanding transmissions |
1341 | * have completed, unless all devices are in dismantle phase. |
1342 | */ |
1343 | void dev_deactivate_many(struct list_head *head) |
1344 | { |
1345 | struct net_device *dev; |
1346 | |
1347 | list_for_each_entry(dev, head, close_list) { |
1348 | netdev_for_each_tx_queue(dev, f: dev_deactivate_queue, |
1349 | arg: &noop_qdisc); |
1350 | if (dev_ingress_queue(dev)) |
1351 | dev_deactivate_queue(dev, dev_queue: dev_ingress_queue(dev), |
1352 | qdisc_default: &noop_qdisc); |
1353 | |
1354 | dev_watchdog_down(dev); |
1355 | } |
1356 | |
1357 | /* Wait for outstanding qdisc-less dev_queue_xmit calls or |
1358 | * outstanding qdisc enqueuing calls. |
1359 | * This is avoided if all devices are in dismantle phase : |
1360 | * Caller will call synchronize_net() for us |
1361 | */ |
1362 | synchronize_net(); |
1363 | |
1364 | list_for_each_entry(dev, head, close_list) { |
1365 | netdev_for_each_tx_queue(dev, f: dev_reset_queue, NULL); |
1366 | |
1367 | if (dev_ingress_queue(dev)) |
1368 | dev_reset_queue(dev, dev_queue: dev_ingress_queue(dev), NULL); |
1369 | } |
1370 | |
1371 | /* Wait for outstanding qdisc_run calls. */ |
1372 | list_for_each_entry(dev, head, close_list) { |
1373 | while (some_qdisc_is_busy(dev)) { |
1374 | /* wait_event() would avoid this sleep-loop but would |
1375 | * require expensive checks in the fast paths of packet |
1376 | * processing which isn't worth it. |
1377 | */ |
1378 | schedule_timeout_uninterruptible(timeout: 1); |
1379 | } |
1380 | } |
1381 | } |
1382 | |
1383 | void dev_deactivate(struct net_device *dev) |
1384 | { |
1385 | LIST_HEAD(single); |
1386 | |
1387 | list_add(new: &dev->close_list, head: &single); |
1388 | dev_deactivate_many(head: &single); |
1389 | list_del(entry: &single); |
1390 | } |
1391 | EXPORT_SYMBOL(dev_deactivate); |
1392 | |
1393 | static int qdisc_change_tx_queue_len(struct net_device *dev, |
1394 | struct netdev_queue *dev_queue) |
1395 | { |
1396 | struct Qdisc *qdisc = rtnl_dereference(dev_queue->qdisc_sleeping); |
1397 | const struct Qdisc_ops *ops = qdisc->ops; |
1398 | |
1399 | if (ops->change_tx_queue_len) |
1400 | return ops->change_tx_queue_len(qdisc, dev->tx_queue_len); |
1401 | return 0; |
1402 | } |
1403 | |
1404 | void dev_qdisc_change_real_num_tx(struct net_device *dev, |
1405 | unsigned int new_real_tx) |
1406 | { |
1407 | struct Qdisc *qdisc = rtnl_dereference(dev->qdisc); |
1408 | |
1409 | if (qdisc->ops->change_real_num_tx) |
1410 | qdisc->ops->change_real_num_tx(qdisc, new_real_tx); |
1411 | } |
1412 | |
1413 | void mq_change_real_num_tx(struct Qdisc *sch, unsigned int new_real_tx) |
1414 | { |
1415 | #ifdef CONFIG_NET_SCHED |
1416 | struct net_device *dev = qdisc_dev(qdisc: sch); |
1417 | struct Qdisc *qdisc; |
1418 | unsigned int i; |
1419 | |
1420 | for (i = new_real_tx; i < dev->real_num_tx_queues; i++) { |
1421 | qdisc = rtnl_dereference(netdev_get_tx_queue(dev, i)->qdisc_sleeping); |
1422 | /* Only update the default qdiscs we created, |
1423 | * qdiscs with handles are always hashed. |
1424 | */ |
1425 | if (qdisc != &noop_qdisc && !qdisc->handle) |
1426 | qdisc_hash_del(q: qdisc); |
1427 | } |
1428 | for (i = dev->real_num_tx_queues; i < new_real_tx; i++) { |
1429 | qdisc = rtnl_dereference(netdev_get_tx_queue(dev, i)->qdisc_sleeping); |
1430 | if (qdisc != &noop_qdisc && !qdisc->handle) |
1431 | qdisc_hash_add(q: qdisc, invisible: false); |
1432 | } |
1433 | #endif |
1434 | } |
1435 | EXPORT_SYMBOL(mq_change_real_num_tx); |
1436 | |
1437 | int dev_qdisc_change_tx_queue_len(struct net_device *dev) |
1438 | { |
1439 | bool up = dev->flags & IFF_UP; |
1440 | unsigned int i; |
1441 | int ret = 0; |
1442 | |
1443 | if (up) |
1444 | dev_deactivate(dev); |
1445 | |
1446 | for (i = 0; i < dev->num_tx_queues; i++) { |
1447 | ret = qdisc_change_tx_queue_len(dev, dev_queue: &dev->_tx[i]); |
1448 | |
1449 | /* TODO: revert changes on a partial failure */ |
1450 | if (ret) |
1451 | break; |
1452 | } |
1453 | |
1454 | if (up) |
1455 | dev_activate(dev); |
1456 | return ret; |
1457 | } |
1458 | |
1459 | static void dev_init_scheduler_queue(struct net_device *dev, |
1460 | struct netdev_queue *dev_queue, |
1461 | void *_qdisc) |
1462 | { |
1463 | struct Qdisc *qdisc = _qdisc; |
1464 | |
1465 | rcu_assign_pointer(dev_queue->qdisc, qdisc); |
1466 | rcu_assign_pointer(dev_queue->qdisc_sleeping, qdisc); |
1467 | } |
1468 | |
1469 | void dev_init_scheduler(struct net_device *dev) |
1470 | { |
1471 | rcu_assign_pointer(dev->qdisc, &noop_qdisc); |
1472 | netdev_for_each_tx_queue(dev, f: dev_init_scheduler_queue, arg: &noop_qdisc); |
1473 | if (dev_ingress_queue(dev)) |
1474 | dev_init_scheduler_queue(dev, dev_queue: dev_ingress_queue(dev), qdisc: &noop_qdisc); |
1475 | |
1476 | timer_setup(&dev->watchdog_timer, dev_watchdog, 0); |
1477 | } |
1478 | |
1479 | void dev_shutdown(struct net_device *dev) |
1480 | { |
1481 | netdev_for_each_tx_queue(dev, f: shutdown_scheduler_queue, arg: &noop_qdisc); |
1482 | if (dev_ingress_queue(dev)) |
1483 | shutdown_scheduler_queue(dev, dev_queue: dev_ingress_queue(dev), qdisc_default: &noop_qdisc); |
1484 | qdisc_put(rtnl_dereference(dev->qdisc)); |
1485 | rcu_assign_pointer(dev->qdisc, &noop_qdisc); |
1486 | |
1487 | WARN_ON(timer_pending(&dev->watchdog_timer)); |
1488 | } |
1489 | |
1490 | /** |
1491 | * psched_ratecfg_precompute__() - Pre-compute values for reciprocal division |
1492 | * @rate: Rate to compute reciprocal division values of |
1493 | * @mult: Multiplier for reciprocal division |
1494 | * @shift: Shift for reciprocal division |
1495 | * |
1496 | * The multiplier and shift for reciprocal division by rate are stored |
1497 | * in mult and shift. |
1498 | * |
1499 | * The deal here is to replace a divide by a reciprocal one |
1500 | * in fast path (a reciprocal divide is a multiply and a shift) |
1501 | * |
1502 | * Normal formula would be : |
1503 | * time_in_ns = (NSEC_PER_SEC * len) / rate_bps |
1504 | * |
1505 | * We compute mult/shift to use instead : |
1506 | * time_in_ns = (len * mult) >> shift; |
1507 | * |
1508 | * We try to get the highest possible mult value for accuracy, |
1509 | * but have to make sure no overflows will ever happen. |
1510 | * |
1511 | * reciprocal_value() is not used here it doesn't handle 64-bit values. |
1512 | */ |
1513 | static void psched_ratecfg_precompute__(u64 rate, u32 *mult, u8 *shift) |
1514 | { |
1515 | u64 factor = NSEC_PER_SEC; |
1516 | |
1517 | *mult = 1; |
1518 | *shift = 0; |
1519 | |
1520 | if (rate <= 0) |
1521 | return; |
1522 | |
1523 | for (;;) { |
1524 | *mult = div64_u64(dividend: factor, divisor: rate); |
1525 | if (*mult & (1U << 31) || factor & (1ULL << 63)) |
1526 | break; |
1527 | factor <<= 1; |
1528 | (*shift)++; |
1529 | } |
1530 | } |
1531 | |
1532 | void psched_ratecfg_precompute(struct psched_ratecfg *r, |
1533 | const struct tc_ratespec *conf, |
1534 | u64 rate64) |
1535 | { |
1536 | memset(r, 0, sizeof(*r)); |
1537 | r->overhead = conf->overhead; |
1538 | r->mpu = conf->mpu; |
1539 | r->rate_bytes_ps = max_t(u64, conf->rate, rate64); |
1540 | r->linklayer = (conf->linklayer & TC_LINKLAYER_MASK); |
1541 | psched_ratecfg_precompute__(rate: r->rate_bytes_ps, mult: &r->mult, shift: &r->shift); |
1542 | } |
1543 | EXPORT_SYMBOL(psched_ratecfg_precompute); |
1544 | |
1545 | void psched_ppscfg_precompute(struct psched_pktrate *r, u64 pktrate64) |
1546 | { |
1547 | r->rate_pkts_ps = pktrate64; |
1548 | psched_ratecfg_precompute__(rate: r->rate_pkts_ps, mult: &r->mult, shift: &r->shift); |
1549 | } |
1550 | EXPORT_SYMBOL(psched_ppscfg_precompute); |
1551 | |
1552 | void mini_qdisc_pair_swap(struct mini_Qdisc_pair *miniqp, |
1553 | struct tcf_proto *tp_head) |
1554 | { |
1555 | /* Protected with chain0->filter_chain_lock. |
1556 | * Can't access chain directly because tp_head can be NULL. |
1557 | */ |
1558 | struct mini_Qdisc *miniq_old = |
1559 | rcu_dereference_protected(*miniqp->p_miniq, 1); |
1560 | struct mini_Qdisc *miniq; |
1561 | |
1562 | if (!tp_head) { |
1563 | RCU_INIT_POINTER(*miniqp->p_miniq, NULL); |
1564 | } else { |
1565 | miniq = miniq_old != &miniqp->miniq1 ? |
1566 | &miniqp->miniq1 : &miniqp->miniq2; |
1567 | |
1568 | /* We need to make sure that readers won't see the miniq |
1569 | * we are about to modify. So ensure that at least one RCU |
1570 | * grace period has elapsed since the miniq was made |
1571 | * inactive. |
1572 | */ |
1573 | if (IS_ENABLED(CONFIG_PREEMPT_RT)) |
1574 | cond_synchronize_rcu(oldstate: miniq->rcu_state); |
1575 | else if (!poll_state_synchronize_rcu(oldstate: miniq->rcu_state)) |
1576 | synchronize_rcu_expedited(); |
1577 | |
1578 | miniq->filter_list = tp_head; |
1579 | rcu_assign_pointer(*miniqp->p_miniq, miniq); |
1580 | } |
1581 | |
1582 | if (miniq_old) |
1583 | /* This is counterpart of the rcu sync above. We need to |
1584 | * block potential new user of miniq_old until all readers |
1585 | * are not seeing it. |
1586 | */ |
1587 | miniq_old->rcu_state = start_poll_synchronize_rcu(); |
1588 | } |
1589 | EXPORT_SYMBOL(mini_qdisc_pair_swap); |
1590 | |
1591 | void mini_qdisc_pair_block_init(struct mini_Qdisc_pair *miniqp, |
1592 | struct tcf_block *block) |
1593 | { |
1594 | miniqp->miniq1.block = block; |
1595 | miniqp->miniq2.block = block; |
1596 | } |
1597 | EXPORT_SYMBOL(mini_qdisc_pair_block_init); |
1598 | |
1599 | void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc, |
1600 | struct mini_Qdisc __rcu **p_miniq) |
1601 | { |
1602 | miniqp->miniq1.cpu_bstats = qdisc->cpu_bstats; |
1603 | miniqp->miniq1.cpu_qstats = qdisc->cpu_qstats; |
1604 | miniqp->miniq2.cpu_bstats = qdisc->cpu_bstats; |
1605 | miniqp->miniq2.cpu_qstats = qdisc->cpu_qstats; |
1606 | miniqp->miniq1.rcu_state = get_state_synchronize_rcu(); |
1607 | miniqp->miniq2.rcu_state = miniqp->miniq1.rcu_state; |
1608 | miniqp->p_miniq = p_miniq; |
1609 | } |
1610 | EXPORT_SYMBOL(mini_qdisc_pair_init); |
1611 | |