1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * Definitions for the 'struct sk_buff' memory handlers.
4 *
5 * Authors:
6 * Alan Cox, <gw4pts@gw4pts.ampr.org>
7 * Florian La Roche, <rzsfl@rz.uni-sb.de>
8 */
9
10#ifndef _LINUX_SKBUFF_H
11#define _LINUX_SKBUFF_H
12
13#include <linux/kernel.h>
14#include <linux/compiler.h>
15#include <linux/time.h>
16#include <linux/bug.h>
17#include <linux/bvec.h>
18#include <linux/cache.h>
19#include <linux/rbtree.h>
20#include <linux/socket.h>
21#include <linux/refcount.h>
22
23#include <linux/atomic.h>
24#include <asm/types.h>
25#include <linux/spinlock.h>
26#include <net/checksum.h>
27#include <linux/rcupdate.h>
28#include <linux/dma-mapping.h>
29#include <linux/netdev_features.h>
30#include <net/flow_dissector.h>
31#include <linux/in6.h>
32#include <linux/if_packet.h>
33#include <linux/llist.h>
34#include <linux/page_frag_cache.h>
35#include <net/flow.h>
36#if IS_ENABLED(CONFIG_NF_CONNTRACK)
37#include <linux/netfilter/nf_conntrack_common.h>
38#endif
39#include <net/net_debug.h>
40#include <net/dropreason-core.h>
41#include <net/netmem.h>
42
43/**
44 * DOC: skb checksums
45 *
46 * The interface for checksum offload between the stack and networking drivers
47 * is as follows...
48 *
49 * IP checksum related features
50 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
51 *
52 * Drivers advertise checksum offload capabilities in the features of a device.
53 * From the stack's point of view these are capabilities offered by the driver.
54 * A driver typically only advertises features that it is capable of offloading
55 * to its device.
56 *
57 * .. flat-table:: Checksum related device features
58 * :widths: 1 10
59 *
60 * * - %NETIF_F_HW_CSUM
61 * - The driver (or its device) is able to compute one
62 * IP (one's complement) checksum for any combination
63 * of protocols or protocol layering. The checksum is
64 * computed and set in a packet per the CHECKSUM_PARTIAL
65 * interface (see below).
66 *
67 * * - %NETIF_F_IP_CSUM
68 * - Driver (device) is only able to checksum plain
69 * TCP or UDP packets over IPv4. These are specifically
70 * unencapsulated packets of the form IPv4|TCP or
71 * IPv4|UDP where the Protocol field in the IPv4 header
72 * is TCP or UDP. The IPv4 header may contain IP options.
73 * This feature cannot be set in features for a device
74 * with NETIF_F_HW_CSUM also set. This feature is being
75 * DEPRECATED (see below).
76 *
77 * * - %NETIF_F_IPV6_CSUM
78 * - Driver (device) is only able to checksum plain
79 * TCP or UDP packets over IPv6. These are specifically
80 * unencapsulated packets of the form IPv6|TCP or
81 * IPv6|UDP where the Next Header field in the IPv6
82 * header is either TCP or UDP. IPv6 extension headers
83 * are not supported with this feature. This feature
84 * cannot be set in features for a device with
85 * NETIF_F_HW_CSUM also set. This feature is being
86 * DEPRECATED (see below).
87 *
88 * * - %NETIF_F_RXCSUM
89 * - Driver (device) performs receive checksum offload.
90 * This flag is only used to disable the RX checksum
91 * feature for a device. The stack will accept receive
92 * checksum indication in packets received on a device
93 * regardless of whether NETIF_F_RXCSUM is set.
94 *
95 * Checksumming of received packets by device
96 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
97 *
98 * Indication of checksum verification is set in &sk_buff.ip_summed.
99 * Possible values are:
100 *
101 * - %CHECKSUM_NONE
102 *
103 * Device did not checksum this packet e.g. due to lack of capabilities.
104 * The packet contains full (though not verified) checksum in packet but
105 * not in skb->csum. Thus, skb->csum is undefined in this case.
106 *
107 * - %CHECKSUM_UNNECESSARY
108 *
109 * The hardware you're dealing with doesn't calculate the full checksum
110 * (as in %CHECKSUM_COMPLETE), but it does parse headers and verify checksums
111 * for specific protocols. For such packets it will set %CHECKSUM_UNNECESSARY
112 * if their checksums are okay. &sk_buff.csum is still undefined in this case
113 * though. A driver or device must never modify the checksum field in the
114 * packet even if checksum is verified.
115 *
116 * %CHECKSUM_UNNECESSARY is applicable to following protocols:
117 *
118 * - TCP: IPv6 and IPv4.
119 * - UDP: IPv4 and IPv6. A device may apply CHECKSUM_UNNECESSARY to a
120 * zero UDP checksum for either IPv4 or IPv6, the networking stack
121 * may perform further validation in this case.
122 * - GRE: only if the checksum is present in the header.
123 * - SCTP: indicates the CRC in SCTP header has been validated.
124 * - FCOE: indicates the CRC in FC frame has been validated.
125 *
126 * &sk_buff.csum_level indicates the number of consecutive checksums found in
127 * the packet minus one that have been verified as %CHECKSUM_UNNECESSARY.
128 * For instance if a device receives an IPv6->UDP->GRE->IPv4->TCP packet
129 * and a device is able to verify the checksums for UDP (possibly zero),
130 * GRE (checksum flag is set) and TCP, &sk_buff.csum_level would be set to
131 * two. If the device were only able to verify the UDP checksum and not
132 * GRE, either because it doesn't support GRE checksum or because GRE
133 * checksum is bad, skb->csum_level would be set to zero (TCP checksum is
134 * not considered in this case).
135 *
136 * - %CHECKSUM_COMPLETE
137 *
138 * This is the most generic way. The device supplied checksum of the _whole_
139 * packet as seen by netif_rx() and fills in &sk_buff.csum. This means the
140 * hardware doesn't need to parse L3/L4 headers to implement this.
141 *
142 * Notes:
143 *
144 * - Even if device supports only some protocols, but is able to produce
145 * skb->csum, it MUST use CHECKSUM_COMPLETE, not CHECKSUM_UNNECESSARY.
146 * - CHECKSUM_COMPLETE is not applicable to SCTP and FCoE protocols.
147 *
148 * - %CHECKSUM_PARTIAL
149 *
150 * A checksum is set up to be offloaded to a device as described in the
151 * output description for CHECKSUM_PARTIAL. This may occur on a packet
152 * received directly from another Linux OS, e.g., a virtualized Linux kernel
153 * on the same host, or it may be set in the input path in GRO or remote
154 * checksum offload. For the purposes of checksum verification, the checksum
155 * referred to by skb->csum_start + skb->csum_offset and any preceding
156 * checksums in the packet are considered verified. Any checksums in the
157 * packet that are after the checksum being offloaded are not considered to
158 * be verified.
159 *
160 * Checksumming on transmit for non-GSO
161 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
162 *
163 * The stack requests checksum offload in the &sk_buff.ip_summed for a packet.
164 * Values are:
165 *
166 * - %CHECKSUM_PARTIAL
167 *
168 * The driver is required to checksum the packet as seen by hard_start_xmit()
169 * from &sk_buff.csum_start up to the end, and to record/write the checksum at
170 * offset &sk_buff.csum_start + &sk_buff.csum_offset.
171 * A driver may verify that the
172 * csum_start and csum_offset values are valid values given the length and
173 * offset of the packet, but it should not attempt to validate that the
174 * checksum refers to a legitimate transport layer checksum -- it is the
175 * purview of the stack to validate that csum_start and csum_offset are set
176 * correctly.
177 *
178 * When the stack requests checksum offload for a packet, the driver MUST
179 * ensure that the checksum is set correctly. A driver can either offload the
180 * checksum calculation to the device, or call skb_checksum_help (in the case
181 * that the device does not support offload for a particular checksum).
182 *
183 * %NETIF_F_IP_CSUM and %NETIF_F_IPV6_CSUM are being deprecated in favor of
184 * %NETIF_F_HW_CSUM. New devices should use %NETIF_F_HW_CSUM to indicate
185 * checksum offload capability.
186 * skb_csum_hwoffload_help() can be called to resolve %CHECKSUM_PARTIAL based
187 * on network device checksumming capabilities: if a packet does not match
188 * them, skb_checksum_help() or skb_crc32c_help() (depending on the value of
189 * &sk_buff.csum_not_inet, see :ref:`crc`)
190 * is called to resolve the checksum.
191 *
192 * - %CHECKSUM_NONE
193 *
194 * The skb was already checksummed by the protocol, or a checksum is not
195 * required.
196 *
197 * - %CHECKSUM_UNNECESSARY
198 *
199 * This has the same meaning as CHECKSUM_NONE for checksum offload on
200 * output.
201 *
202 * - %CHECKSUM_COMPLETE
203 *
204 * Not used in checksum output. If a driver observes a packet with this value
205 * set in skbuff, it should treat the packet as if %CHECKSUM_NONE were set.
206 *
207 * .. _crc:
208 *
209 * Non-IP checksum (CRC) offloads
210 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
211 *
212 * .. flat-table::
213 * :widths: 1 10
214 *
215 * * - %NETIF_F_SCTP_CRC
216 * - This feature indicates that a device is capable of
217 * offloading the SCTP CRC in a packet. To perform this offload the stack
218 * will set csum_start and csum_offset accordingly, set ip_summed to
219 * %CHECKSUM_PARTIAL and set csum_not_inet to 1, to provide an indication
220 * in the skbuff that the %CHECKSUM_PARTIAL refers to CRC32c.
221 * A driver that supports both IP checksum offload and SCTP CRC32c offload
222 * must verify which offload is configured for a packet by testing the
223 * value of &sk_buff.csum_not_inet; skb_crc32c_csum_help() is provided to
224 * resolve %CHECKSUM_PARTIAL on skbs where csum_not_inet is set to 1.
225 *
226 * * - %NETIF_F_FCOE_CRC
227 * - This feature indicates that a device is capable of offloading the FCOE
228 * CRC in a packet. To perform this offload the stack will set ip_summed
229 * to %CHECKSUM_PARTIAL and set csum_start and csum_offset
230 * accordingly. Note that there is no indication in the skbuff that the
231 * %CHECKSUM_PARTIAL refers to an FCOE checksum, so a driver that supports
232 * both IP checksum offload and FCOE CRC offload must verify which offload
233 * is configured for a packet, presumably by inspecting packet headers.
234 *
235 * Checksumming on output with GSO
236 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
237 *
238 * In the case of a GSO packet (skb_is_gso() is true), checksum offload
239 * is implied by the SKB_GSO_* flags in gso_type. Most obviously, if the
240 * gso_type is %SKB_GSO_TCPV4 or %SKB_GSO_TCPV6, TCP checksum offload as
241 * part of the GSO operation is implied. If a checksum is being offloaded
242 * with GSO then ip_summed is %CHECKSUM_PARTIAL, and both csum_start and
243 * csum_offset are set to refer to the outermost checksum being offloaded
244 * (two offloaded checksums are possible with UDP encapsulation).
245 */
246
247/* Don't change this without changing skb_csum_unnecessary! */
248#define CHECKSUM_NONE 0
249#define CHECKSUM_UNNECESSARY 1
250#define CHECKSUM_COMPLETE 2
251#define CHECKSUM_PARTIAL 3
252
253/* Maximum value in skb->csum_level */
254#define SKB_MAX_CSUM_LEVEL 3
255
256#define SKB_DATA_ALIGN(X) ALIGN(X, SMP_CACHE_BYTES)
257#define SKB_WITH_OVERHEAD(X) \
258 ((X) - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
259
260/* For X bytes available in skb->head, what is the minimal
261 * allocation needed, knowing struct skb_shared_info needs
262 * to be aligned.
263 */
264#define SKB_HEAD_ALIGN(X) (SKB_DATA_ALIGN(X) + \
265 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
266
267#define SKB_MAX_ORDER(X, ORDER) \
268 SKB_WITH_OVERHEAD((PAGE_SIZE << (ORDER)) - (X))
269#define SKB_MAX_HEAD(X) (SKB_MAX_ORDER((X), 0))
270#define SKB_MAX_ALLOC (SKB_MAX_ORDER(0, 2))
271
272/* return minimum truesize of one skb containing X bytes of data */
273#define SKB_TRUESIZE(X) ((X) + \
274 SKB_DATA_ALIGN(sizeof(struct sk_buff)) + \
275 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
276
277struct net_device;
278struct scatterlist;
279struct pipe_inode_info;
280struct iov_iter;
281struct napi_struct;
282struct bpf_prog;
283union bpf_attr;
284struct skb_ext;
285struct ts_config;
286
287#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
288struct nf_bridge_info {
289 enum {
290 BRNF_PROTO_UNCHANGED,
291 BRNF_PROTO_8021Q,
292 BRNF_PROTO_PPPOE
293 } orig_proto:8;
294 u8 pkt_otherhost:1;
295 u8 in_prerouting:1;
296 u8 bridged_dnat:1;
297 u8 sabotage_in_done:1;
298 __u16 frag_max_size;
299 int physinif;
300
301 /* always valid & non-NULL from FORWARD on, for physdev match */
302 struct net_device *physoutdev;
303 union {
304 /* prerouting: detect dnat in orig/reply direction */
305 __be32 ipv4_daddr;
306 struct in6_addr ipv6_daddr;
307
308 /* after prerouting + nat detected: store original source
309 * mac since neigh resolution overwrites it, only used while
310 * skb is out in neigh layer.
311 */
312 char neigh_header[8];
313 };
314};
315#endif
316
317#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
318/* Chain in tc_skb_ext will be used to share the tc chain with
319 * ovs recirc_id. It will be set to the current chain by tc
320 * and read by ovs to recirc_id.
321 */
322struct tc_skb_ext {
323 union {
324 u64 act_miss_cookie;
325 __u32 chain;
326 };
327 __u16 mru;
328 __u16 zone;
329 u8 post_ct:1;
330 u8 post_ct_snat:1;
331 u8 post_ct_dnat:1;
332 u8 act_miss:1; /* Set if act_miss_cookie is used */
333 u8 l2_miss:1; /* Set by bridge upon FDB or MDB miss */
334};
335#endif
336
337struct sk_buff_head {
338 /* These two members must be first to match sk_buff. */
339 struct_group_tagged(sk_buff_list, list,
340 struct sk_buff *next;
341 struct sk_buff *prev;
342 );
343
344 __u32 qlen;
345 spinlock_t lock;
346};
347
348struct sk_buff;
349
350#ifndef CONFIG_MAX_SKB_FRAGS
351# define CONFIG_MAX_SKB_FRAGS 17
352#endif
353
354#define MAX_SKB_FRAGS CONFIG_MAX_SKB_FRAGS
355
356/* Set skb_shinfo(skb)->gso_size to this in case you want skb_segment to
357 * segment using its current segmentation instead.
358 */
359#define GSO_BY_FRAGS 0xFFFF
360
361typedef struct skb_frag {
362 netmem_ref netmem;
363 unsigned int len;
364 unsigned int offset;
365} skb_frag_t;
366
367/**
368 * skb_frag_size() - Returns the size of a skb fragment
369 * @frag: skb fragment
370 */
371static inline unsigned int skb_frag_size(const skb_frag_t *frag)
372{
373 return frag->len;
374}
375
376/**
377 * skb_frag_size_set() - Sets the size of a skb fragment
378 * @frag: skb fragment
379 * @size: size of fragment
380 */
381static inline void skb_frag_size_set(skb_frag_t *frag, unsigned int size)
382{
383 frag->len = size;
384}
385
386/**
387 * skb_frag_size_add() - Increments the size of a skb fragment by @delta
388 * @frag: skb fragment
389 * @delta: value to add
390 */
391static inline void skb_frag_size_add(skb_frag_t *frag, int delta)
392{
393 frag->len += delta;
394}
395
396/**
397 * skb_frag_size_sub() - Decrements the size of a skb fragment by @delta
398 * @frag: skb fragment
399 * @delta: value to subtract
400 */
401static inline void skb_frag_size_sub(skb_frag_t *frag, int delta)
402{
403 frag->len -= delta;
404}
405
406/**
407 * skb_frag_must_loop - Test if %p is a high memory page
408 * @p: fragment's page
409 */
410static inline bool skb_frag_must_loop(struct page *p)
411{
412#if defined(CONFIG_HIGHMEM)
413 if (IS_ENABLED(CONFIG_DEBUG_KMAP_LOCAL_FORCE_MAP) || PageHighMem(p))
414 return true;
415#endif
416 return false;
417}
418
419/**
420 * skb_frag_foreach_page - loop over pages in a fragment
421 *
422 * @f: skb frag to operate on
423 * @f_off: offset from start of f->netmem
424 * @f_len: length from f_off to loop over
425 * @p: (temp var) current page
426 * @p_off: (temp var) offset from start of current page,
427 * non-zero only on first page.
428 * @p_len: (temp var) length in current page,
429 * < PAGE_SIZE only on first and last page.
430 * @copied: (temp var) length so far, excluding current p_len.
431 *
432 * A fragment can hold a compound page, in which case per-page
433 * operations, notably kmap_atomic, must be called for each
434 * regular page.
435 */
436#define skb_frag_foreach_page(f, f_off, f_len, p, p_off, p_len, copied) \
437 for (p = skb_frag_page(f) + ((f_off) >> PAGE_SHIFT), \
438 p_off = (f_off) & (PAGE_SIZE - 1), \
439 p_len = skb_frag_must_loop(p) ? \
440 min_t(u32, f_len, PAGE_SIZE - p_off) : f_len, \
441 copied = 0; \
442 copied < f_len; \
443 copied += p_len, p++, p_off = 0, \
444 p_len = min_t(u32, f_len - copied, PAGE_SIZE)) \
445
446/**
447 * struct skb_shared_hwtstamps - hardware time stamps
448 * @hwtstamp: hardware time stamp transformed into duration
449 * since arbitrary point in time
450 * @netdev_data: address/cookie of network device driver used as
451 * reference to actual hardware time stamp
452 *
453 * Software time stamps generated by ktime_get_real() are stored in
454 * skb->tstamp.
455 *
456 * hwtstamps can only be compared against other hwtstamps from
457 * the same device.
458 *
459 * This structure is attached to packets as part of the
460 * &skb_shared_info. Use skb_hwtstamps() to get a pointer.
461 */
462struct skb_shared_hwtstamps {
463 union {
464 ktime_t hwtstamp;
465 void *netdev_data;
466 };
467};
468
469/* Definitions for tx_flags in struct skb_shared_info */
470enum {
471 /* generate hardware time stamp */
472 SKBTX_HW_TSTAMP_NOBPF = 1 << 0,
473
474 /* generate software time stamp when queueing packet to NIC */
475 SKBTX_SW_TSTAMP = 1 << 1,
476
477 /* device driver is going to provide hardware time stamp */
478 SKBTX_IN_PROGRESS = 1 << 2,
479
480 /* generate software time stamp on packet tx completion */
481 SKBTX_COMPLETION_TSTAMP = 1 << 3,
482
483 /* determine hardware time stamp based on time or cycles */
484 SKBTX_HW_TSTAMP_NETDEV = 1 << 5,
485
486 /* generate software time stamp when entering packet scheduling */
487 SKBTX_SCHED_TSTAMP = 1 << 6,
488
489 /* used for bpf extension when a bpf program is loaded */
490 SKBTX_BPF = 1 << 7,
491};
492
493#define SKBTX_HW_TSTAMP (SKBTX_HW_TSTAMP_NOBPF | SKBTX_BPF)
494
495#define SKBTX_ANY_SW_TSTAMP (SKBTX_SW_TSTAMP | \
496 SKBTX_SCHED_TSTAMP | \
497 SKBTX_BPF | \
498 SKBTX_COMPLETION_TSTAMP)
499#define SKBTX_ANY_TSTAMP (SKBTX_HW_TSTAMP | \
500 SKBTX_ANY_SW_TSTAMP)
501
502/* Definitions for flags in struct skb_shared_info */
503enum {
504 /* use zcopy routines */
505 SKBFL_ZEROCOPY_ENABLE = BIT(0),
506
507 /* This indicates at least one fragment might be overwritten
508 * (as in vmsplice(), sendfile() ...)
509 * If we need to compute a TX checksum, we'll need to copy
510 * all frags to avoid possible bad checksum
511 */
512 SKBFL_SHARED_FRAG = BIT(1),
513
514 /* segment contains only zerocopy data and should not be
515 * charged to the kernel memory.
516 */
517 SKBFL_PURE_ZEROCOPY = BIT(2),
518
519 SKBFL_DONT_ORPHAN = BIT(3),
520
521 /* page references are managed by the ubuf_info, so it's safe to
522 * use frags only up until ubuf_info is released
523 */
524 SKBFL_MANAGED_FRAG_REFS = BIT(4),
525};
526
527#define SKBFL_ZEROCOPY_FRAG (SKBFL_ZEROCOPY_ENABLE | SKBFL_SHARED_FRAG)
528#define SKBFL_ALL_ZEROCOPY (SKBFL_ZEROCOPY_FRAG | SKBFL_PURE_ZEROCOPY | \
529 SKBFL_DONT_ORPHAN | SKBFL_MANAGED_FRAG_REFS)
530
531struct ubuf_info_ops {
532 void (*complete)(struct sk_buff *, struct ubuf_info *,
533 bool zerocopy_success);
534 /* has to be compatible with skb_zcopy_set() */
535 int (*link_skb)(struct sk_buff *skb, struct ubuf_info *uarg);
536};
537
538/*
539 * The callback notifies userspace to release buffers when skb DMA is done in
540 * lower device, the skb last reference should be 0 when calling this.
541 * The zerocopy_success argument is true if zero copy transmit occurred,
542 * false on data copy or out of memory error caused by data copy attempt.
543 * The ctx field is used to track device context.
544 * The desc field is used to track userspace buffer index.
545 */
546struct ubuf_info {
547 const struct ubuf_info_ops *ops;
548 refcount_t refcnt;
549 u8 flags;
550};
551
552struct ubuf_info_msgzc {
553 struct ubuf_info ubuf;
554
555 union {
556 struct {
557 unsigned long desc;
558 void *ctx;
559 };
560 struct {
561 u32 id;
562 u16 len;
563 u16 zerocopy:1;
564 u32 bytelen;
565 };
566 };
567
568 struct mmpin {
569 struct user_struct *user;
570 unsigned int num_pg;
571 } mmp;
572};
573
574#define skb_uarg(SKB) ((struct ubuf_info *)(skb_shinfo(SKB)->destructor_arg))
575#define uarg_to_msgzc(ubuf_ptr) container_of((ubuf_ptr), struct ubuf_info_msgzc, \
576 ubuf)
577
578int mm_account_pinned_pages(struct mmpin *mmp, size_t size);
579void mm_unaccount_pinned_pages(struct mmpin *mmp);
580
581/* Preserve some data across TX submission and completion.
582 *
583 * Note, this state is stored in the driver. Extending the layout
584 * might need some special care.
585 */
586struct xsk_tx_metadata_compl {
587 __u64 *tx_timestamp;
588};
589
590/* This data is invariant across clones and lives at
591 * the end of the header data, ie. at skb->end.
592 */
593struct skb_shared_info {
594 __u8 flags;
595 __u8 meta_len;
596 __u8 nr_frags;
597 __u8 tx_flags;
598 unsigned short gso_size;
599 /* Warning: this field is not always filled in (UFO)! */
600 unsigned short gso_segs;
601 struct sk_buff *frag_list;
602 union {
603 struct skb_shared_hwtstamps hwtstamps;
604 struct xsk_tx_metadata_compl xsk_meta;
605 };
606 unsigned int gso_type;
607 u32 tskey;
608
609 /*
610 * Warning : all fields before dataref are cleared in __alloc_skb()
611 */
612 atomic_t dataref;
613
614 union {
615 struct {
616 u32 xdp_frags_size;
617 u32 xdp_frags_truesize;
618 };
619
620 /*
621 * Intermediate layers must ensure that destructor_arg
622 * remains valid until skb destructor.
623 */
624 void *destructor_arg;
625 };
626
627 /* must be last field, see pskb_expand_head() */
628 skb_frag_t frags[MAX_SKB_FRAGS];
629};
630
631/**
632 * DOC: dataref and headerless skbs
633 *
634 * Transport layers send out clones of payload skbs they hold for
635 * retransmissions. To allow lower layers of the stack to prepend their headers
636 * we split &skb_shared_info.dataref into two halves.
637 * The lower 16 bits count the overall number of references.
638 * The higher 16 bits indicate how many of the references are payload-only.
639 * skb_header_cloned() checks if skb is allowed to add / write the headers.
640 *
641 * The creator of the skb (e.g. TCP) marks its skb as &sk_buff.nohdr
642 * (via __skb_header_release()). Any clone created from marked skb will get
643 * &sk_buff.hdr_len populated with the available headroom.
644 * If there's the only clone in existence it's able to modify the headroom
645 * at will. The sequence of calls inside the transport layer is::
646 *
647 * <alloc skb>
648 * skb_reserve()
649 * __skb_header_release()
650 * skb_clone()
651 * // send the clone down the stack
652 *
653 * This is not a very generic construct and it depends on the transport layers
654 * doing the right thing. In practice there's usually only one payload-only skb.
655 * Having multiple payload-only skbs with different lengths of hdr_len is not
656 * possible. The payload-only skbs should never leave their owner.
657 */
658#define SKB_DATAREF_SHIFT 16
659#define SKB_DATAREF_MASK ((1 << SKB_DATAREF_SHIFT) - 1)
660
661
662enum {
663 SKB_FCLONE_UNAVAILABLE, /* skb has no fclone (from head_cache) */
664 SKB_FCLONE_ORIG, /* orig skb (from fclone_cache) */
665 SKB_FCLONE_CLONE, /* companion fclone skb (from fclone_cache) */
666};
667
668enum {
669 SKB_GSO_TCPV4 = 1 << 0,
670
671 /* This indicates the skb is from an untrusted source. */
672 SKB_GSO_DODGY = 1 << 1,
673
674 /* This indicates the tcp segment has CWR set. */
675 SKB_GSO_TCP_ECN = 1 << 2,
676
677 SKB_GSO_TCP_FIXEDID = 1 << 3,
678
679 SKB_GSO_TCPV6 = 1 << 4,
680
681 SKB_GSO_FCOE = 1 << 5,
682
683 SKB_GSO_GRE = 1 << 6,
684
685 SKB_GSO_GRE_CSUM = 1 << 7,
686
687 SKB_GSO_IPXIP4 = 1 << 8,
688
689 SKB_GSO_IPXIP6 = 1 << 9,
690
691 SKB_GSO_UDP_TUNNEL = 1 << 10,
692
693 SKB_GSO_UDP_TUNNEL_CSUM = 1 << 11,
694
695 SKB_GSO_PARTIAL = 1 << 12,
696
697 SKB_GSO_TUNNEL_REMCSUM = 1 << 13,
698
699 SKB_GSO_SCTP = 1 << 14,
700
701 SKB_GSO_ESP = 1 << 15,
702
703 SKB_GSO_UDP = 1 << 16,
704
705 SKB_GSO_UDP_L4 = 1 << 17,
706
707 SKB_GSO_FRAGLIST = 1 << 18,
708
709 SKB_GSO_TCP_ACCECN = 1 << 19,
710};
711
712#if BITS_PER_LONG > 32
713#define NET_SKBUFF_DATA_USES_OFFSET 1
714#endif
715
716#ifdef NET_SKBUFF_DATA_USES_OFFSET
717typedef unsigned int sk_buff_data_t;
718#else
719typedef unsigned char *sk_buff_data_t;
720#endif
721
722enum skb_tstamp_type {
723 SKB_CLOCK_REALTIME,
724 SKB_CLOCK_MONOTONIC,
725 SKB_CLOCK_TAI,
726 __SKB_CLOCK_MAX = SKB_CLOCK_TAI,
727};
728
729/**
730 * DOC: Basic sk_buff geometry
731 *
732 * struct sk_buff itself is a metadata structure and does not hold any packet
733 * data. All the data is held in associated buffers.
734 *
735 * &sk_buff.head points to the main "head" buffer. The head buffer is divided
736 * into two parts:
737 *
738 * - data buffer, containing headers and sometimes payload;
739 * this is the part of the skb operated on by the common helpers
740 * such as skb_put() or skb_pull();
741 * - shared info (struct skb_shared_info) which holds an array of pointers
742 * to read-only data in the (page, offset, length) format.
743 *
744 * Optionally &skb_shared_info.frag_list may point to another skb.
745 *
746 * Basic diagram may look like this::
747 *
748 * ---------------
749 * | sk_buff |
750 * ---------------
751 * ,--------------------------- + head
752 * / ,----------------- + data
753 * / / ,----------- + tail
754 * | | | , + end
755 * | | | |
756 * v v v v
757 * -----------------------------------------------
758 * | headroom | data | tailroom | skb_shared_info |
759 * -----------------------------------------------
760 * + [page frag]
761 * + [page frag]
762 * + [page frag]
763 * + [page frag] ---------
764 * + frag_list --> | sk_buff |
765 * ---------
766 *
767 */
768
769/**
770 * struct sk_buff - socket buffer
771 * @next: Next buffer in list
772 * @prev: Previous buffer in list
773 * @tstamp: Time we arrived/left
774 * @skb_mstamp_ns: (aka @tstamp) earliest departure time; start point
775 * for retransmit timer
776 * @rbnode: RB tree node, alternative to next/prev for netem/tcp
777 * @list: queue head
778 * @ll_node: anchor in an llist (eg socket defer_list)
779 * @sk: Socket we are owned by
780 * @dev: Device we arrived on/are leaving by
781 * @dev_scratch: (aka @dev) alternate use of @dev when @dev would be %NULL
782 * @cb: Control buffer. Free for use by every layer. Put private vars here
783 * @_skb_refdst: destination entry (with norefcount bit)
784 * @len: Length of actual data
785 * @data_len: Data length
786 * @mac_len: Length of link layer header
787 * @hdr_len: writable header length of cloned skb
788 * @csum: Checksum (must include start/offset pair)
789 * @csum_start: Offset from skb->head where checksumming should start
790 * @csum_offset: Offset from csum_start where checksum should be stored
791 * @priority: Packet queueing priority
792 * @ignore_df: allow local fragmentation
793 * @cloned: Head may be cloned (check refcnt to be sure)
794 * @ip_summed: Driver fed us an IP checksum
795 * @nohdr: Payload reference only, must not modify header
796 * @pkt_type: Packet class
797 * @fclone: skbuff clone status
798 * @ipvs_property: skbuff is owned by ipvs
799 * @inner_protocol_type: whether the inner protocol is
800 * ENCAP_TYPE_ETHER or ENCAP_TYPE_IPPROTO
801 * @remcsum_offload: remote checksum offload is enabled
802 * @offload_fwd_mark: Packet was L2-forwarded in hardware
803 * @offload_l3_fwd_mark: Packet was L3-forwarded in hardware
804 * @tc_skip_classify: do not classify packet. set by IFB device
805 * @tc_at_ingress: used within tc_classify to distinguish in/egress
806 * @redirected: packet was redirected by packet classifier
807 * @from_ingress: packet was redirected from the ingress path
808 * @nf_skip_egress: packet shall skip nf egress - see netfilter_netdev.h
809 * @peeked: this packet has been seen already, so stats have been
810 * done for it, don't do them again
811 * @nf_trace: netfilter packet trace flag
812 * @protocol: Packet protocol from driver
813 * @destructor: Destruct function
814 * @tcp_tsorted_anchor: list structure for TCP (tp->tsorted_sent_queue)
815 * @_sk_redir: socket redirection information for skmsg
816 * @_nfct: Associated connection, if any (with nfctinfo bits)
817 * @skb_iif: ifindex of device we arrived on
818 * @tc_index: Traffic control index
819 * @hash: the packet hash
820 * @queue_mapping: Queue mapping for multiqueue devices
821 * @head_frag: skb was allocated from page fragments,
822 * not allocated by kmalloc() or vmalloc().
823 * @pfmemalloc: skbuff was allocated from PFMEMALLOC reserves
824 * @pp_recycle: mark the packet for recycling instead of freeing (implies
825 * page_pool support on driver)
826 * @active_extensions: active extensions (skb_ext_id types)
827 * @ndisc_nodetype: router type (from link layer)
828 * @ooo_okay: allow the mapping of a socket to a queue to be changed
829 * @l4_hash: indicate hash is a canonical 4-tuple hash over transport
830 * ports.
831 * @sw_hash: indicates hash was computed in software stack
832 * @wifi_acked_valid: wifi_acked was set
833 * @wifi_acked: whether frame was acked on wifi or not
834 * @no_fcs: Request NIC to treat last 4 bytes as Ethernet FCS
835 * @encapsulation: indicates the inner headers in the skbuff are valid
836 * @encap_hdr_csum: software checksum is needed
837 * @csum_valid: checksum is already valid
838 * @csum_not_inet: use CRC32c to resolve CHECKSUM_PARTIAL
839 * @csum_complete_sw: checksum was completed by software
840 * @csum_level: indicates the number of consecutive checksums found in
841 * the packet minus one that have been verified as
842 * CHECKSUM_UNNECESSARY (max 3)
843 * @unreadable: indicates that at least 1 of the fragments in this skb is
844 * unreadable.
845 * @dst_pending_confirm: need to confirm neighbour
846 * @decrypted: Decrypted SKB
847 * @slow_gro: state present at GRO time, slower prepare step required
848 * @tstamp_type: When set, skb->tstamp has the
849 * delivery_time clock base of skb->tstamp.
850 * @napi_id: id of the NAPI struct this skb came from
851 * @sender_cpu: (aka @napi_id) source CPU in XPS
852 * @alloc_cpu: CPU which did the skb allocation.
853 * @secmark: security marking
854 * @mark: Generic packet mark
855 * @reserved_tailroom: (aka @mark) number of bytes of free space available
856 * at the tail of an sk_buff
857 * @vlan_all: vlan fields (proto & tci)
858 * @vlan_proto: vlan encapsulation protocol
859 * @vlan_tci: vlan tag control information
860 * @inner_protocol: Protocol (encapsulation)
861 * @inner_ipproto: (aka @inner_protocol) stores ipproto when
862 * skb->inner_protocol_type == ENCAP_TYPE_IPPROTO;
863 * @inner_transport_header: Inner transport layer header (encapsulation)
864 * @inner_network_header: Network layer header (encapsulation)
865 * @inner_mac_header: Link layer header (encapsulation)
866 * @transport_header: Transport layer header
867 * @network_header: Network layer header
868 * @mac_header: Link layer header
869 * @kcov_handle: KCOV remote handle for remote coverage collection
870 * @tail: Tail pointer
871 * @end: End pointer
872 * @head: Head of buffer
873 * @data: Data head pointer
874 * @truesize: Buffer size
875 * @users: User count - see {datagram,tcp}.c
876 * @extensions: allocated extensions, valid if active_extensions is nonzero
877 */
878
879struct sk_buff {
880 union {
881 struct {
882 /* These two members must be first to match sk_buff_head. */
883 struct sk_buff *next;
884 struct sk_buff *prev;
885
886 union {
887 struct net_device *dev;
888 /* Some protocols might use this space to store information,
889 * while device pointer would be NULL.
890 * UDP receive path is one user.
891 */
892 unsigned long dev_scratch;
893 };
894 };
895 struct rb_node rbnode; /* used in netem, ip4 defrag, and tcp stack */
896 struct list_head list;
897 struct llist_node ll_node;
898 };
899
900 struct sock *sk;
901
902 union {
903 ktime_t tstamp;
904 u64 skb_mstamp_ns; /* earliest departure time */
905 };
906 /*
907 * This is the control buffer. It is free to use for every
908 * layer. Please put your private variables there. If you
909 * want to keep them across layers you have to do a skb_clone()
910 * first. This is owned by whoever has the skb queued ATM.
911 */
912 char cb[48] __aligned(8);
913
914 union {
915 struct {
916 unsigned long _skb_refdst;
917 void (*destructor)(struct sk_buff *skb);
918 };
919 struct list_head tcp_tsorted_anchor;
920#ifdef CONFIG_NET_SOCK_MSG
921 unsigned long _sk_redir;
922#endif
923 };
924
925#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
926 unsigned long _nfct;
927#endif
928 unsigned int len,
929 data_len;
930 __u16 mac_len,
931 hdr_len;
932
933 /* Following fields are _not_ copied in __copy_skb_header()
934 * Note that queue_mapping is here mostly to fill a hole.
935 */
936 __u16 queue_mapping;
937
938/* if you move cloned around you also must adapt those constants */
939#ifdef __BIG_ENDIAN_BITFIELD
940#define CLONED_MASK (1 << 7)
941#else
942#define CLONED_MASK 1
943#endif
944#define CLONED_OFFSET offsetof(struct sk_buff, __cloned_offset)
945
946 /* private: */
947 __u8 __cloned_offset[0];
948 /* public: */
949 __u8 cloned:1,
950 nohdr:1,
951 fclone:2,
952 peeked:1,
953 head_frag:1,
954 pfmemalloc:1,
955 pp_recycle:1; /* page_pool recycle indicator */
956#ifdef CONFIG_SKB_EXTENSIONS
957 __u8 active_extensions;
958#endif
959
960 /* Fields enclosed in headers group are copied
961 * using a single memcpy() in __copy_skb_header()
962 */
963 struct_group(headers,
964
965 /* private: */
966 __u8 __pkt_type_offset[0];
967 /* public: */
968 __u8 pkt_type:3; /* see PKT_TYPE_MAX */
969 __u8 ignore_df:1;
970 __u8 dst_pending_confirm:1;
971 __u8 ip_summed:2;
972 __u8 ooo_okay:1;
973
974 /* private: */
975 __u8 __mono_tc_offset[0];
976 /* public: */
977 __u8 tstamp_type:2; /* See skb_tstamp_type */
978#ifdef CONFIG_NET_XGRESS
979 __u8 tc_at_ingress:1; /* See TC_AT_INGRESS_MASK */
980 __u8 tc_skip_classify:1;
981#endif
982 __u8 remcsum_offload:1;
983 __u8 csum_complete_sw:1;
984 __u8 csum_level:2;
985 __u8 inner_protocol_type:1;
986
987 __u8 l4_hash:1;
988 __u8 sw_hash:1;
989#ifdef CONFIG_WIRELESS
990 __u8 wifi_acked_valid:1;
991 __u8 wifi_acked:1;
992#endif
993 __u8 no_fcs:1;
994 /* Indicates the inner headers are valid in the skbuff. */
995 __u8 encapsulation:1;
996 __u8 encap_hdr_csum:1;
997 __u8 csum_valid:1;
998#ifdef CONFIG_IPV6_NDISC_NODETYPE
999 __u8 ndisc_nodetype:2;
1000#endif
1001
1002#if IS_ENABLED(CONFIG_IP_VS)
1003 __u8 ipvs_property:1;
1004#endif
1005#if IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TRACE) || IS_ENABLED(CONFIG_NF_TABLES)
1006 __u8 nf_trace:1;
1007#endif
1008#ifdef CONFIG_NET_SWITCHDEV
1009 __u8 offload_fwd_mark:1;
1010 __u8 offload_l3_fwd_mark:1;
1011#endif
1012 __u8 redirected:1;
1013#ifdef CONFIG_NET_REDIRECT
1014 __u8 from_ingress:1;
1015#endif
1016#ifdef CONFIG_NETFILTER_SKIP_EGRESS
1017 __u8 nf_skip_egress:1;
1018#endif
1019#ifdef CONFIG_SKB_DECRYPTED
1020 __u8 decrypted:1;
1021#endif
1022 __u8 slow_gro:1;
1023#if IS_ENABLED(CONFIG_IP_SCTP)
1024 __u8 csum_not_inet:1;
1025#endif
1026 __u8 unreadable:1;
1027#if defined(CONFIG_NET_SCHED) || defined(CONFIG_NET_XGRESS)
1028 __u16 tc_index; /* traffic control index */
1029#endif
1030
1031 u16 alloc_cpu;
1032
1033 union {
1034 __wsum csum;
1035 struct {
1036 __u16 csum_start;
1037 __u16 csum_offset;
1038 };
1039 };
1040 __u32 priority;
1041 int skb_iif;
1042 __u32 hash;
1043 union {
1044 u32 vlan_all;
1045 struct {
1046 __be16 vlan_proto;
1047 __u16 vlan_tci;
1048 };
1049 };
1050#if defined(CONFIG_NET_RX_BUSY_POLL) || defined(CONFIG_XPS)
1051 union {
1052 unsigned int napi_id;
1053 unsigned int sender_cpu;
1054 };
1055#endif
1056#ifdef CONFIG_NETWORK_SECMARK
1057 __u32 secmark;
1058#endif
1059
1060 union {
1061 __u32 mark;
1062 __u32 reserved_tailroom;
1063 };
1064
1065 union {
1066 __be16 inner_protocol;
1067 __u8 inner_ipproto;
1068 };
1069
1070 __u16 inner_transport_header;
1071 __u16 inner_network_header;
1072 __u16 inner_mac_header;
1073
1074 __be16 protocol;
1075 __u16 transport_header;
1076 __u16 network_header;
1077 __u16 mac_header;
1078
1079#ifdef CONFIG_KCOV
1080 u64 kcov_handle;
1081#endif
1082
1083 ); /* end headers group */
1084
1085 /* These elements must be at the end, see alloc_skb() for details. */
1086 sk_buff_data_t tail;
1087 sk_buff_data_t end;
1088 unsigned char *head,
1089 *data;
1090 unsigned int truesize;
1091 refcount_t users;
1092
1093#ifdef CONFIG_SKB_EXTENSIONS
1094 /* only usable after checking ->active_extensions != 0 */
1095 struct skb_ext *extensions;
1096#endif
1097};
1098
1099/* if you move pkt_type around you also must adapt those constants */
1100#ifdef __BIG_ENDIAN_BITFIELD
1101#define PKT_TYPE_MAX (7 << 5)
1102#else
1103#define PKT_TYPE_MAX 7
1104#endif
1105#define PKT_TYPE_OFFSET offsetof(struct sk_buff, __pkt_type_offset)
1106
1107/* if you move tc_at_ingress or tstamp_type
1108 * around, you also must adapt these constants.
1109 */
1110#ifdef __BIG_ENDIAN_BITFIELD
1111#define SKB_TSTAMP_TYPE_MASK (3 << 6)
1112#define SKB_TSTAMP_TYPE_RSHIFT (6)
1113#define TC_AT_INGRESS_MASK (1 << 5)
1114#else
1115#define SKB_TSTAMP_TYPE_MASK (3)
1116#define TC_AT_INGRESS_MASK (1 << 2)
1117#endif
1118#define SKB_BF_MONO_TC_OFFSET offsetof(struct sk_buff, __mono_tc_offset)
1119
1120#ifdef __KERNEL__
1121/*
1122 * Handling routines are only of interest to the kernel
1123 */
1124
1125#define SKB_ALLOC_FCLONE 0x01
1126#define SKB_ALLOC_RX 0x02
1127#define SKB_ALLOC_NAPI 0x04
1128
1129/**
1130 * skb_pfmemalloc - Test if the skb was allocated from PFMEMALLOC reserves
1131 * @skb: buffer
1132 */
1133static inline bool skb_pfmemalloc(const struct sk_buff *skb)
1134{
1135 return unlikely(skb->pfmemalloc);
1136}
1137
1138/*
1139 * skb might have a dst pointer attached, refcounted or not.
1140 * _skb_refdst low order bit is set if refcount was _not_ taken
1141 */
1142#define SKB_DST_NOREF 1UL
1143#define SKB_DST_PTRMASK ~(SKB_DST_NOREF)
1144
1145/**
1146 * skb_dst - returns skb dst_entry
1147 * @skb: buffer
1148 *
1149 * Returns: skb dst_entry, regardless of reference taken or not.
1150 */
1151static inline struct dst_entry *skb_dst(const struct sk_buff *skb)
1152{
1153 /* If refdst was not refcounted, check we still are in a
1154 * rcu_read_lock section
1155 */
1156 WARN_ON((skb->_skb_refdst & SKB_DST_NOREF) &&
1157 !rcu_read_lock_held() &&
1158 !rcu_read_lock_bh_held());
1159 return (struct dst_entry *)(skb->_skb_refdst & SKB_DST_PTRMASK);
1160}
1161
1162/**
1163 * skb_dst_set - sets skb dst
1164 * @skb: buffer
1165 * @dst: dst entry
1166 *
1167 * Sets skb dst, assuming a reference was taken on dst and should
1168 * be released by skb_dst_drop()
1169 */
1170static inline void skb_dst_set(struct sk_buff *skb, struct dst_entry *dst)
1171{
1172 skb->slow_gro |= !!dst;
1173 skb->_skb_refdst = (unsigned long)dst;
1174}
1175
1176/**
1177 * skb_dst_set_noref - sets skb dst, hopefully, without taking reference
1178 * @skb: buffer
1179 * @dst: dst entry
1180 *
1181 * Sets skb dst, assuming a reference was not taken on dst.
1182 * If dst entry is cached, we do not take reference and dst_release
1183 * will be avoided by refdst_drop. If dst entry is not cached, we take
1184 * reference, so that last dst_release can destroy the dst immediately.
1185 */
1186static inline void skb_dst_set_noref(struct sk_buff *skb, struct dst_entry *dst)
1187{
1188 WARN_ON(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
1189 skb->slow_gro |= !!dst;
1190 skb->_skb_refdst = (unsigned long)dst | SKB_DST_NOREF;
1191}
1192
1193/**
1194 * skb_dst_is_noref - Test if skb dst isn't refcounted
1195 * @skb: buffer
1196 */
1197static inline bool skb_dst_is_noref(const struct sk_buff *skb)
1198{
1199 return (skb->_skb_refdst & SKB_DST_NOREF) && skb_dst(skb);
1200}
1201
1202/* For mangling skb->pkt_type from user space side from applications
1203 * such as nft, tc, etc, we only allow a conservative subset of
1204 * possible pkt_types to be set.
1205*/
1206static inline bool skb_pkt_type_ok(u32 ptype)
1207{
1208 return ptype <= PACKET_OTHERHOST;
1209}
1210
1211/**
1212 * skb_napi_id - Returns the skb's NAPI id
1213 * @skb: buffer
1214 */
1215static inline unsigned int skb_napi_id(const struct sk_buff *skb)
1216{
1217#ifdef CONFIG_NET_RX_BUSY_POLL
1218 return skb->napi_id;
1219#else
1220 return 0;
1221#endif
1222}
1223
1224static inline bool skb_wifi_acked_valid(const struct sk_buff *skb)
1225{
1226#ifdef CONFIG_WIRELESS
1227 return skb->wifi_acked_valid;
1228#else
1229 return 0;
1230#endif
1231}
1232
1233/**
1234 * skb_unref - decrement the skb's reference count
1235 * @skb: buffer
1236 *
1237 * Returns: true if we can free the skb.
1238 */
1239static inline bool skb_unref(struct sk_buff *skb)
1240{
1241 if (unlikely(!skb))
1242 return false;
1243 if (!IS_ENABLED(CONFIG_DEBUG_NET) && likely(refcount_read(&skb->users) == 1))
1244 smp_rmb();
1245 else if (likely(!refcount_dec_and_test(&skb->users)))
1246 return false;
1247
1248 return true;
1249}
1250
1251static inline bool skb_data_unref(const struct sk_buff *skb,
1252 struct skb_shared_info *shinfo)
1253{
1254 int bias;
1255
1256 if (!skb->cloned)
1257 return true;
1258
1259 bias = skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1;
1260
1261 if (atomic_read(v: &shinfo->dataref) == bias)
1262 smp_rmb();
1263 else if (atomic_sub_return(i: bias, v: &shinfo->dataref))
1264 return false;
1265
1266 return true;
1267}
1268
1269void __fix_address sk_skb_reason_drop(struct sock *sk, struct sk_buff *skb,
1270 enum skb_drop_reason reason);
1271
1272static inline void
1273kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason)
1274{
1275 sk_skb_reason_drop(NULL, skb, reason);
1276}
1277
1278/**
1279 * kfree_skb - free an sk_buff with 'NOT_SPECIFIED' reason
1280 * @skb: buffer to free
1281 */
1282static inline void kfree_skb(struct sk_buff *skb)
1283{
1284 kfree_skb_reason(skb, reason: SKB_DROP_REASON_NOT_SPECIFIED);
1285}
1286
1287void skb_release_head_state(struct sk_buff *skb);
1288void kfree_skb_list_reason(struct sk_buff *segs,
1289 enum skb_drop_reason reason);
1290void skb_dump(const char *level, const struct sk_buff *skb, bool full_pkt);
1291void skb_tx_error(struct sk_buff *skb);
1292
1293static inline void kfree_skb_list(struct sk_buff *segs)
1294{
1295 kfree_skb_list_reason(segs, reason: SKB_DROP_REASON_NOT_SPECIFIED);
1296}
1297
1298#ifdef CONFIG_TRACEPOINTS
1299void consume_skb(struct sk_buff *skb);
1300#else
1301static inline void consume_skb(struct sk_buff *skb)
1302{
1303 return kfree_skb(skb);
1304}
1305#endif
1306
1307void __consume_stateless_skb(struct sk_buff *skb);
1308void __kfree_skb(struct sk_buff *skb);
1309
1310void kfree_skb_partial(struct sk_buff *skb, bool head_stolen);
1311bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
1312 bool *fragstolen, int *delta_truesize);
1313
1314struct sk_buff *__alloc_skb(unsigned int size, gfp_t priority, int flags,
1315 int node);
1316struct sk_buff *__build_skb(void *data, unsigned int frag_size);
1317struct sk_buff *build_skb(void *data, unsigned int frag_size);
1318struct sk_buff *build_skb_around(struct sk_buff *skb,
1319 void *data, unsigned int frag_size);
1320void skb_attempt_defer_free(struct sk_buff *skb);
1321
1322u32 napi_skb_cache_get_bulk(void **skbs, u32 n);
1323struct sk_buff *napi_build_skb(void *data, unsigned int frag_size);
1324struct sk_buff *slab_build_skb(void *data);
1325
1326/**
1327 * alloc_skb - allocate a network buffer
1328 * @size: size to allocate
1329 * @priority: allocation mask
1330 *
1331 * This function is a convenient wrapper around __alloc_skb().
1332 */
1333static inline struct sk_buff *alloc_skb(unsigned int size,
1334 gfp_t priority)
1335{
1336 return __alloc_skb(size, priority, flags: 0, NUMA_NO_NODE);
1337}
1338
1339struct sk_buff *alloc_skb_with_frags(unsigned long header_len,
1340 unsigned long data_len,
1341 int max_page_order,
1342 int *errcode,
1343 gfp_t gfp_mask);
1344struct sk_buff *alloc_skb_for_msg(struct sk_buff *first);
1345
1346/* Layout of fast clones : [skb1][skb2][fclone_ref] */
1347struct sk_buff_fclones {
1348 struct sk_buff skb1;
1349
1350 struct sk_buff skb2;
1351
1352 refcount_t fclone_ref;
1353};
1354
1355/**
1356 * skb_fclone_busy - check if fclone is busy
1357 * @sk: socket
1358 * @skb: buffer
1359 *
1360 * Returns: true if skb is a fast clone, and its clone is not freed.
1361 * Some drivers call skb_orphan() in their ndo_start_xmit(),
1362 * so we also check that didn't happen.
1363 */
1364static inline bool skb_fclone_busy(const struct sock *sk,
1365 const struct sk_buff *skb)
1366{
1367 const struct sk_buff_fclones *fclones;
1368
1369 fclones = container_of(skb, struct sk_buff_fclones, skb1);
1370
1371 return skb->fclone == SKB_FCLONE_ORIG &&
1372 refcount_read(r: &fclones->fclone_ref) > 1 &&
1373 READ_ONCE(fclones->skb2.sk) == sk;
1374}
1375
1376/**
1377 * alloc_skb_fclone - allocate a network buffer from fclone cache
1378 * @size: size to allocate
1379 * @priority: allocation mask
1380 *
1381 * This function is a convenient wrapper around __alloc_skb().
1382 */
1383static inline struct sk_buff *alloc_skb_fclone(unsigned int size,
1384 gfp_t priority)
1385{
1386 return __alloc_skb(size, priority, SKB_ALLOC_FCLONE, NUMA_NO_NODE);
1387}
1388
1389struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src);
1390void skb_headers_offset_update(struct sk_buff *skb, int off);
1391int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask);
1392struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t priority);
1393void skb_copy_header(struct sk_buff *new, const struct sk_buff *old);
1394struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t priority);
1395struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom,
1396 gfp_t gfp_mask, bool fclone);
1397static inline struct sk_buff *__pskb_copy(struct sk_buff *skb, int headroom,
1398 gfp_t gfp_mask)
1399{
1400 return __pskb_copy_fclone(skb, headroom, gfp_mask, fclone: false);
1401}
1402
1403int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, gfp_t gfp_mask);
1404struct sk_buff *skb_realloc_headroom(struct sk_buff *skb,
1405 unsigned int headroom);
1406struct sk_buff *skb_expand_head(struct sk_buff *skb, unsigned int headroom);
1407struct sk_buff *skb_copy_expand(const struct sk_buff *skb, int newheadroom,
1408 int newtailroom, gfp_t priority);
1409int __must_check skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg,
1410 int offset, int len);
1411int __must_check skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg,
1412 int offset, int len);
1413int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer);
1414int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error);
1415
1416/**
1417 * skb_pad - zero pad the tail of an skb
1418 * @skb: buffer to pad
1419 * @pad: space to pad
1420 *
1421 * Ensure that a buffer is followed by a padding area that is zero
1422 * filled. Used by network drivers which may DMA or transfer data
1423 * beyond the buffer end onto the wire.
1424 *
1425 * May return error in out of memory cases. The skb is freed on error.
1426 */
1427static inline int skb_pad(struct sk_buff *skb, int pad)
1428{
1429 return __skb_pad(skb, pad, free_on_error: true);
1430}
1431#define dev_kfree_skb(a) consume_skb(a)
1432
1433int skb_append_pagefrags(struct sk_buff *skb, struct page *page,
1434 int offset, size_t size, size_t max_frags);
1435
1436struct skb_seq_state {
1437 __u32 lower_offset;
1438 __u32 upper_offset;
1439 __u32 frag_idx;
1440 __u32 stepped_offset;
1441 struct sk_buff *root_skb;
1442 struct sk_buff *cur_skb;
1443 __u8 *frag_data;
1444 __u32 frag_off;
1445};
1446
1447void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
1448 unsigned int to, struct skb_seq_state *st);
1449unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
1450 struct skb_seq_state *st);
1451void skb_abort_seq_read(struct skb_seq_state *st);
1452int skb_copy_seq_read(struct skb_seq_state *st, int offset, void *to, int len);
1453
1454unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
1455 unsigned int to, struct ts_config *config);
1456
1457/*
1458 * Packet hash types specify the type of hash in skb_set_hash.
1459 *
1460 * Hash types refer to the protocol layer addresses which are used to
1461 * construct a packet's hash. The hashes are used to differentiate or identify
1462 * flows of the protocol layer for the hash type. Hash types are either
1463 * layer-2 (L2), layer-3 (L3), or layer-4 (L4).
1464 *
1465 * Properties of hashes:
1466 *
1467 * 1) Two packets in different flows have different hash values
1468 * 2) Two packets in the same flow should have the same hash value
1469 *
1470 * A hash at a higher layer is considered to be more specific. A driver should
1471 * set the most specific hash possible.
1472 *
1473 * A driver cannot indicate a more specific hash than the layer at which a hash
1474 * was computed. For instance an L3 hash cannot be set as an L4 hash.
1475 *
1476 * A driver may indicate a hash level which is less specific than the
1477 * actual layer the hash was computed on. For instance, a hash computed
1478 * at L4 may be considered an L3 hash. This should only be done if the
1479 * driver can't unambiguously determine that the HW computed the hash at
1480 * the higher layer. Note that the "should" in the second property above
1481 * permits this.
1482 */
1483enum pkt_hash_types {
1484 PKT_HASH_TYPE_NONE, /* Undefined type */
1485 PKT_HASH_TYPE_L2, /* Input: src_MAC, dest_MAC */
1486 PKT_HASH_TYPE_L3, /* Input: src_IP, dst_IP */
1487 PKT_HASH_TYPE_L4, /* Input: src_IP, dst_IP, src_port, dst_port */
1488};
1489
1490static inline void skb_clear_hash(struct sk_buff *skb)
1491{
1492 skb->hash = 0;
1493 skb->sw_hash = 0;
1494 skb->l4_hash = 0;
1495}
1496
1497static inline void skb_clear_hash_if_not_l4(struct sk_buff *skb)
1498{
1499 if (!skb->l4_hash)
1500 skb_clear_hash(skb);
1501}
1502
1503static inline void
1504__skb_set_hash(struct sk_buff *skb, __u32 hash, bool is_sw, bool is_l4)
1505{
1506 skb->l4_hash = is_l4;
1507 skb->sw_hash = is_sw;
1508 skb->hash = hash;
1509}
1510
1511static inline void
1512skb_set_hash(struct sk_buff *skb, __u32 hash, enum pkt_hash_types type)
1513{
1514 /* Used by drivers to set hash from HW */
1515 __skb_set_hash(skb, hash, is_sw: false, is_l4: type == PKT_HASH_TYPE_L4);
1516}
1517
1518static inline void
1519__skb_set_sw_hash(struct sk_buff *skb, __u32 hash, bool is_l4)
1520{
1521 __skb_set_hash(skb, hash, is_sw: true, is_l4);
1522}
1523
1524u32 __skb_get_hash_symmetric_net(const struct net *net, const struct sk_buff *skb);
1525
1526static inline u32 __skb_get_hash_symmetric(const struct sk_buff *skb)
1527{
1528 return __skb_get_hash_symmetric_net(NULL, skb);
1529}
1530
1531void __skb_get_hash_net(const struct net *net, struct sk_buff *skb);
1532u32 skb_get_poff(const struct sk_buff *skb);
1533u32 __skb_get_poff(const struct sk_buff *skb, const void *data,
1534 const struct flow_keys_basic *keys, int hlen);
1535__be32 skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto,
1536 const void *data, int hlen_proto);
1537
1538void skb_flow_dissector_init(struct flow_dissector *flow_dissector,
1539 const struct flow_dissector_key *key,
1540 unsigned int key_count);
1541
1542struct bpf_flow_dissector;
1543u32 bpf_flow_dissect(struct bpf_prog *prog, struct bpf_flow_dissector *ctx,
1544 __be16 proto, int nhoff, int hlen, unsigned int flags);
1545
1546bool __skb_flow_dissect(const struct net *net,
1547 const struct sk_buff *skb,
1548 struct flow_dissector *flow_dissector,
1549 void *target_container, const void *data,
1550 __be16 proto, int nhoff, int hlen, unsigned int flags);
1551
1552static inline bool skb_flow_dissect(const struct sk_buff *skb,
1553 struct flow_dissector *flow_dissector,
1554 void *target_container, unsigned int flags)
1555{
1556 return __skb_flow_dissect(NULL, skb, flow_dissector,
1557 target_container, NULL, proto: 0, nhoff: 0, hlen: 0, flags);
1558}
1559
1560static inline bool skb_flow_dissect_flow_keys(const struct sk_buff *skb,
1561 struct flow_keys *flow,
1562 unsigned int flags)
1563{
1564 memset(flow, 0, sizeof(*flow));
1565 return __skb_flow_dissect(NULL, skb, flow_dissector: &flow_keys_dissector,
1566 target_container: flow, NULL, proto: 0, nhoff: 0, hlen: 0, flags);
1567}
1568
1569static inline bool
1570skb_flow_dissect_flow_keys_basic(const struct net *net,
1571 const struct sk_buff *skb,
1572 struct flow_keys_basic *flow,
1573 const void *data, __be16 proto,
1574 int nhoff, int hlen, unsigned int flags)
1575{
1576 memset(flow, 0, sizeof(*flow));
1577 return __skb_flow_dissect(net, skb, flow_dissector: &flow_keys_basic_dissector, target_container: flow,
1578 data, proto, nhoff, hlen, flags);
1579}
1580
1581void skb_flow_dissect_meta(const struct sk_buff *skb,
1582 struct flow_dissector *flow_dissector,
1583 void *target_container);
1584
1585/* Gets a skb connection tracking info, ctinfo map should be a
1586 * map of mapsize to translate enum ip_conntrack_info states
1587 * to user states.
1588 */
1589void
1590skb_flow_dissect_ct(const struct sk_buff *skb,
1591 struct flow_dissector *flow_dissector,
1592 void *target_container,
1593 u16 *ctinfo_map, size_t mapsize,
1594 bool post_ct, u16 zone);
1595void
1596skb_flow_dissect_tunnel_info(const struct sk_buff *skb,
1597 struct flow_dissector *flow_dissector,
1598 void *target_container);
1599
1600void skb_flow_dissect_hash(const struct sk_buff *skb,
1601 struct flow_dissector *flow_dissector,
1602 void *target_container);
1603
1604static inline __u32 skb_get_hash_net(const struct net *net, struct sk_buff *skb)
1605{
1606 if (!skb->l4_hash && !skb->sw_hash)
1607 __skb_get_hash_net(net, skb);
1608
1609 return skb->hash;
1610}
1611
1612static inline __u32 skb_get_hash(struct sk_buff *skb)
1613{
1614 if (!skb->l4_hash && !skb->sw_hash)
1615 __skb_get_hash_net(NULL, skb);
1616
1617 return skb->hash;
1618}
1619
1620static inline __u32 skb_get_hash_flowi6(struct sk_buff *skb, const struct flowi6 *fl6)
1621{
1622 if (!skb->l4_hash && !skb->sw_hash) {
1623 struct flow_keys keys;
1624 __u32 hash = __get_hash_from_flowi6(fl6, keys: &keys);
1625
1626 __skb_set_sw_hash(skb, hash, is_l4: flow_keys_have_l4(keys: &keys));
1627 }
1628
1629 return skb->hash;
1630}
1631
1632__u32 skb_get_hash_perturb(const struct sk_buff *skb,
1633 const siphash_key_t *perturb);
1634
1635static inline __u32 skb_get_hash_raw(const struct sk_buff *skb)
1636{
1637 return skb->hash;
1638}
1639
1640static inline void skb_copy_hash(struct sk_buff *to, const struct sk_buff *from)
1641{
1642 to->hash = from->hash;
1643 to->sw_hash = from->sw_hash;
1644 to->l4_hash = from->l4_hash;
1645};
1646
1647static inline int skb_cmp_decrypted(const struct sk_buff *skb1,
1648 const struct sk_buff *skb2)
1649{
1650#ifdef CONFIG_SKB_DECRYPTED
1651 return skb2->decrypted - skb1->decrypted;
1652#else
1653 return 0;
1654#endif
1655}
1656
1657static inline bool skb_is_decrypted(const struct sk_buff *skb)
1658{
1659#ifdef CONFIG_SKB_DECRYPTED
1660 return skb->decrypted;
1661#else
1662 return false;
1663#endif
1664}
1665
1666static inline void skb_copy_decrypted(struct sk_buff *to,
1667 const struct sk_buff *from)
1668{
1669#ifdef CONFIG_SKB_DECRYPTED
1670 to->decrypted = from->decrypted;
1671#endif
1672}
1673
1674#ifdef NET_SKBUFF_DATA_USES_OFFSET
1675static inline unsigned char *skb_end_pointer(const struct sk_buff *skb)
1676{
1677 return skb->head + skb->end;
1678}
1679
1680static inline unsigned int skb_end_offset(const struct sk_buff *skb)
1681{
1682 return skb->end;
1683}
1684
1685static inline void skb_set_end_offset(struct sk_buff *skb, unsigned int offset)
1686{
1687 skb->end = offset;
1688}
1689#else
1690static inline unsigned char *skb_end_pointer(const struct sk_buff *skb)
1691{
1692 return skb->end;
1693}
1694
1695static inline unsigned int skb_end_offset(const struct sk_buff *skb)
1696{
1697 return skb->end - skb->head;
1698}
1699
1700static inline void skb_set_end_offset(struct sk_buff *skb, unsigned int offset)
1701{
1702 skb->end = skb->head + offset;
1703}
1704#endif
1705
1706extern const struct ubuf_info_ops msg_zerocopy_ubuf_ops;
1707
1708struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size,
1709 struct ubuf_info *uarg, bool devmem);
1710
1711void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref);
1712
1713struct net_devmem_dmabuf_binding;
1714
1715int __zerocopy_sg_from_iter(struct msghdr *msg, struct sock *sk,
1716 struct sk_buff *skb, struct iov_iter *from,
1717 size_t length,
1718 struct net_devmem_dmabuf_binding *binding);
1719
1720int zerocopy_fill_skb_from_iter(struct sk_buff *skb,
1721 struct iov_iter *from, size_t length);
1722
1723static inline int skb_zerocopy_iter_dgram(struct sk_buff *skb,
1724 struct msghdr *msg, int len)
1725{
1726 return __zerocopy_sg_from_iter(msg, sk: skb->sk, skb, from: &msg->msg_iter, length: len,
1727 NULL);
1728}
1729
1730int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
1731 struct msghdr *msg, int len,
1732 struct ubuf_info *uarg,
1733 struct net_devmem_dmabuf_binding *binding);
1734
1735/* Internal */
1736#define skb_shinfo(SKB) ((struct skb_shared_info *)(skb_end_pointer(SKB)))
1737
1738static inline struct skb_shared_hwtstamps *skb_hwtstamps(struct sk_buff *skb)
1739{
1740 return &skb_shinfo(skb)->hwtstamps;
1741}
1742
1743static inline struct ubuf_info *skb_zcopy(struct sk_buff *skb)
1744{
1745 bool is_zcopy = skb && skb_shinfo(skb)->flags & SKBFL_ZEROCOPY_ENABLE;
1746
1747 return is_zcopy ? skb_uarg(skb) : NULL;
1748}
1749
1750static inline bool skb_zcopy_pure(const struct sk_buff *skb)
1751{
1752 return skb_shinfo(skb)->flags & SKBFL_PURE_ZEROCOPY;
1753}
1754
1755static inline bool skb_zcopy_managed(const struct sk_buff *skb)
1756{
1757 return skb_shinfo(skb)->flags & SKBFL_MANAGED_FRAG_REFS;
1758}
1759
1760static inline bool skb_pure_zcopy_same(const struct sk_buff *skb1,
1761 const struct sk_buff *skb2)
1762{
1763 return skb_zcopy_pure(skb: skb1) == skb_zcopy_pure(skb: skb2);
1764}
1765
1766static inline void net_zcopy_get(struct ubuf_info *uarg)
1767{
1768 refcount_inc(r: &uarg->refcnt);
1769}
1770
1771static inline void skb_zcopy_init(struct sk_buff *skb, struct ubuf_info *uarg)
1772{
1773 skb_shinfo(skb)->destructor_arg = uarg;
1774 skb_shinfo(skb)->flags |= uarg->flags;
1775}
1776
1777static inline void skb_zcopy_set(struct sk_buff *skb, struct ubuf_info *uarg,
1778 bool *have_ref)
1779{
1780 if (skb && uarg && !skb_zcopy(skb)) {
1781 if (unlikely(have_ref && *have_ref))
1782 *have_ref = false;
1783 else
1784 net_zcopy_get(uarg);
1785 skb_zcopy_init(skb, uarg);
1786 }
1787}
1788
1789static inline void skb_zcopy_set_nouarg(struct sk_buff *skb, void *val)
1790{
1791 skb_shinfo(skb)->destructor_arg = (void *)((uintptr_t) val | 0x1UL);
1792 skb_shinfo(skb)->flags |= SKBFL_ZEROCOPY_FRAG;
1793}
1794
1795static inline bool skb_zcopy_is_nouarg(struct sk_buff *skb)
1796{
1797 return (uintptr_t) skb_shinfo(skb)->destructor_arg & 0x1UL;
1798}
1799
1800static inline void *skb_zcopy_get_nouarg(struct sk_buff *skb)
1801{
1802 return (void *)((uintptr_t) skb_shinfo(skb)->destructor_arg & ~0x1UL);
1803}
1804
1805static inline void net_zcopy_put(struct ubuf_info *uarg)
1806{
1807 if (uarg)
1808 uarg->ops->complete(NULL, uarg, true);
1809}
1810
1811static inline void net_zcopy_put_abort(struct ubuf_info *uarg, bool have_uref)
1812{
1813 if (uarg) {
1814 if (uarg->ops == &msg_zerocopy_ubuf_ops)
1815 msg_zerocopy_put_abort(uarg, have_uref);
1816 else if (have_uref)
1817 net_zcopy_put(uarg);
1818 }
1819}
1820
1821/* Release a reference on a zerocopy structure */
1822static inline void skb_zcopy_clear(struct sk_buff *skb, bool zerocopy_success)
1823{
1824 struct ubuf_info *uarg = skb_zcopy(skb);
1825
1826 if (uarg) {
1827 if (!skb_zcopy_is_nouarg(skb))
1828 uarg->ops->complete(skb, uarg, zerocopy_success);
1829
1830 skb_shinfo(skb)->flags &= ~SKBFL_ALL_ZEROCOPY;
1831 }
1832}
1833
1834void __skb_zcopy_downgrade_managed(struct sk_buff *skb);
1835
1836static inline void skb_zcopy_downgrade_managed(struct sk_buff *skb)
1837{
1838 if (unlikely(skb_zcopy_managed(skb)))
1839 __skb_zcopy_downgrade_managed(skb);
1840}
1841
1842/* Return true if frags in this skb are readable by the host. */
1843static inline bool skb_frags_readable(const struct sk_buff *skb)
1844{
1845 return !skb->unreadable;
1846}
1847
1848static inline void skb_mark_not_on_list(struct sk_buff *skb)
1849{
1850 skb->next = NULL;
1851}
1852
1853static inline void skb_poison_list(struct sk_buff *skb)
1854{
1855#ifdef CONFIG_DEBUG_NET
1856 skb->next = SKB_LIST_POISON_NEXT;
1857#endif
1858}
1859
1860/* Iterate through singly-linked GSO fragments of an skb. */
1861#define skb_list_walk_safe(first, skb, next_skb) \
1862 for ((skb) = (first), (next_skb) = (skb) ? (skb)->next : NULL; (skb); \
1863 (skb) = (next_skb), (next_skb) = (skb) ? (skb)->next : NULL)
1864
1865static inline void skb_list_del_init(struct sk_buff *skb)
1866{
1867 __list_del_entry(entry: &skb->list);
1868 skb_mark_not_on_list(skb);
1869}
1870
1871/**
1872 * skb_queue_empty - check if a queue is empty
1873 * @list: queue head
1874 *
1875 * Returns true if the queue is empty, false otherwise.
1876 */
1877static inline int skb_queue_empty(const struct sk_buff_head *list)
1878{
1879 return list->next == (const struct sk_buff *) list;
1880}
1881
1882/**
1883 * skb_queue_empty_lockless - check if a queue is empty
1884 * @list: queue head
1885 *
1886 * Returns true if the queue is empty, false otherwise.
1887 * This variant can be used in lockless contexts.
1888 */
1889static inline bool skb_queue_empty_lockless(const struct sk_buff_head *list)
1890{
1891 return READ_ONCE(list->next) == (const struct sk_buff *) list;
1892}
1893
1894
1895/**
1896 * skb_queue_is_last - check if skb is the last entry in the queue
1897 * @list: queue head
1898 * @skb: buffer
1899 *
1900 * Returns true if @skb is the last buffer on the list.
1901 */
1902static inline bool skb_queue_is_last(const struct sk_buff_head *list,
1903 const struct sk_buff *skb)
1904{
1905 return skb->next == (const struct sk_buff *) list;
1906}
1907
1908/**
1909 * skb_queue_is_first - check if skb is the first entry in the queue
1910 * @list: queue head
1911 * @skb: buffer
1912 *
1913 * Returns true if @skb is the first buffer on the list.
1914 */
1915static inline bool skb_queue_is_first(const struct sk_buff_head *list,
1916 const struct sk_buff *skb)
1917{
1918 return skb->prev == (const struct sk_buff *) list;
1919}
1920
1921/**
1922 * skb_queue_next - return the next packet in the queue
1923 * @list: queue head
1924 * @skb: current buffer
1925 *
1926 * Return the next packet in @list after @skb. It is only valid to
1927 * call this if skb_queue_is_last() evaluates to false.
1928 */
1929static inline struct sk_buff *skb_queue_next(const struct sk_buff_head *list,
1930 const struct sk_buff *skb)
1931{
1932 /* This BUG_ON may seem severe, but if we just return then we
1933 * are going to dereference garbage.
1934 */
1935 BUG_ON(skb_queue_is_last(list, skb));
1936 return skb->next;
1937}
1938
1939/**
1940 * skb_queue_prev - return the prev packet in the queue
1941 * @list: queue head
1942 * @skb: current buffer
1943 *
1944 * Return the prev packet in @list before @skb. It is only valid to
1945 * call this if skb_queue_is_first() evaluates to false.
1946 */
1947static inline struct sk_buff *skb_queue_prev(const struct sk_buff_head *list,
1948 const struct sk_buff *skb)
1949{
1950 /* This BUG_ON may seem severe, but if we just return then we
1951 * are going to dereference garbage.
1952 */
1953 BUG_ON(skb_queue_is_first(list, skb));
1954 return skb->prev;
1955}
1956
1957/**
1958 * skb_get - reference buffer
1959 * @skb: buffer to reference
1960 *
1961 * Makes another reference to a socket buffer and returns a pointer
1962 * to the buffer.
1963 */
1964static inline struct sk_buff *skb_get(struct sk_buff *skb)
1965{
1966 refcount_inc(r: &skb->users);
1967 return skb;
1968}
1969
1970/*
1971 * If users == 1, we are the only owner and can avoid redundant atomic changes.
1972 */
1973
1974/**
1975 * skb_cloned - is the buffer a clone
1976 * @skb: buffer to check
1977 *
1978 * Returns true if the buffer was generated with skb_clone() and is
1979 * one of multiple shared copies of the buffer. Cloned buffers are
1980 * shared data so must not be written to under normal circumstances.
1981 */
1982static inline int skb_cloned(const struct sk_buff *skb)
1983{
1984 return skb->cloned &&
1985 (atomic_read(v: &skb_shinfo(skb)->dataref) & SKB_DATAREF_MASK) != 1;
1986}
1987
1988static inline int skb_unclone(struct sk_buff *skb, gfp_t pri)
1989{
1990 might_sleep_if(gfpflags_allow_blocking(pri));
1991
1992 if (skb_cloned(skb))
1993 return pskb_expand_head(skb, nhead: 0, ntail: 0, gfp_mask: pri);
1994
1995 return 0;
1996}
1997
1998/* This variant of skb_unclone() makes sure skb->truesize
1999 * and skb_end_offset() are not changed, whenever a new skb->head is needed.
2000 *
2001 * Indeed there is no guarantee that ksize(kmalloc(X)) == ksize(kmalloc(X))
2002 * when various debugging features are in place.
2003 */
2004int __skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri);
2005static inline int skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri)
2006{
2007 might_sleep_if(gfpflags_allow_blocking(pri));
2008
2009 if (skb_cloned(skb))
2010 return __skb_unclone_keeptruesize(skb, pri);
2011 return 0;
2012}
2013
2014/**
2015 * skb_header_cloned - is the header a clone
2016 * @skb: buffer to check
2017 *
2018 * Returns true if modifying the header part of the buffer requires
2019 * the data to be copied.
2020 */
2021static inline int skb_header_cloned(const struct sk_buff *skb)
2022{
2023 int dataref;
2024
2025 if (!skb->cloned)
2026 return 0;
2027
2028 dataref = atomic_read(v: &skb_shinfo(skb)->dataref);
2029 dataref = (dataref & SKB_DATAREF_MASK) - (dataref >> SKB_DATAREF_SHIFT);
2030 return dataref != 1;
2031}
2032
2033static inline int skb_header_unclone(struct sk_buff *skb, gfp_t pri)
2034{
2035 might_sleep_if(gfpflags_allow_blocking(pri));
2036
2037 if (skb_header_cloned(skb))
2038 return pskb_expand_head(skb, nhead: 0, ntail: 0, gfp_mask: pri);
2039
2040 return 0;
2041}
2042
2043/**
2044 * __skb_header_release() - allow clones to use the headroom
2045 * @skb: buffer to operate on
2046 *
2047 * See "DOC: dataref and headerless skbs".
2048 */
2049static inline void __skb_header_release(struct sk_buff *skb)
2050{
2051 skb->nohdr = 1;
2052 atomic_set(v: &skb_shinfo(skb)->dataref, i: 1 + (1 << SKB_DATAREF_SHIFT));
2053}
2054
2055
2056/**
2057 * skb_shared - is the buffer shared
2058 * @skb: buffer to check
2059 *
2060 * Returns true if more than one person has a reference to this
2061 * buffer.
2062 */
2063static inline int skb_shared(const struct sk_buff *skb)
2064{
2065 return refcount_read(r: &skb->users) != 1;
2066}
2067
2068/**
2069 * skb_share_check - check if buffer is shared and if so clone it
2070 * @skb: buffer to check
2071 * @pri: priority for memory allocation
2072 *
2073 * If the buffer is shared the buffer is cloned and the old copy
2074 * drops a reference. A new clone with a single reference is returned.
2075 * If the buffer is not shared the original buffer is returned. When
2076 * being called from interrupt status or with spinlocks held pri must
2077 * be GFP_ATOMIC.
2078 *
2079 * NULL is returned on a memory allocation failure.
2080 */
2081static inline struct sk_buff *skb_share_check(struct sk_buff *skb, gfp_t pri)
2082{
2083 might_sleep_if(gfpflags_allow_blocking(pri));
2084 if (skb_shared(skb)) {
2085 struct sk_buff *nskb = skb_clone(skb, priority: pri);
2086
2087 if (likely(nskb))
2088 consume_skb(skb);
2089 else
2090 kfree_skb(skb);
2091 skb = nskb;
2092 }
2093 return skb;
2094}
2095
2096/*
2097 * Copy shared buffers into a new sk_buff. We effectively do COW on
2098 * packets to handle cases where we have a local reader and forward
2099 * and a couple of other messy ones. The normal one is tcpdumping
2100 * a packet that's being forwarded.
2101 */
2102
2103/**
2104 * skb_unshare - make a copy of a shared buffer
2105 * @skb: buffer to check
2106 * @pri: priority for memory allocation
2107 *
2108 * If the socket buffer is a clone then this function creates a new
2109 * copy of the data, drops a reference count on the old copy and returns
2110 * the new copy with the reference count at 1. If the buffer is not a clone
2111 * the original buffer is returned. When called with a spinlock held or
2112 * from interrupt state @pri must be %GFP_ATOMIC
2113 *
2114 * %NULL is returned on a memory allocation failure.
2115 */
2116static inline struct sk_buff *skb_unshare(struct sk_buff *skb,
2117 gfp_t pri)
2118{
2119 might_sleep_if(gfpflags_allow_blocking(pri));
2120 if (skb_cloned(skb)) {
2121 struct sk_buff *nskb = skb_copy(skb, priority: pri);
2122
2123 /* Free our shared copy */
2124 if (likely(nskb))
2125 consume_skb(skb);
2126 else
2127 kfree_skb(skb);
2128 skb = nskb;
2129 }
2130 return skb;
2131}
2132
2133/**
2134 * skb_peek - peek at the head of an &sk_buff_head
2135 * @list_: list to peek at
2136 *
2137 * Peek an &sk_buff. Unlike most other operations you _MUST_
2138 * be careful with this one. A peek leaves the buffer on the
2139 * list and someone else may run off with it. You must hold
2140 * the appropriate locks or have a private queue to do this.
2141 *
2142 * Returns %NULL for an empty list or a pointer to the head element.
2143 * The reference count is not incremented and the reference is therefore
2144 * volatile. Use with caution.
2145 */
2146static inline struct sk_buff *skb_peek(const struct sk_buff_head *list_)
2147{
2148 struct sk_buff *skb = list_->next;
2149
2150 if (skb == (struct sk_buff *)list_)
2151 skb = NULL;
2152 return skb;
2153}
2154
2155/**
2156 * __skb_peek - peek at the head of a non-empty &sk_buff_head
2157 * @list_: list to peek at
2158 *
2159 * Like skb_peek(), but the caller knows that the list is not empty.
2160 */
2161static inline struct sk_buff *__skb_peek(const struct sk_buff_head *list_)
2162{
2163 return list_->next;
2164}
2165
2166/**
2167 * skb_peek_next - peek skb following the given one from a queue
2168 * @skb: skb to start from
2169 * @list_: list to peek at
2170 *
2171 * Returns %NULL when the end of the list is met or a pointer to the
2172 * next element. The reference count is not incremented and the
2173 * reference is therefore volatile. Use with caution.
2174 */
2175static inline struct sk_buff *skb_peek_next(struct sk_buff *skb,
2176 const struct sk_buff_head *list_)
2177{
2178 struct sk_buff *next = skb->next;
2179
2180 if (next == (struct sk_buff *)list_)
2181 next = NULL;
2182 return next;
2183}
2184
2185/**
2186 * skb_peek_tail - peek at the tail of an &sk_buff_head
2187 * @list_: list to peek at
2188 *
2189 * Peek an &sk_buff. Unlike most other operations you _MUST_
2190 * be careful with this one. A peek leaves the buffer on the
2191 * list and someone else may run off with it. You must hold
2192 * the appropriate locks or have a private queue to do this.
2193 *
2194 * Returns %NULL for an empty list or a pointer to the tail element.
2195 * The reference count is not incremented and the reference is therefore
2196 * volatile. Use with caution.
2197 */
2198static inline struct sk_buff *skb_peek_tail(const struct sk_buff_head *list_)
2199{
2200 struct sk_buff *skb = READ_ONCE(list_->prev);
2201
2202 if (skb == (struct sk_buff *)list_)
2203 skb = NULL;
2204 return skb;
2205
2206}
2207
2208/**
2209 * skb_queue_len - get queue length
2210 * @list_: list to measure
2211 *
2212 * Return the length of an &sk_buff queue.
2213 */
2214static inline __u32 skb_queue_len(const struct sk_buff_head *list_)
2215{
2216 return list_->qlen;
2217}
2218
2219/**
2220 * skb_queue_len_lockless - get queue length
2221 * @list_: list to measure
2222 *
2223 * Return the length of an &sk_buff queue.
2224 * This variant can be used in lockless contexts.
2225 */
2226static inline __u32 skb_queue_len_lockless(const struct sk_buff_head *list_)
2227{
2228 return READ_ONCE(list_->qlen);
2229}
2230
2231/**
2232 * __skb_queue_head_init - initialize non-spinlock portions of sk_buff_head
2233 * @list: queue to initialize
2234 *
2235 * This initializes only the list and queue length aspects of
2236 * an sk_buff_head object. This allows to initialize the list
2237 * aspects of an sk_buff_head without reinitializing things like
2238 * the spinlock. It can also be used for on-stack sk_buff_head
2239 * objects where the spinlock is known to not be used.
2240 */
2241static inline void __skb_queue_head_init(struct sk_buff_head *list)
2242{
2243 list->prev = list->next = (struct sk_buff *)list;
2244 list->qlen = 0;
2245}
2246
2247/*
2248 * This function creates a split out lock class for each invocation;
2249 * this is needed for now since a whole lot of users of the skb-queue
2250 * infrastructure in drivers have different locking usage (in hardirq)
2251 * than the networking core (in softirq only). In the long run either the
2252 * network layer or drivers should need annotation to consolidate the
2253 * main types of usage into 3 classes.
2254 */
2255static inline void skb_queue_head_init(struct sk_buff_head *list)
2256{
2257 spin_lock_init(&list->lock);
2258 __skb_queue_head_init(list);
2259}
2260
2261static inline void skb_queue_head_init_class(struct sk_buff_head *list,
2262 struct lock_class_key *class)
2263{
2264 skb_queue_head_init(list);
2265 lockdep_set_class(&list->lock, class);
2266}
2267
2268/*
2269 * Insert an sk_buff on a list.
2270 *
2271 * The "__skb_xxxx()" functions are the non-atomic ones that
2272 * can only be called with interrupts disabled.
2273 */
2274static inline void __skb_insert(struct sk_buff *newsk,
2275 struct sk_buff *prev, struct sk_buff *next,
2276 struct sk_buff_head *list)
2277{
2278 /* See skb_queue_empty_lockless() and skb_peek_tail()
2279 * for the opposite READ_ONCE()
2280 */
2281 WRITE_ONCE(newsk->next, next);
2282 WRITE_ONCE(newsk->prev, prev);
2283 WRITE_ONCE(((struct sk_buff_list *)next)->prev, newsk);
2284 WRITE_ONCE(((struct sk_buff_list *)prev)->next, newsk);
2285 WRITE_ONCE(list->qlen, list->qlen + 1);
2286}
2287
2288static inline void __skb_queue_splice(const struct sk_buff_head *list,
2289 struct sk_buff *prev,
2290 struct sk_buff *next)
2291{
2292 struct sk_buff *first = list->next;
2293 struct sk_buff *last = list->prev;
2294
2295 WRITE_ONCE(first->prev, prev);
2296 WRITE_ONCE(prev->next, first);
2297
2298 WRITE_ONCE(last->next, next);
2299 WRITE_ONCE(next->prev, last);
2300}
2301
2302/**
2303 * skb_queue_splice - join two skb lists, this is designed for stacks
2304 * @list: the new list to add
2305 * @head: the place to add it in the first list
2306 */
2307static inline void skb_queue_splice(const struct sk_buff_head *list,
2308 struct sk_buff_head *head)
2309{
2310 if (!skb_queue_empty(list)) {
2311 __skb_queue_splice(list, prev: (struct sk_buff *) head, next: head->next);
2312 head->qlen += list->qlen;
2313 }
2314}
2315
2316/**
2317 * skb_queue_splice_init - join two skb lists and reinitialise the emptied list
2318 * @list: the new list to add
2319 * @head: the place to add it in the first list
2320 *
2321 * The list at @list is reinitialised
2322 */
2323static inline void skb_queue_splice_init(struct sk_buff_head *list,
2324 struct sk_buff_head *head)
2325{
2326 if (!skb_queue_empty(list)) {
2327 __skb_queue_splice(list, prev: (struct sk_buff *) head, next: head->next);
2328 head->qlen += list->qlen;
2329 __skb_queue_head_init(list);
2330 }
2331}
2332
2333/**
2334 * skb_queue_splice_tail - join two skb lists, each list being a queue
2335 * @list: the new list to add
2336 * @head: the place to add it in the first list
2337 */
2338static inline void skb_queue_splice_tail(const struct sk_buff_head *list,
2339 struct sk_buff_head *head)
2340{
2341 if (!skb_queue_empty(list)) {
2342 __skb_queue_splice(list, prev: head->prev, next: (struct sk_buff *) head);
2343 head->qlen += list->qlen;
2344 }
2345}
2346
2347/**
2348 * skb_queue_splice_tail_init - join two skb lists and reinitialise the emptied list
2349 * @list: the new list to add
2350 * @head: the place to add it in the first list
2351 *
2352 * Each of the lists is a queue.
2353 * The list at @list is reinitialised
2354 */
2355static inline void skb_queue_splice_tail_init(struct sk_buff_head *list,
2356 struct sk_buff_head *head)
2357{
2358 if (!skb_queue_empty(list)) {
2359 __skb_queue_splice(list, prev: head->prev, next: (struct sk_buff *) head);
2360 head->qlen += list->qlen;
2361 __skb_queue_head_init(list);
2362 }
2363}
2364
2365/**
2366 * __skb_queue_after - queue a buffer at the list head
2367 * @list: list to use
2368 * @prev: place after this buffer
2369 * @newsk: buffer to queue
2370 *
2371 * Queue a buffer int the middle of a list. This function takes no locks
2372 * and you must therefore hold required locks before calling it.
2373 *
2374 * A buffer cannot be placed on two lists at the same time.
2375 */
2376static inline void __skb_queue_after(struct sk_buff_head *list,
2377 struct sk_buff *prev,
2378 struct sk_buff *newsk)
2379{
2380 __skb_insert(newsk, prev, next: ((struct sk_buff_list *)prev)->next, list);
2381}
2382
2383void skb_append(struct sk_buff *old, struct sk_buff *newsk,
2384 struct sk_buff_head *list);
2385
2386static inline void __skb_queue_before(struct sk_buff_head *list,
2387 struct sk_buff *next,
2388 struct sk_buff *newsk)
2389{
2390 __skb_insert(newsk, prev: ((struct sk_buff_list *)next)->prev, next, list);
2391}
2392
2393/**
2394 * __skb_queue_head - queue a buffer at the list head
2395 * @list: list to use
2396 * @newsk: buffer to queue
2397 *
2398 * Queue a buffer at the start of a list. This function takes no locks
2399 * and you must therefore hold required locks before calling it.
2400 *
2401 * A buffer cannot be placed on two lists at the same time.
2402 */
2403static inline void __skb_queue_head(struct sk_buff_head *list,
2404 struct sk_buff *newsk)
2405{
2406 __skb_queue_after(list, prev: (struct sk_buff *)list, newsk);
2407}
2408void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk);
2409
2410/**
2411 * __skb_queue_tail - queue a buffer at the list tail
2412 * @list: list to use
2413 * @newsk: buffer to queue
2414 *
2415 * Queue a buffer at the end of a list. This function takes no locks
2416 * and you must therefore hold required locks before calling it.
2417 *
2418 * A buffer cannot be placed on two lists at the same time.
2419 */
2420static inline void __skb_queue_tail(struct sk_buff_head *list,
2421 struct sk_buff *newsk)
2422{
2423 __skb_queue_before(list, next: (struct sk_buff *)list, newsk);
2424}
2425void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk);
2426
2427/*
2428 * remove sk_buff from list. _Must_ be called atomically, and with
2429 * the list known..
2430 */
2431void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list);
2432static inline void __skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
2433{
2434 struct sk_buff *next, *prev;
2435
2436 WRITE_ONCE(list->qlen, list->qlen - 1);
2437 next = skb->next;
2438 prev = skb->prev;
2439 skb->next = skb->prev = NULL;
2440 WRITE_ONCE(next->prev, prev);
2441 WRITE_ONCE(prev->next, next);
2442}
2443
2444/**
2445 * __skb_dequeue - remove from the head of the queue
2446 * @list: list to dequeue from
2447 *
2448 * Remove the head of the list. This function does not take any locks
2449 * so must be used with appropriate locks held only. The head item is
2450 * returned or %NULL if the list is empty.
2451 */
2452static inline struct sk_buff *__skb_dequeue(struct sk_buff_head *list)
2453{
2454 struct sk_buff *skb = skb_peek(list_: list);
2455 if (skb)
2456 __skb_unlink(skb, list);
2457 return skb;
2458}
2459struct sk_buff *skb_dequeue(struct sk_buff_head *list);
2460
2461/**
2462 * __skb_dequeue_tail - remove from the tail of the queue
2463 * @list: list to dequeue from
2464 *
2465 * Remove the tail of the list. This function does not take any locks
2466 * so must be used with appropriate locks held only. The tail item is
2467 * returned or %NULL if the list is empty.
2468 */
2469static inline struct sk_buff *__skb_dequeue_tail(struct sk_buff_head *list)
2470{
2471 struct sk_buff *skb = skb_peek_tail(list_: list);
2472 if (skb)
2473 __skb_unlink(skb, list);
2474 return skb;
2475}
2476struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list);
2477
2478
2479static inline bool skb_is_nonlinear(const struct sk_buff *skb)
2480{
2481 return skb->data_len;
2482}
2483
2484static inline unsigned int skb_headlen(const struct sk_buff *skb)
2485{
2486 return skb->len - skb->data_len;
2487}
2488
2489static inline unsigned int __skb_pagelen(const struct sk_buff *skb)
2490{
2491 unsigned int i, len = 0;
2492
2493 for (i = skb_shinfo(skb)->nr_frags - 1; (int)i >= 0; i--)
2494 len += skb_frag_size(frag: &skb_shinfo(skb)->frags[i]);
2495 return len;
2496}
2497
2498static inline unsigned int skb_pagelen(const struct sk_buff *skb)
2499{
2500 return skb_headlen(skb) + __skb_pagelen(skb);
2501}
2502
2503static inline void skb_frag_fill_netmem_desc(skb_frag_t *frag,
2504 netmem_ref netmem, int off,
2505 int size)
2506{
2507 frag->netmem = netmem;
2508 frag->offset = off;
2509 skb_frag_size_set(frag, size);
2510}
2511
2512static inline void skb_frag_fill_page_desc(skb_frag_t *frag,
2513 struct page *page,
2514 int off, int size)
2515{
2516 skb_frag_fill_netmem_desc(frag, netmem: page_to_netmem(page), off, size);
2517}
2518
2519static inline void __skb_fill_netmem_desc_noacc(struct skb_shared_info *shinfo,
2520 int i, netmem_ref netmem,
2521 int off, int size)
2522{
2523 skb_frag_t *frag = &shinfo->frags[i];
2524
2525 skb_frag_fill_netmem_desc(frag, netmem, off, size);
2526}
2527
2528static inline void __skb_fill_page_desc_noacc(struct skb_shared_info *shinfo,
2529 int i, struct page *page,
2530 int off, int size)
2531{
2532 __skb_fill_netmem_desc_noacc(shinfo, i, netmem: page_to_netmem(page), off,
2533 size);
2534}
2535
2536/**
2537 * skb_len_add - adds a number to len fields of skb
2538 * @skb: buffer to add len to
2539 * @delta: number of bytes to add
2540 */
2541static inline void skb_len_add(struct sk_buff *skb, int delta)
2542{
2543 skb->len += delta;
2544 skb->data_len += delta;
2545 skb->truesize += delta;
2546}
2547
2548/**
2549 * __skb_fill_netmem_desc - initialise a fragment in an skb
2550 * @skb: buffer containing fragment to be initialised
2551 * @i: fragment index to initialise
2552 * @netmem: the netmem to use for this fragment
2553 * @off: the offset to the data with @page
2554 * @size: the length of the data
2555 *
2556 * Initialises the @i'th fragment of @skb to point to &size bytes at
2557 * offset @off within @page.
2558 *
2559 * Does not take any additional reference on the fragment.
2560 */
2561static inline void __skb_fill_netmem_desc(struct sk_buff *skb, int i,
2562 netmem_ref netmem, int off, int size)
2563{
2564 struct page *page;
2565
2566 __skb_fill_netmem_desc_noacc(skb_shinfo(skb), i, netmem, off, size);
2567
2568 if (netmem_is_net_iov(netmem)) {
2569 skb->unreadable = true;
2570 return;
2571 }
2572
2573 page = netmem_to_page(netmem);
2574
2575 /* Propagate page pfmemalloc to the skb if we can. The problem is
2576 * that not all callers have unique ownership of the page but rely
2577 * on page_is_pfmemalloc doing the right thing(tm).
2578 */
2579 page = compound_head(page);
2580 if (page_is_pfmemalloc(page))
2581 skb->pfmemalloc = true;
2582}
2583
2584static inline void __skb_fill_page_desc(struct sk_buff *skb, int i,
2585 struct page *page, int off, int size)
2586{
2587 __skb_fill_netmem_desc(skb, i, netmem: page_to_netmem(page), off, size);
2588}
2589
2590static inline void skb_fill_netmem_desc(struct sk_buff *skb, int i,
2591 netmem_ref netmem, int off, int size)
2592{
2593 __skb_fill_netmem_desc(skb, i, netmem, off, size);
2594 skb_shinfo(skb)->nr_frags = i + 1;
2595}
2596
2597/**
2598 * skb_fill_page_desc - initialise a paged fragment in an skb
2599 * @skb: buffer containing fragment to be initialised
2600 * @i: paged fragment index to initialise
2601 * @page: the page to use for this fragment
2602 * @off: the offset to the data with @page
2603 * @size: the length of the data
2604 *
2605 * As per __skb_fill_page_desc() -- initialises the @i'th fragment of
2606 * @skb to point to @size bytes at offset @off within @page. In
2607 * addition updates @skb such that @i is the last fragment.
2608 *
2609 * Does not take any additional reference on the fragment.
2610 */
2611static inline void skb_fill_page_desc(struct sk_buff *skb, int i,
2612 struct page *page, int off, int size)
2613{
2614 skb_fill_netmem_desc(skb, i, netmem: page_to_netmem(page), off, size);
2615}
2616
2617/**
2618 * skb_fill_page_desc_noacc - initialise a paged fragment in an skb
2619 * @skb: buffer containing fragment to be initialised
2620 * @i: paged fragment index to initialise
2621 * @page: the page to use for this fragment
2622 * @off: the offset to the data with @page
2623 * @size: the length of the data
2624 *
2625 * Variant of skb_fill_page_desc() which does not deal with
2626 * pfmemalloc, if page is not owned by us.
2627 */
2628static inline void skb_fill_page_desc_noacc(struct sk_buff *skb, int i,
2629 struct page *page, int off,
2630 int size)
2631{
2632 struct skb_shared_info *shinfo = skb_shinfo(skb);
2633
2634 __skb_fill_page_desc_noacc(shinfo, i, page, off, size);
2635 shinfo->nr_frags = i + 1;
2636}
2637
2638void skb_add_rx_frag_netmem(struct sk_buff *skb, int i, netmem_ref netmem,
2639 int off, int size, unsigned int truesize);
2640
2641static inline void skb_add_rx_frag(struct sk_buff *skb, int i,
2642 struct page *page, int off, int size,
2643 unsigned int truesize)
2644{
2645 skb_add_rx_frag_netmem(skb, i, netmem: page_to_netmem(page), off, size,
2646 truesize);
2647}
2648
2649void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size,
2650 unsigned int truesize);
2651
2652#define SKB_LINEAR_ASSERT(skb) BUG_ON(skb_is_nonlinear(skb))
2653
2654#ifdef NET_SKBUFF_DATA_USES_OFFSET
2655static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb)
2656{
2657 return skb->head + skb->tail;
2658}
2659
2660static inline void skb_reset_tail_pointer(struct sk_buff *skb)
2661{
2662 skb->tail = skb->data - skb->head;
2663}
2664
2665static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset)
2666{
2667 skb_reset_tail_pointer(skb);
2668 skb->tail += offset;
2669}
2670
2671#else /* NET_SKBUFF_DATA_USES_OFFSET */
2672static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb)
2673{
2674 return skb->tail;
2675}
2676
2677static inline void skb_reset_tail_pointer(struct sk_buff *skb)
2678{
2679 skb->tail = skb->data;
2680}
2681
2682static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset)
2683{
2684 skb->tail = skb->data + offset;
2685}
2686
2687#endif /* NET_SKBUFF_DATA_USES_OFFSET */
2688
2689static inline void skb_assert_len(struct sk_buff *skb)
2690{
2691#ifdef CONFIG_DEBUG_NET
2692 if (WARN_ONCE(!skb->len, "%s\n", __func__))
2693 DO_ONCE_LITE(skb_dump, KERN_ERR, skb, false);
2694#endif /* CONFIG_DEBUG_NET */
2695}
2696
2697#if defined(CONFIG_FAIL_SKB_REALLOC)
2698void skb_might_realloc(struct sk_buff *skb);
2699#else
2700static inline void skb_might_realloc(struct sk_buff *skb) {}
2701#endif
2702
2703/*
2704 * Add data to an sk_buff
2705 */
2706void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len);
2707void *skb_put(struct sk_buff *skb, unsigned int len);
2708static inline void *__skb_put(struct sk_buff *skb, unsigned int len)
2709{
2710 void *tmp = skb_tail_pointer(skb);
2711 SKB_LINEAR_ASSERT(skb);
2712 skb->tail += len;
2713 skb->len += len;
2714 return tmp;
2715}
2716
2717static inline void *__skb_put_zero(struct sk_buff *skb, unsigned int len)
2718{
2719 void *tmp = __skb_put(skb, len);
2720
2721 memset(tmp, 0, len);
2722 return tmp;
2723}
2724
2725static inline void *__skb_put_data(struct sk_buff *skb, const void *data,
2726 unsigned int len)
2727{
2728 void *tmp = __skb_put(skb, len);
2729
2730 memcpy(tmp, data, len);
2731 return tmp;
2732}
2733
2734static inline void __skb_put_u8(struct sk_buff *skb, u8 val)
2735{
2736 *(u8 *)__skb_put(skb, len: 1) = val;
2737}
2738
2739static inline void *skb_put_zero(struct sk_buff *skb, unsigned int len)
2740{
2741 void *tmp = skb_put(skb, len);
2742
2743 memset(tmp, 0, len);
2744
2745 return tmp;
2746}
2747
2748static inline void *skb_put_data(struct sk_buff *skb, const void *data,
2749 unsigned int len)
2750{
2751 void *tmp = skb_put(skb, len);
2752
2753 memcpy(tmp, data, len);
2754
2755 return tmp;
2756}
2757
2758static inline void skb_put_u8(struct sk_buff *skb, u8 val)
2759{
2760 *(u8 *)skb_put(skb, len: 1) = val;
2761}
2762
2763void *skb_push(struct sk_buff *skb, unsigned int len);
2764static inline void *__skb_push(struct sk_buff *skb, unsigned int len)
2765{
2766 DEBUG_NET_WARN_ON_ONCE(len > INT_MAX);
2767
2768 skb->data -= len;
2769 skb->len += len;
2770 return skb->data;
2771}
2772
2773void *skb_pull(struct sk_buff *skb, unsigned int len);
2774static inline void *__skb_pull(struct sk_buff *skb, unsigned int len)
2775{
2776 DEBUG_NET_WARN_ON_ONCE(len > INT_MAX);
2777
2778 skb->len -= len;
2779 if (unlikely(skb->len < skb->data_len)) {
2780#if defined(CONFIG_DEBUG_NET)
2781 skb->len += len;
2782 pr_err("__skb_pull(len=%u)\n", len);
2783 skb_dump(KERN_ERR, skb, full_pkt: false);
2784#endif
2785 BUG();
2786 }
2787 return skb->data += len;
2788}
2789
2790static inline void *skb_pull_inline(struct sk_buff *skb, unsigned int len)
2791{
2792 return unlikely(len > skb->len) ? NULL : __skb_pull(skb, len);
2793}
2794
2795void *skb_pull_data(struct sk_buff *skb, size_t len);
2796
2797void *__pskb_pull_tail(struct sk_buff *skb, int delta);
2798
2799static inline enum skb_drop_reason
2800pskb_may_pull_reason(struct sk_buff *skb, unsigned int len)
2801{
2802 DEBUG_NET_WARN_ON_ONCE(len > INT_MAX);
2803 skb_might_realloc(skb);
2804
2805 if (likely(len <= skb_headlen(skb)))
2806 return SKB_NOT_DROPPED_YET;
2807
2808 if (unlikely(len > skb->len))
2809 return SKB_DROP_REASON_PKT_TOO_SMALL;
2810
2811 if (unlikely(!__pskb_pull_tail(skb, len - skb_headlen(skb))))
2812 return SKB_DROP_REASON_NOMEM;
2813
2814 return SKB_NOT_DROPPED_YET;
2815}
2816
2817static inline bool pskb_may_pull(struct sk_buff *skb, unsigned int len)
2818{
2819 return pskb_may_pull_reason(skb, len) == SKB_NOT_DROPPED_YET;
2820}
2821
2822static inline void *pskb_pull(struct sk_buff *skb, unsigned int len)
2823{
2824 if (!pskb_may_pull(skb, len))
2825 return NULL;
2826
2827 skb->len -= len;
2828 return skb->data += len;
2829}
2830
2831void skb_condense(struct sk_buff *skb);
2832
2833/**
2834 * skb_headroom - bytes at buffer head
2835 * @skb: buffer to check
2836 *
2837 * Return the number of bytes of free space at the head of an &sk_buff.
2838 */
2839static inline unsigned int skb_headroom(const struct sk_buff *skb)
2840{
2841 return skb->data - skb->head;
2842}
2843
2844/**
2845 * skb_tailroom - bytes at buffer end
2846 * @skb: buffer to check
2847 *
2848 * Return the number of bytes of free space at the tail of an sk_buff
2849 */
2850static inline int skb_tailroom(const struct sk_buff *skb)
2851{
2852 return skb_is_nonlinear(skb) ? 0 : skb->end - skb->tail;
2853}
2854
2855/**
2856 * skb_availroom - bytes at buffer end
2857 * @skb: buffer to check
2858 *
2859 * Return the number of bytes of free space at the tail of an sk_buff
2860 * allocated by sk_stream_alloc()
2861 */
2862static inline int skb_availroom(const struct sk_buff *skb)
2863{
2864 if (skb_is_nonlinear(skb))
2865 return 0;
2866
2867 return skb->end - skb->tail - skb->reserved_tailroom;
2868}
2869
2870/**
2871 * skb_reserve - adjust headroom
2872 * @skb: buffer to alter
2873 * @len: bytes to move
2874 *
2875 * Increase the headroom of an empty &sk_buff by reducing the tail
2876 * room. This is only allowed for an empty buffer.
2877 */
2878static inline void skb_reserve(struct sk_buff *skb, int len)
2879{
2880 skb->data += len;
2881 skb->tail += len;
2882}
2883
2884/**
2885 * skb_tailroom_reserve - adjust reserved_tailroom
2886 * @skb: buffer to alter
2887 * @mtu: maximum amount of headlen permitted
2888 * @needed_tailroom: minimum amount of reserved_tailroom
2889 *
2890 * Set reserved_tailroom so that headlen can be as large as possible but
2891 * not larger than mtu and tailroom cannot be smaller than
2892 * needed_tailroom.
2893 * The required headroom should already have been reserved before using
2894 * this function.
2895 */
2896static inline void skb_tailroom_reserve(struct sk_buff *skb, unsigned int mtu,
2897 unsigned int needed_tailroom)
2898{
2899 SKB_LINEAR_ASSERT(skb);
2900 if (mtu < skb_tailroom(skb) - needed_tailroom)
2901 /* use at most mtu */
2902 skb->reserved_tailroom = skb_tailroom(skb) - mtu;
2903 else
2904 /* use up to all available space */
2905 skb->reserved_tailroom = needed_tailroom;
2906}
2907
2908#define ENCAP_TYPE_ETHER 0
2909#define ENCAP_TYPE_IPPROTO 1
2910
2911static inline void skb_set_inner_protocol(struct sk_buff *skb,
2912 __be16 protocol)
2913{
2914 skb->inner_protocol = protocol;
2915 skb->inner_protocol_type = ENCAP_TYPE_ETHER;
2916}
2917
2918static inline void skb_set_inner_ipproto(struct sk_buff *skb,
2919 __u8 ipproto)
2920{
2921 skb->inner_ipproto = ipproto;
2922 skb->inner_protocol_type = ENCAP_TYPE_IPPROTO;
2923}
2924
2925static inline void skb_reset_inner_headers(struct sk_buff *skb)
2926{
2927 skb->inner_mac_header = skb->mac_header;
2928 skb->inner_network_header = skb->network_header;
2929 skb->inner_transport_header = skb->transport_header;
2930}
2931
2932static inline int skb_mac_header_was_set(const struct sk_buff *skb)
2933{
2934 return skb->mac_header != (typeof(skb->mac_header))~0U;
2935}
2936
2937static inline void skb_reset_mac_len(struct sk_buff *skb)
2938{
2939 if (!skb_mac_header_was_set(skb)) {
2940 DEBUG_NET_WARN_ON_ONCE(1);
2941 skb->mac_len = 0;
2942 } else {
2943 skb->mac_len = skb->network_header - skb->mac_header;
2944 }
2945}
2946
2947static inline unsigned char *skb_inner_transport_header(const struct sk_buff
2948 *skb)
2949{
2950 return skb->head + skb->inner_transport_header;
2951}
2952
2953static inline int skb_inner_transport_offset(const struct sk_buff *skb)
2954{
2955 return skb_inner_transport_header(skb) - skb->data;
2956}
2957
2958static inline void skb_reset_inner_transport_header(struct sk_buff *skb)
2959{
2960 long offset = skb->data - skb->head;
2961
2962 DEBUG_NET_WARN_ON_ONCE(offset != (typeof(skb->inner_transport_header))offset);
2963 skb->inner_transport_header = offset;
2964}
2965
2966static inline void skb_set_inner_transport_header(struct sk_buff *skb,
2967 const int offset)
2968{
2969 skb_reset_inner_transport_header(skb);
2970 skb->inner_transport_header += offset;
2971}
2972
2973static inline unsigned char *skb_inner_network_header(const struct sk_buff *skb)
2974{
2975 return skb->head + skb->inner_network_header;
2976}
2977
2978static inline void skb_reset_inner_network_header(struct sk_buff *skb)
2979{
2980 long offset = skb->data - skb->head;
2981
2982 DEBUG_NET_WARN_ON_ONCE(offset != (typeof(skb->inner_network_header))offset);
2983 skb->inner_network_header = offset;
2984}
2985
2986static inline void skb_set_inner_network_header(struct sk_buff *skb,
2987 const int offset)
2988{
2989 skb_reset_inner_network_header(skb);
2990 skb->inner_network_header += offset;
2991}
2992
2993static inline bool skb_inner_network_header_was_set(const struct sk_buff *skb)
2994{
2995 return skb->inner_network_header > 0;
2996}
2997
2998static inline unsigned char *skb_inner_mac_header(const struct sk_buff *skb)
2999{
3000 return skb->head + skb->inner_mac_header;
3001}
3002
3003static inline void skb_reset_inner_mac_header(struct sk_buff *skb)
3004{
3005 long offset = skb->data - skb->head;
3006
3007 DEBUG_NET_WARN_ON_ONCE(offset != (typeof(skb->inner_mac_header))offset);
3008 skb->inner_mac_header = offset;
3009}
3010
3011static inline void skb_set_inner_mac_header(struct sk_buff *skb,
3012 const int offset)
3013{
3014 skb_reset_inner_mac_header(skb);
3015 skb->inner_mac_header += offset;
3016}
3017static inline bool skb_transport_header_was_set(const struct sk_buff *skb)
3018{
3019 return skb->transport_header != (typeof(skb->transport_header))~0U;
3020}
3021
3022static inline unsigned char *skb_transport_header(const struct sk_buff *skb)
3023{
3024 DEBUG_NET_WARN_ON_ONCE(!skb_transport_header_was_set(skb));
3025 return skb->head + skb->transport_header;
3026}
3027
3028static inline void skb_reset_transport_header(struct sk_buff *skb)
3029{
3030 long offset = skb->data - skb->head;
3031
3032 DEBUG_NET_WARN_ON_ONCE(offset != (typeof(skb->transport_header))offset);
3033 skb->transport_header = offset;
3034}
3035
3036static inline void skb_set_transport_header(struct sk_buff *skb,
3037 const int offset)
3038{
3039 skb_reset_transport_header(skb);
3040 skb->transport_header += offset;
3041}
3042
3043static inline unsigned char *skb_network_header(const struct sk_buff *skb)
3044{
3045 return skb->head + skb->network_header;
3046}
3047
3048static inline void skb_reset_network_header(struct sk_buff *skb)
3049{
3050 long offset = skb->data - skb->head;
3051
3052 DEBUG_NET_WARN_ON_ONCE(offset != (typeof(skb->network_header))offset);
3053 skb->network_header = offset;
3054}
3055
3056static inline void skb_set_network_header(struct sk_buff *skb, const int offset)
3057{
3058 skb_reset_network_header(skb);
3059 skb->network_header += offset;
3060}
3061
3062static inline unsigned char *skb_mac_header(const struct sk_buff *skb)
3063{
3064 DEBUG_NET_WARN_ON_ONCE(!skb_mac_header_was_set(skb));
3065 return skb->head + skb->mac_header;
3066}
3067
3068static inline int skb_mac_offset(const struct sk_buff *skb)
3069{
3070 return skb_mac_header(skb) - skb->data;
3071}
3072
3073static inline u32 skb_mac_header_len(const struct sk_buff *skb)
3074{
3075 DEBUG_NET_WARN_ON_ONCE(!skb_mac_header_was_set(skb));
3076 return skb->network_header - skb->mac_header;
3077}
3078
3079static inline void skb_unset_mac_header(struct sk_buff *skb)
3080{
3081 skb->mac_header = (typeof(skb->mac_header))~0U;
3082}
3083
3084static inline void skb_reset_mac_header(struct sk_buff *skb)
3085{
3086 long offset = skb->data - skb->head;
3087
3088 DEBUG_NET_WARN_ON_ONCE(offset != (typeof(skb->mac_header))offset);
3089 skb->mac_header = offset;
3090}
3091
3092static inline void skb_set_mac_header(struct sk_buff *skb, const int offset)
3093{
3094 skb_reset_mac_header(skb);
3095 skb->mac_header += offset;
3096}
3097
3098static inline void skb_pop_mac_header(struct sk_buff *skb)
3099{
3100 skb->mac_header = skb->network_header;
3101}
3102
3103static inline void skb_probe_transport_header(struct sk_buff *skb)
3104{
3105 struct flow_keys_basic keys;
3106
3107 if (skb_transport_header_was_set(skb))
3108 return;
3109
3110 if (skb_flow_dissect_flow_keys_basic(NULL, skb, flow: &keys,
3111 NULL, proto: 0, nhoff: 0, hlen: 0, flags: 0))
3112 skb_set_transport_header(skb, offset: keys.control.thoff);
3113}
3114
3115static inline void skb_mac_header_rebuild(struct sk_buff *skb)
3116{
3117 if (skb_mac_header_was_set(skb)) {
3118 const unsigned char *old_mac = skb_mac_header(skb);
3119
3120 skb_set_mac_header(skb, offset: -skb->mac_len);
3121 memmove(skb_mac_header(skb), old_mac, skb->mac_len);
3122 }
3123}
3124
3125/* Move the full mac header up to current network_header.
3126 * Leaves skb->data pointing at offset skb->mac_len into the mac_header.
3127 * Must be provided the complete mac header length.
3128 */
3129static inline void skb_mac_header_rebuild_full(struct sk_buff *skb, u32 full_mac_len)
3130{
3131 if (skb_mac_header_was_set(skb)) {
3132 const unsigned char *old_mac = skb_mac_header(skb);
3133
3134 skb_set_mac_header(skb, offset: -full_mac_len);
3135 memmove(skb_mac_header(skb), old_mac, full_mac_len);
3136 __skb_push(skb, len: full_mac_len - skb->mac_len);
3137 }
3138}
3139
3140static inline int skb_checksum_start_offset(const struct sk_buff *skb)
3141{
3142 return skb->csum_start - skb_headroom(skb);
3143}
3144
3145static inline unsigned char *skb_checksum_start(const struct sk_buff *skb)
3146{
3147 return skb->head + skb->csum_start;
3148}
3149
3150static inline int skb_transport_offset(const struct sk_buff *skb)
3151{
3152 return skb_transport_header(skb) - skb->data;
3153}
3154
3155static inline u32 skb_network_header_len(const struct sk_buff *skb)
3156{
3157 DEBUG_NET_WARN_ON_ONCE(!skb_transport_header_was_set(skb));
3158 return skb->transport_header - skb->network_header;
3159}
3160
3161static inline u32 skb_inner_network_header_len(const struct sk_buff *skb)
3162{
3163 return skb->inner_transport_header - skb->inner_network_header;
3164}
3165
3166static inline int skb_network_offset(const struct sk_buff *skb)
3167{
3168 return skb_network_header(skb) - skb->data;
3169}
3170
3171static inline int skb_inner_network_offset(const struct sk_buff *skb)
3172{
3173 return skb_inner_network_header(skb) - skb->data;
3174}
3175
3176static inline enum skb_drop_reason
3177pskb_network_may_pull_reason(struct sk_buff *skb, unsigned int len)
3178{
3179 return pskb_may_pull_reason(skb, len: skb_network_offset(skb) + len);
3180}
3181
3182static inline int pskb_network_may_pull(struct sk_buff *skb, unsigned int len)
3183{
3184 return pskb_network_may_pull_reason(skb, len) == SKB_NOT_DROPPED_YET;
3185}
3186
3187/*
3188 * CPUs often take a performance hit when accessing unaligned memory
3189 * locations. The actual performance hit varies, it can be small if the
3190 * hardware handles it or large if we have to take an exception and fix it
3191 * in software.
3192 *
3193 * Since an ethernet header is 14 bytes network drivers often end up with
3194 * the IP header at an unaligned offset. The IP header can be aligned by
3195 * shifting the start of the packet by 2 bytes. Drivers should do this
3196 * with:
3197 *
3198 * skb_reserve(skb, NET_IP_ALIGN);
3199 *
3200 * The downside to this alignment of the IP header is that the DMA is now
3201 * unaligned. On some architectures the cost of an unaligned DMA is high
3202 * and this cost outweighs the gains made by aligning the IP header.
3203 *
3204 * Since this trade off varies between architectures, we allow NET_IP_ALIGN
3205 * to be overridden.
3206 */
3207#ifndef NET_IP_ALIGN
3208#define NET_IP_ALIGN 2
3209#endif
3210
3211/*
3212 * The networking layer reserves some headroom in skb data (via
3213 * dev_alloc_skb). This is used to avoid having to reallocate skb data when
3214 * the header has to grow. In the default case, if the header has to grow
3215 * 32 bytes or less we avoid the reallocation.
3216 *
3217 * Unfortunately this headroom changes the DMA alignment of the resulting
3218 * network packet. As for NET_IP_ALIGN, this unaligned DMA is expensive
3219 * on some architectures. An architecture can override this value,
3220 * perhaps setting it to a cacheline in size (since that will maintain
3221 * cacheline alignment of the DMA). It must be a power of 2.
3222 *
3223 * Various parts of the networking layer expect at least 32 bytes of
3224 * headroom, you should not reduce this.
3225 *
3226 * Using max(32, L1_CACHE_BYTES) makes sense (especially with RPS)
3227 * to reduce average number of cache lines per packet.
3228 * get_rps_cpu() for example only access one 64 bytes aligned block :
3229 * NET_IP_ALIGN(2) + ethernet_header(14) + IP_header(20/40) + ports(8)
3230 */
3231#ifndef NET_SKB_PAD
3232#define NET_SKB_PAD max(32, L1_CACHE_BYTES)
3233#endif
3234
3235int ___pskb_trim(struct sk_buff *skb, unsigned int len);
3236
3237static inline void __skb_set_length(struct sk_buff *skb, unsigned int len)
3238{
3239 if (WARN_ON(skb_is_nonlinear(skb)))
3240 return;
3241 skb->len = len;
3242 skb_set_tail_pointer(skb, offset: len);
3243}
3244
3245static inline void __skb_trim(struct sk_buff *skb, unsigned int len)
3246{
3247 __skb_set_length(skb, len);
3248}
3249
3250void skb_trim(struct sk_buff *skb, unsigned int len);
3251
3252static inline int __pskb_trim(struct sk_buff *skb, unsigned int len)
3253{
3254 if (skb->data_len)
3255 return ___pskb_trim(skb, len);
3256 __skb_trim(skb, len);
3257 return 0;
3258}
3259
3260static inline int pskb_trim(struct sk_buff *skb, unsigned int len)
3261{
3262 skb_might_realloc(skb);
3263 return (len < skb->len) ? __pskb_trim(skb, len) : 0;
3264}
3265
3266/**
3267 * pskb_trim_unique - remove end from a paged unique (not cloned) buffer
3268 * @skb: buffer to alter
3269 * @len: new length
3270 *
3271 * This is identical to pskb_trim except that the caller knows that
3272 * the skb is not cloned so we should never get an error due to out-
3273 * of-memory.
3274 */
3275static inline void pskb_trim_unique(struct sk_buff *skb, unsigned int len)
3276{
3277 int err = pskb_trim(skb, len);
3278 BUG_ON(err);
3279}
3280
3281static inline int __skb_grow(struct sk_buff *skb, unsigned int len)
3282{
3283 unsigned int diff = len - skb->len;
3284
3285 if (skb_tailroom(skb) < diff) {
3286 int ret = pskb_expand_head(skb, nhead: 0, ntail: diff - skb_tailroom(skb),
3287 GFP_ATOMIC);
3288 if (ret)
3289 return ret;
3290 }
3291 __skb_set_length(skb, len);
3292 return 0;
3293}
3294
3295/**
3296 * skb_orphan - orphan a buffer
3297 * @skb: buffer to orphan
3298 *
3299 * If a buffer currently has an owner then we call the owner's
3300 * destructor function and make the @skb unowned. The buffer continues
3301 * to exist but is no longer charged to its former owner.
3302 */
3303static inline void skb_orphan(struct sk_buff *skb)
3304{
3305 if (skb->destructor) {
3306 skb->destructor(skb);
3307 skb->destructor = NULL;
3308 skb->sk = NULL;
3309 } else {
3310 BUG_ON(skb->sk);
3311 }
3312}
3313
3314/**
3315 * skb_orphan_frags - orphan the frags contained in a buffer
3316 * @skb: buffer to orphan frags from
3317 * @gfp_mask: allocation mask for replacement pages
3318 *
3319 * For each frag in the SKB which needs a destructor (i.e. has an
3320 * owner) create a copy of that frag and release the original
3321 * page by calling the destructor.
3322 */
3323static inline int skb_orphan_frags(struct sk_buff *skb, gfp_t gfp_mask)
3324{
3325 if (likely(!skb_zcopy(skb)))
3326 return 0;
3327 if (skb_shinfo(skb)->flags & SKBFL_DONT_ORPHAN)
3328 return 0;
3329 return skb_copy_ubufs(skb, gfp_mask);
3330}
3331
3332/* Frags must be orphaned, even if refcounted, if skb might loop to rx path */
3333static inline int skb_orphan_frags_rx(struct sk_buff *skb, gfp_t gfp_mask)
3334{
3335 if (likely(!skb_zcopy(skb)))
3336 return 0;
3337 return skb_copy_ubufs(skb, gfp_mask);
3338}
3339
3340/**
3341 * __skb_queue_purge_reason - empty a list
3342 * @list: list to empty
3343 * @reason: drop reason
3344 *
3345 * Delete all buffers on an &sk_buff list. Each buffer is removed from
3346 * the list and one reference dropped. This function does not take the
3347 * list lock and the caller must hold the relevant locks to use it.
3348 */
3349static inline void __skb_queue_purge_reason(struct sk_buff_head *list,
3350 enum skb_drop_reason reason)
3351{
3352 struct sk_buff *skb;
3353
3354 while ((skb = __skb_dequeue(list)) != NULL)
3355 kfree_skb_reason(skb, reason);
3356}
3357
3358static inline void __skb_queue_purge(struct sk_buff_head *list)
3359{
3360 __skb_queue_purge_reason(list, reason: SKB_DROP_REASON_QUEUE_PURGE);
3361}
3362
3363void skb_queue_purge_reason(struct sk_buff_head *list,
3364 enum skb_drop_reason reason);
3365
3366static inline void skb_queue_purge(struct sk_buff_head *list)
3367{
3368 skb_queue_purge_reason(list, reason: SKB_DROP_REASON_QUEUE_PURGE);
3369}
3370
3371unsigned int skb_rbtree_purge(struct rb_root *root);
3372void skb_errqueue_purge(struct sk_buff_head *list);
3373
3374void *__netdev_alloc_frag_align(unsigned int fragsz, unsigned int align_mask);
3375
3376/**
3377 * netdev_alloc_frag - allocate a page fragment
3378 * @fragsz: fragment size
3379 *
3380 * Allocates a frag from a page for receive buffer.
3381 * Uses GFP_ATOMIC allocations.
3382 */
3383static inline void *netdev_alloc_frag(unsigned int fragsz)
3384{
3385 return __netdev_alloc_frag_align(fragsz, align_mask: ~0u);
3386}
3387
3388static inline void *netdev_alloc_frag_align(unsigned int fragsz,
3389 unsigned int align)
3390{
3391 WARN_ON_ONCE(!is_power_of_2(align));
3392 return __netdev_alloc_frag_align(fragsz, align_mask: -align);
3393}
3394
3395struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int length,
3396 gfp_t gfp_mask);
3397
3398/**
3399 * netdev_alloc_skb - allocate an skbuff for rx on a specific device
3400 * @dev: network device to receive on
3401 * @length: length to allocate
3402 *
3403 * Allocate a new &sk_buff and assign it a usage count of one. The
3404 * buffer has unspecified headroom built in. Users should allocate
3405 * the headroom they think they need without accounting for the
3406 * built in space. The built in space is used for optimisations.
3407 *
3408 * %NULL is returned if there is no free memory. Although this function
3409 * allocates memory it can be called from an interrupt.
3410 */
3411static inline struct sk_buff *netdev_alloc_skb(struct net_device *dev,
3412 unsigned int length)
3413{
3414 return __netdev_alloc_skb(dev, length, GFP_ATOMIC);
3415}
3416
3417/* legacy helper around __netdev_alloc_skb() */
3418static inline struct sk_buff *__dev_alloc_skb(unsigned int length,
3419 gfp_t gfp_mask)
3420{
3421 return __netdev_alloc_skb(NULL, length, gfp_mask);
3422}
3423
3424/* legacy helper around netdev_alloc_skb() */
3425static inline struct sk_buff *dev_alloc_skb(unsigned int length)
3426{
3427 return netdev_alloc_skb(NULL, length);
3428}
3429
3430
3431static inline struct sk_buff *__netdev_alloc_skb_ip_align(struct net_device *dev,
3432 unsigned int length, gfp_t gfp)
3433{
3434 struct sk_buff *skb = __netdev_alloc_skb(dev, length: length + NET_IP_ALIGN, gfp_mask: gfp);
3435
3436 if (NET_IP_ALIGN && skb)
3437 skb_reserve(skb, NET_IP_ALIGN);
3438 return skb;
3439}
3440
3441static inline struct sk_buff *netdev_alloc_skb_ip_align(struct net_device *dev,
3442 unsigned int length)
3443{
3444 return __netdev_alloc_skb_ip_align(dev, length, GFP_ATOMIC);
3445}
3446
3447static inline void skb_free_frag(void *addr)
3448{
3449 page_frag_free(addr);
3450}
3451
3452void *__napi_alloc_frag_align(unsigned int fragsz, unsigned int align_mask);
3453
3454static inline void *napi_alloc_frag(unsigned int fragsz)
3455{
3456 return __napi_alloc_frag_align(fragsz, align_mask: ~0u);
3457}
3458
3459static inline void *napi_alloc_frag_align(unsigned int fragsz,
3460 unsigned int align)
3461{
3462 WARN_ON_ONCE(!is_power_of_2(align));
3463 return __napi_alloc_frag_align(fragsz, align_mask: -align);
3464}
3465
3466struct sk_buff *napi_alloc_skb(struct napi_struct *napi, unsigned int length);
3467void napi_consume_skb(struct sk_buff *skb, int budget);
3468
3469void napi_skb_free_stolen_head(struct sk_buff *skb);
3470void __napi_kfree_skb(struct sk_buff *skb, enum skb_drop_reason reason);
3471
3472/**
3473 * __dev_alloc_pages - allocate page for network Rx
3474 * @gfp_mask: allocation priority. Set __GFP_NOMEMALLOC if not for network Rx
3475 * @order: size of the allocation
3476 *
3477 * Allocate a new page.
3478 *
3479 * %NULL is returned if there is no free memory.
3480*/
3481static inline struct page *__dev_alloc_pages_noprof(gfp_t gfp_mask,
3482 unsigned int order)
3483{
3484 /* This piece of code contains several assumptions.
3485 * 1. This is for device Rx, therefore a cold page is preferred.
3486 * 2. The expectation is the user wants a compound page.
3487 * 3. If requesting a order 0 page it will not be compound
3488 * due to the check to see if order has a value in prep_new_page
3489 * 4. __GFP_MEMALLOC is ignored if __GFP_NOMEMALLOC is set due to
3490 * code in gfp_to_alloc_flags that should be enforcing this.
3491 */
3492 gfp_mask |= __GFP_COMP | __GFP_MEMALLOC;
3493
3494 return alloc_pages_node_noprof(NUMA_NO_NODE, gfp_mask, order);
3495}
3496#define __dev_alloc_pages(...) alloc_hooks(__dev_alloc_pages_noprof(__VA_ARGS__))
3497
3498/*
3499 * This specialized allocator has to be a macro for its allocations to be
3500 * accounted separately (to have a separate alloc_tag).
3501 */
3502#define dev_alloc_pages(_order) __dev_alloc_pages(GFP_ATOMIC | __GFP_NOWARN, _order)
3503
3504/**
3505 * __dev_alloc_page - allocate a page for network Rx
3506 * @gfp_mask: allocation priority. Set __GFP_NOMEMALLOC if not for network Rx
3507 *
3508 * Allocate a new page.
3509 *
3510 * %NULL is returned if there is no free memory.
3511 */
3512static inline struct page *__dev_alloc_page_noprof(gfp_t gfp_mask)
3513{
3514 return __dev_alloc_pages_noprof(gfp_mask, order: 0);
3515}
3516#define __dev_alloc_page(...) alloc_hooks(__dev_alloc_page_noprof(__VA_ARGS__))
3517
3518/*
3519 * This specialized allocator has to be a macro for its allocations to be
3520 * accounted separately (to have a separate alloc_tag).
3521 */
3522#define dev_alloc_page() dev_alloc_pages(0)
3523
3524/**
3525 * dev_page_is_reusable - check whether a page can be reused for network Rx
3526 * @page: the page to test
3527 *
3528 * A page shouldn't be considered for reusing/recycling if it was allocated
3529 * under memory pressure or at a distant memory node.
3530 *
3531 * Returns: false if this page should be returned to page allocator, true
3532 * otherwise.
3533 */
3534static inline bool dev_page_is_reusable(const struct page *page)
3535{
3536 return likely(page_to_nid(page) == numa_mem_id() &&
3537 !page_is_pfmemalloc(page));
3538}
3539
3540/**
3541 * skb_propagate_pfmemalloc - Propagate pfmemalloc if skb is allocated after RX page
3542 * @page: The page that was allocated from skb_alloc_page
3543 * @skb: The skb that may need pfmemalloc set
3544 */
3545static inline void skb_propagate_pfmemalloc(const struct page *page,
3546 struct sk_buff *skb)
3547{
3548 if (page_is_pfmemalloc(page))
3549 skb->pfmemalloc = true;
3550}
3551
3552/**
3553 * skb_frag_off() - Returns the offset of a skb fragment
3554 * @frag: the paged fragment
3555 */
3556static inline unsigned int skb_frag_off(const skb_frag_t *frag)
3557{
3558 return frag->offset;
3559}
3560
3561/**
3562 * skb_frag_off_add() - Increments the offset of a skb fragment by @delta
3563 * @frag: skb fragment
3564 * @delta: value to add
3565 */
3566static inline void skb_frag_off_add(skb_frag_t *frag, int delta)
3567{
3568 frag->offset += delta;
3569}
3570
3571/**
3572 * skb_frag_off_set() - Sets the offset of a skb fragment
3573 * @frag: skb fragment
3574 * @offset: offset of fragment
3575 */
3576static inline void skb_frag_off_set(skb_frag_t *frag, unsigned int offset)
3577{
3578 frag->offset = offset;
3579}
3580
3581/**
3582 * skb_frag_off_copy() - Sets the offset of a skb fragment from another fragment
3583 * @fragto: skb fragment where offset is set
3584 * @fragfrom: skb fragment offset is copied from
3585 */
3586static inline void skb_frag_off_copy(skb_frag_t *fragto,
3587 const skb_frag_t *fragfrom)
3588{
3589 fragto->offset = fragfrom->offset;
3590}
3591
3592/* Return: true if the skb_frag contains a net_iov. */
3593static inline bool skb_frag_is_net_iov(const skb_frag_t *frag)
3594{
3595 return netmem_is_net_iov(netmem: frag->netmem);
3596}
3597
3598/**
3599 * skb_frag_net_iov - retrieve the net_iov referred to by fragment
3600 * @frag: the fragment
3601 *
3602 * Return: the &struct net_iov associated with @frag. Returns NULL if this
3603 * frag has no associated net_iov.
3604 */
3605static inline struct net_iov *skb_frag_net_iov(const skb_frag_t *frag)
3606{
3607 if (!skb_frag_is_net_iov(frag))
3608 return NULL;
3609
3610 return netmem_to_net_iov(netmem: frag->netmem);
3611}
3612
3613/**
3614 * skb_frag_page - retrieve the page referred to by a paged fragment
3615 * @frag: the paged fragment
3616 *
3617 * Return: the &struct page associated with @frag. Returns NULL if this frag
3618 * has no associated page.
3619 */
3620static inline struct page *skb_frag_page(const skb_frag_t *frag)
3621{
3622 if (skb_frag_is_net_iov(frag))
3623 return NULL;
3624
3625 return netmem_to_page(netmem: frag->netmem);
3626}
3627
3628/**
3629 * skb_frag_netmem - retrieve the netmem referred to by a fragment
3630 * @frag: the fragment
3631 *
3632 * Return: the &netmem_ref associated with @frag.
3633 */
3634static inline netmem_ref skb_frag_netmem(const skb_frag_t *frag)
3635{
3636 return frag->netmem;
3637}
3638
3639int skb_pp_cow_data(struct page_pool *pool, struct sk_buff **pskb,
3640 unsigned int headroom);
3641int skb_cow_data_for_xdp(struct page_pool *pool, struct sk_buff **pskb,
3642 const struct bpf_prog *prog);
3643
3644/**
3645 * skb_frag_address - gets the address of the data contained in a paged fragment
3646 * @frag: the paged fragment buffer
3647 *
3648 * Returns: the address of the data within @frag. The page must already
3649 * be mapped.
3650 */
3651static inline void *skb_frag_address(const skb_frag_t *frag)
3652{
3653 if (!skb_frag_page(frag))
3654 return NULL;
3655
3656 return page_address(skb_frag_page(frag)) + skb_frag_off(frag);
3657}
3658
3659/**
3660 * skb_frag_address_safe - gets the address of the data contained in a paged fragment
3661 * @frag: the paged fragment buffer
3662 *
3663 * Returns: the address of the data within @frag. Checks that the page
3664 * is mapped and returns %NULL otherwise.
3665 */
3666static inline void *skb_frag_address_safe(const skb_frag_t *frag)
3667{
3668 void *ptr = page_address(skb_frag_page(frag));
3669 if (unlikely(!ptr))
3670 return NULL;
3671
3672 return ptr + skb_frag_off(frag);
3673}
3674
3675/**
3676 * skb_frag_page_copy() - sets the page in a fragment from another fragment
3677 * @fragto: skb fragment where page is set
3678 * @fragfrom: skb fragment page is copied from
3679 */
3680static inline void skb_frag_page_copy(skb_frag_t *fragto,
3681 const skb_frag_t *fragfrom)
3682{
3683 fragto->netmem = fragfrom->netmem;
3684}
3685
3686bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio);
3687
3688/**
3689 * __skb_frag_dma_map - maps a paged fragment via the DMA API
3690 * @dev: the device to map the fragment to
3691 * @frag: the paged fragment to map
3692 * @offset: the offset within the fragment (starting at the
3693 * fragment's own offset)
3694 * @size: the number of bytes to map
3695 * @dir: the direction of the mapping (``PCI_DMA_*``)
3696 *
3697 * Maps the page associated with @frag to @device.
3698 */
3699static inline dma_addr_t __skb_frag_dma_map(struct device *dev,
3700 const skb_frag_t *frag,
3701 size_t offset, size_t size,
3702 enum dma_data_direction dir)
3703{
3704 if (skb_frag_is_net_iov(frag)) {
3705 return netmem_to_net_iov(netmem: frag->netmem)->dma_addr + offset +
3706 frag->offset;
3707 }
3708 return dma_map_page(dev, skb_frag_page(frag),
3709 skb_frag_off(frag) + offset, size, dir);
3710}
3711
3712#define skb_frag_dma_map(dev, frag, ...) \
3713 CONCATENATE(_skb_frag_dma_map, \
3714 COUNT_ARGS(__VA_ARGS__))(dev, frag, ##__VA_ARGS__)
3715
3716#define __skb_frag_dma_map1(dev, frag, offset, uf, uo) ({ \
3717 const skb_frag_t *uf = (frag); \
3718 size_t uo = (offset); \
3719 \
3720 __skb_frag_dma_map(dev, uf, uo, skb_frag_size(uf) - uo, \
3721 DMA_TO_DEVICE); \
3722})
3723#define _skb_frag_dma_map1(dev, frag, offset) \
3724 __skb_frag_dma_map1(dev, frag, offset, __UNIQUE_ID(frag_), \
3725 __UNIQUE_ID(offset_))
3726#define _skb_frag_dma_map0(dev, frag) \
3727 _skb_frag_dma_map1(dev, frag, 0)
3728#define _skb_frag_dma_map2(dev, frag, offset, size) \
3729 __skb_frag_dma_map(dev, frag, offset, size, DMA_TO_DEVICE)
3730#define _skb_frag_dma_map3(dev, frag, offset, size, dir) \
3731 __skb_frag_dma_map(dev, frag, offset, size, dir)
3732
3733static inline struct sk_buff *pskb_copy(struct sk_buff *skb,
3734 gfp_t gfp_mask)
3735{
3736 return __pskb_copy(skb, headroom: skb_headroom(skb), gfp_mask);
3737}
3738
3739
3740static inline struct sk_buff *pskb_copy_for_clone(struct sk_buff *skb,
3741 gfp_t gfp_mask)
3742{
3743 return __pskb_copy_fclone(skb, headroom: skb_headroom(skb), gfp_mask, fclone: true);
3744}
3745
3746
3747/**
3748 * skb_clone_writable - is the header of a clone writable
3749 * @skb: buffer to check
3750 * @len: length up to which to write
3751 *
3752 * Returns true if modifying the header part of the cloned buffer
3753 * does not requires the data to be copied.
3754 */
3755static inline int skb_clone_writable(const struct sk_buff *skb, unsigned int len)
3756{
3757 return !skb_header_cloned(skb) &&
3758 skb_headroom(skb) + len <= skb->hdr_len;
3759}
3760
3761static inline int skb_try_make_writable(struct sk_buff *skb,
3762 unsigned int write_len)
3763{
3764 return skb_cloned(skb) && !skb_clone_writable(skb, len: write_len) &&
3765 pskb_expand_head(skb, nhead: 0, ntail: 0, GFP_ATOMIC);
3766}
3767
3768static inline int __skb_cow(struct sk_buff *skb, unsigned int headroom,
3769 int cloned)
3770{
3771 int delta = 0;
3772
3773 if (headroom > skb_headroom(skb))
3774 delta = headroom - skb_headroom(skb);
3775
3776 if (delta || cloned)
3777 return pskb_expand_head(skb, ALIGN(delta, NET_SKB_PAD), ntail: 0,
3778 GFP_ATOMIC);
3779 return 0;
3780}
3781
3782/**
3783 * skb_cow - copy header of skb when it is required
3784 * @skb: buffer to cow
3785 * @headroom: needed headroom
3786 *
3787 * If the skb passed lacks sufficient headroom or its data part
3788 * is shared, data is reallocated. If reallocation fails, an error
3789 * is returned and original skb is not changed.
3790 *
3791 * The result is skb with writable area skb->head...skb->tail
3792 * and at least @headroom of space at head.
3793 */
3794static inline int skb_cow(struct sk_buff *skb, unsigned int headroom)
3795{
3796 return __skb_cow(skb, headroom, cloned: skb_cloned(skb));
3797}
3798
3799/**
3800 * skb_cow_head - skb_cow but only making the head writable
3801 * @skb: buffer to cow
3802 * @headroom: needed headroom
3803 *
3804 * This function is identical to skb_cow except that we replace the
3805 * skb_cloned check by skb_header_cloned. It should be used when
3806 * you only need to push on some header and do not need to modify
3807 * the data.
3808 */
3809static inline int skb_cow_head(struct sk_buff *skb, unsigned int headroom)
3810{
3811 return __skb_cow(skb, headroom, cloned: skb_header_cloned(skb));
3812}
3813
3814/**
3815 * skb_padto - pad an skbuff up to a minimal size
3816 * @skb: buffer to pad
3817 * @len: minimal length
3818 *
3819 * Pads up a buffer to ensure the trailing bytes exist and are
3820 * blanked. If the buffer already contains sufficient data it
3821 * is untouched. Otherwise it is extended. Returns zero on
3822 * success. The skb is freed on error.
3823 */
3824static inline int skb_padto(struct sk_buff *skb, unsigned int len)
3825{
3826 unsigned int size = skb->len;
3827 if (likely(size >= len))
3828 return 0;
3829 return skb_pad(skb, pad: len - size);
3830}
3831
3832/**
3833 * __skb_put_padto - increase size and pad an skbuff up to a minimal size
3834 * @skb: buffer to pad
3835 * @len: minimal length
3836 * @free_on_error: free buffer on error
3837 *
3838 * Pads up a buffer to ensure the trailing bytes exist and are
3839 * blanked. If the buffer already contains sufficient data it
3840 * is untouched. Otherwise it is extended. Returns zero on
3841 * success. The skb is freed on error if @free_on_error is true.
3842 */
3843static inline int __must_check __skb_put_padto(struct sk_buff *skb,
3844 unsigned int len,
3845 bool free_on_error)
3846{
3847 unsigned int size = skb->len;
3848
3849 if (unlikely(size < len)) {
3850 len -= size;
3851 if (__skb_pad(skb, pad: len, free_on_error))
3852 return -ENOMEM;
3853 __skb_put(skb, len);
3854 }
3855 return 0;
3856}
3857
3858/**
3859 * skb_put_padto - increase size and pad an skbuff up to a minimal size
3860 * @skb: buffer to pad
3861 * @len: minimal length
3862 *
3863 * Pads up a buffer to ensure the trailing bytes exist and are
3864 * blanked. If the buffer already contains sufficient data it
3865 * is untouched. Otherwise it is extended. Returns zero on
3866 * success. The skb is freed on error.
3867 */
3868static inline int __must_check skb_put_padto(struct sk_buff *skb, unsigned int len)
3869{
3870 return __skb_put_padto(skb, len, free_on_error: true);
3871}
3872
3873bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum, struct iov_iter *i)
3874 __must_check;
3875
3876static inline bool skb_can_coalesce(struct sk_buff *skb, int i,
3877 const struct page *page, int off)
3878{
3879 if (skb_zcopy(skb))
3880 return false;
3881 if (i) {
3882 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1];
3883
3884 return page == skb_frag_page(frag) &&
3885 off == skb_frag_off(frag) + skb_frag_size(frag);
3886 }
3887 return false;
3888}
3889
3890static inline int __skb_linearize(struct sk_buff *skb)
3891{
3892 return __pskb_pull_tail(skb, delta: skb->data_len) ? 0 : -ENOMEM;
3893}
3894
3895/**
3896 * skb_linearize - convert paged skb to linear one
3897 * @skb: buffer to linarize
3898 *
3899 * If there is no free memory -ENOMEM is returned, otherwise zero
3900 * is returned and the old skb data released.
3901 */
3902static inline int skb_linearize(struct sk_buff *skb)
3903{
3904 return skb_is_nonlinear(skb) ? __skb_linearize(skb) : 0;
3905}
3906
3907/**
3908 * skb_has_shared_frag - can any frag be overwritten
3909 * @skb: buffer to test
3910 *
3911 * Return: true if the skb has at least one frag that might be modified
3912 * by an external entity (as in vmsplice()/sendfile())
3913 */
3914static inline bool skb_has_shared_frag(const struct sk_buff *skb)
3915{
3916 return skb_is_nonlinear(skb) &&
3917 skb_shinfo(skb)->flags & SKBFL_SHARED_FRAG;
3918}
3919
3920/**
3921 * skb_linearize_cow - make sure skb is linear and writable
3922 * @skb: buffer to process
3923 *
3924 * If there is no free memory -ENOMEM is returned, otherwise zero
3925 * is returned and the old skb data released.
3926 */
3927static inline int skb_linearize_cow(struct sk_buff *skb)
3928{
3929 return skb_is_nonlinear(skb) || skb_cloned(skb) ?
3930 __skb_linearize(skb) : 0;
3931}
3932
3933static __always_inline void
3934__skb_postpull_rcsum(struct sk_buff *skb, const void *start, unsigned int len,
3935 unsigned int off)
3936{
3937 if (skb->ip_summed == CHECKSUM_COMPLETE)
3938 skb->csum = csum_block_sub(csum: skb->csum,
3939 csum2: csum_partial(buff: start, len, sum: 0), offset: off);
3940 else if (skb->ip_summed == CHECKSUM_PARTIAL &&
3941 skb_checksum_start_offset(skb) < 0)
3942 skb->ip_summed = CHECKSUM_NONE;
3943}
3944
3945/**
3946 * skb_postpull_rcsum - update checksum for received skb after pull
3947 * @skb: buffer to update
3948 * @start: start of data before pull
3949 * @len: length of data pulled
3950 *
3951 * After doing a pull on a received packet, you need to call this to
3952 * update the CHECKSUM_COMPLETE checksum, or set ip_summed to
3953 * CHECKSUM_NONE so that it can be recomputed from scratch.
3954 */
3955static inline void skb_postpull_rcsum(struct sk_buff *skb,
3956 const void *start, unsigned int len)
3957{
3958 if (skb->ip_summed == CHECKSUM_COMPLETE)
3959 skb->csum = wsum_negate(val: csum_partial(buff: start, len,
3960 sum: wsum_negate(val: skb->csum)));
3961 else if (skb->ip_summed == CHECKSUM_PARTIAL &&
3962 skb_checksum_start_offset(skb) < 0)
3963 skb->ip_summed = CHECKSUM_NONE;
3964}
3965
3966static __always_inline void
3967__skb_postpush_rcsum(struct sk_buff *skb, const void *start, unsigned int len,
3968 unsigned int off)
3969{
3970 if (skb->ip_summed == CHECKSUM_COMPLETE)
3971 skb->csum = csum_block_add(csum: skb->csum,
3972 csum2: csum_partial(buff: start, len, sum: 0), offset: off);
3973}
3974
3975/**
3976 * skb_postpush_rcsum - update checksum for received skb after push
3977 * @skb: buffer to update
3978 * @start: start of data after push
3979 * @len: length of data pushed
3980 *
3981 * After doing a push on a received packet, you need to call this to
3982 * update the CHECKSUM_COMPLETE checksum.
3983 */
3984static inline void skb_postpush_rcsum(struct sk_buff *skb,
3985 const void *start, unsigned int len)
3986{
3987 __skb_postpush_rcsum(skb, start, len, off: 0);
3988}
3989
3990void *skb_pull_rcsum(struct sk_buff *skb, unsigned int len);
3991
3992/**
3993 * skb_push_rcsum - push skb and update receive checksum
3994 * @skb: buffer to update
3995 * @len: length of data pulled
3996 *
3997 * This function performs an skb_push on the packet and updates
3998 * the CHECKSUM_COMPLETE checksum. It should be used on
3999 * receive path processing instead of skb_push unless you know
4000 * that the checksum difference is zero (e.g., a valid IP header)
4001 * or you are setting ip_summed to CHECKSUM_NONE.
4002 */
4003static inline void *skb_push_rcsum(struct sk_buff *skb, unsigned int len)
4004{
4005 skb_push(skb, len);
4006 skb_postpush_rcsum(skb, start: skb->data, len);
4007 return skb->data;
4008}
4009
4010int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len);
4011/**
4012 * pskb_trim_rcsum - trim received skb and update checksum
4013 * @skb: buffer to trim
4014 * @len: new length
4015 *
4016 * This is exactly the same as pskb_trim except that it ensures the
4017 * checksum of received packets are still valid after the operation.
4018 * It can change skb pointers.
4019 */
4020
4021static inline int pskb_trim_rcsum(struct sk_buff *skb, unsigned int len)
4022{
4023 skb_might_realloc(skb);
4024 if (likely(len >= skb->len))
4025 return 0;
4026 return pskb_trim_rcsum_slow(skb, len);
4027}
4028
4029static inline int __skb_trim_rcsum(struct sk_buff *skb, unsigned int len)
4030{
4031 if (skb->ip_summed == CHECKSUM_COMPLETE)
4032 skb->ip_summed = CHECKSUM_NONE;
4033 __skb_trim(skb, len);
4034 return 0;
4035}
4036
4037static inline int __skb_grow_rcsum(struct sk_buff *skb, unsigned int len)
4038{
4039 if (skb->ip_summed == CHECKSUM_COMPLETE)
4040 skb->ip_summed = CHECKSUM_NONE;
4041 return __skb_grow(skb, len);
4042}
4043
4044#define rb_to_skb(rb) rb_entry_safe(rb, struct sk_buff, rbnode)
4045#define skb_rb_first(root) rb_to_skb(rb_first(root))
4046#define skb_rb_last(root) rb_to_skb(rb_last(root))
4047#define skb_rb_next(skb) rb_to_skb(rb_next(&(skb)->rbnode))
4048#define skb_rb_prev(skb) rb_to_skb(rb_prev(&(skb)->rbnode))
4049
4050#define skb_queue_walk(queue, skb) \
4051 for (skb = (queue)->next; \
4052 skb != (struct sk_buff *)(queue); \
4053 skb = skb->next)
4054
4055#define skb_queue_walk_safe(queue, skb, tmp) \
4056 for (skb = (queue)->next, tmp = skb->next; \
4057 skb != (struct sk_buff *)(queue); \
4058 skb = tmp, tmp = skb->next)
4059
4060#define skb_queue_walk_from(queue, skb) \
4061 for (; skb != (struct sk_buff *)(queue); \
4062 skb = skb->next)
4063
4064#define skb_rbtree_walk(skb, root) \
4065 for (skb = skb_rb_first(root); skb != NULL; \
4066 skb = skb_rb_next(skb))
4067
4068#define skb_rbtree_walk_from(skb) \
4069 for (; skb != NULL; \
4070 skb = skb_rb_next(skb))
4071
4072#define skb_rbtree_walk_from_safe(skb, tmp) \
4073 for (; tmp = skb ? skb_rb_next(skb) : NULL, (skb != NULL); \
4074 skb = tmp)
4075
4076#define skb_queue_walk_from_safe(queue, skb, tmp) \
4077 for (tmp = skb->next; \
4078 skb != (struct sk_buff *)(queue); \
4079 skb = tmp, tmp = skb->next)
4080
4081#define skb_queue_reverse_walk(queue, skb) \
4082 for (skb = (queue)->prev; \
4083 skb != (struct sk_buff *)(queue); \
4084 skb = skb->prev)
4085
4086#define skb_queue_reverse_walk_safe(queue, skb, tmp) \
4087 for (skb = (queue)->prev, tmp = skb->prev; \
4088 skb != (struct sk_buff *)(queue); \
4089 skb = tmp, tmp = skb->prev)
4090
4091#define skb_queue_reverse_walk_from_safe(queue, skb, tmp) \
4092 for (tmp = skb->prev; \
4093 skb != (struct sk_buff *)(queue); \
4094 skb = tmp, tmp = skb->prev)
4095
4096static inline bool skb_has_frag_list(const struct sk_buff *skb)
4097{
4098 return skb_shinfo(skb)->frag_list != NULL;
4099}
4100
4101static inline void skb_frag_list_init(struct sk_buff *skb)
4102{
4103 skb_shinfo(skb)->frag_list = NULL;
4104}
4105
4106#define skb_walk_frags(skb, iter) \
4107 for (iter = skb_shinfo(skb)->frag_list; iter; iter = iter->next)
4108
4109
4110int __skb_wait_for_more_packets(struct sock *sk, struct sk_buff_head *queue,
4111 int *err, long *timeo_p,
4112 const struct sk_buff *skb);
4113struct sk_buff *__skb_try_recv_from_queue(struct sk_buff_head *queue,
4114 unsigned int flags,
4115 int *off, int *err,
4116 struct sk_buff **last);
4117struct sk_buff *__skb_try_recv_datagram(struct sock *sk,
4118 struct sk_buff_head *queue,
4119 unsigned int flags, int *off, int *err,
4120 struct sk_buff **last);
4121struct sk_buff *__skb_recv_datagram(struct sock *sk,
4122 struct sk_buff_head *sk_queue,
4123 unsigned int flags, int *off, int *err);
4124struct sk_buff *skb_recv_datagram(struct sock *sk, unsigned int flags, int *err);
4125__poll_t datagram_poll(struct file *file, struct socket *sock,
4126 struct poll_table_struct *wait);
4127int skb_copy_datagram_iter(const struct sk_buff *from, int offset,
4128 struct iov_iter *to, int size);
4129static inline int skb_copy_datagram_msg(const struct sk_buff *from, int offset,
4130 struct msghdr *msg, int size)
4131{
4132 return skb_copy_datagram_iter(from, offset, to: &msg->msg_iter, size);
4133}
4134int skb_copy_and_csum_datagram_msg(struct sk_buff *skb, int hlen,
4135 struct msghdr *msg);
4136int skb_copy_and_crc32c_datagram_iter(const struct sk_buff *skb, int offset,
4137 struct iov_iter *to, int len, u32 *crcp);
4138int skb_copy_datagram_from_iter(struct sk_buff *skb, int offset,
4139 struct iov_iter *from, int len);
4140int zerocopy_sg_from_iter(struct sk_buff *skb, struct iov_iter *frm);
4141void skb_free_datagram(struct sock *sk, struct sk_buff *skb);
4142int skb_kill_datagram(struct sock *sk, struct sk_buff *skb, unsigned int flags);
4143int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len);
4144int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len);
4145__wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset, u8 *to,
4146 int len);
4147int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset,
4148 struct pipe_inode_info *pipe, unsigned int len,
4149 unsigned int flags);
4150int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset,
4151 int len);
4152int skb_send_sock_locked_with_flags(struct sock *sk, struct sk_buff *skb,
4153 int offset, int len, int flags);
4154int skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, int len);
4155void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to);
4156unsigned int skb_zerocopy_headlen(const struct sk_buff *from);
4157int skb_zerocopy(struct sk_buff *to, struct sk_buff *from,
4158 int len, int hlen);
4159void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len);
4160int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen);
4161void skb_scrub_packet(struct sk_buff *skb, bool xnet);
4162struct sk_buff *skb_segment(struct sk_buff *skb, netdev_features_t features);
4163struct sk_buff *skb_segment_list(struct sk_buff *skb, netdev_features_t features,
4164 unsigned int offset);
4165struct sk_buff *skb_vlan_untag(struct sk_buff *skb);
4166int skb_ensure_writable(struct sk_buff *skb, unsigned int write_len);
4167int skb_ensure_writable_head_tail(struct sk_buff *skb, struct net_device *dev);
4168int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci);
4169int skb_vlan_pop(struct sk_buff *skb);
4170int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci);
4171int skb_eth_pop(struct sk_buff *skb);
4172int skb_eth_push(struct sk_buff *skb, const unsigned char *dst,
4173 const unsigned char *src);
4174int skb_mpls_push(struct sk_buff *skb, __be32 mpls_lse, __be16 mpls_proto,
4175 int mac_len, bool ethernet);
4176int skb_mpls_pop(struct sk_buff *skb, __be16 next_proto, int mac_len,
4177 bool ethernet);
4178int skb_mpls_update_lse(struct sk_buff *skb, __be32 mpls_lse);
4179int skb_mpls_dec_ttl(struct sk_buff *skb);
4180struct sk_buff *pskb_extract(struct sk_buff *skb, int off, int to_copy,
4181 gfp_t gfp);
4182
4183static inline int memcpy_from_msg(void *data, struct msghdr *msg, int len)
4184{
4185 return copy_from_iter_full(addr: data, bytes: len, i: &msg->msg_iter) ? 0 : -EFAULT;
4186}
4187
4188static inline int memcpy_to_msg(struct msghdr *msg, void *data, int len)
4189{
4190 return copy_to_iter(addr: data, bytes: len, i: &msg->msg_iter) == len ? 0 : -EFAULT;
4191}
4192
4193__wsum skb_checksum(const struct sk_buff *skb, int offset, int len,
4194 __wsum csum);
4195u32 skb_crc32c(const struct sk_buff *skb, int offset, int len, u32 crc);
4196
4197static inline void * __must_check
4198__skb_header_pointer(const struct sk_buff *skb, int offset, int len,
4199 const void *data, int hlen, void *buffer)
4200{
4201 if (likely(hlen - offset >= len))
4202 return (void *)data + offset;
4203
4204 if (!skb || unlikely(skb_copy_bits(skb, offset, buffer, len) < 0))
4205 return NULL;
4206
4207 return buffer;
4208}
4209
4210static inline void * __must_check
4211skb_header_pointer(const struct sk_buff *skb, int offset, int len, void *buffer)
4212{
4213 return __skb_header_pointer(skb, offset, len, data: skb->data,
4214 hlen: skb_headlen(skb), buffer);
4215}
4216
4217static inline void * __must_check
4218skb_pointer_if_linear(const struct sk_buff *skb, int offset, int len)
4219{
4220 if (likely(skb_headlen(skb) - offset >= len))
4221 return skb->data + offset;
4222 return NULL;
4223}
4224
4225/**
4226 * skb_needs_linearize - check if we need to linearize a given skb
4227 * depending on the given device features.
4228 * @skb: socket buffer to check
4229 * @features: net device features
4230 *
4231 * Returns true if either:
4232 * 1. skb has frag_list and the device doesn't support FRAGLIST, or
4233 * 2. skb is fragmented and the device does not support SG.
4234 */
4235static inline bool skb_needs_linearize(struct sk_buff *skb,
4236 netdev_features_t features)
4237{
4238 return skb_is_nonlinear(skb) &&
4239 ((skb_has_frag_list(skb) && !(features & NETIF_F_FRAGLIST)) ||
4240 (skb_shinfo(skb)->nr_frags && !(features & NETIF_F_SG)));
4241}
4242
4243static inline void skb_copy_from_linear_data(const struct sk_buff *skb,
4244 void *to,
4245 const unsigned int len)
4246{
4247 memcpy(to, skb->data, len);
4248}
4249
4250static inline void skb_copy_from_linear_data_offset(const struct sk_buff *skb,
4251 const int offset, void *to,
4252 const unsigned int len)
4253{
4254 memcpy(to, skb->data + offset, len);
4255}
4256
4257static inline void skb_copy_to_linear_data(struct sk_buff *skb,
4258 const void *from,
4259 const unsigned int len)
4260{
4261 memcpy(skb->data, from, len);
4262}
4263
4264static inline void skb_copy_to_linear_data_offset(struct sk_buff *skb,
4265 const int offset,
4266 const void *from,
4267 const unsigned int len)
4268{
4269 memcpy(skb->data + offset, from, len);
4270}
4271
4272void skb_init(void);
4273
4274static inline ktime_t skb_get_ktime(const struct sk_buff *skb)
4275{
4276 return skb->tstamp;
4277}
4278
4279/**
4280 * skb_get_timestamp - get timestamp from a skb
4281 * @skb: skb to get stamp from
4282 * @stamp: pointer to struct __kernel_old_timeval to store stamp in
4283 *
4284 * Timestamps are stored in the skb as offsets to a base timestamp.
4285 * This function converts the offset back to a struct timeval and stores
4286 * it in stamp.
4287 */
4288static inline void skb_get_timestamp(const struct sk_buff *skb,
4289 struct __kernel_old_timeval *stamp)
4290{
4291 *stamp = ns_to_kernel_old_timeval(nsec: skb->tstamp);
4292}
4293
4294static inline void skb_get_new_timestamp(const struct sk_buff *skb,
4295 struct __kernel_sock_timeval *stamp)
4296{
4297 struct timespec64 ts = ktime_to_timespec64(skb->tstamp);
4298
4299 stamp->tv_sec = ts.tv_sec;
4300 stamp->tv_usec = ts.tv_nsec / 1000;
4301}
4302
4303static inline void skb_get_timestampns(const struct sk_buff *skb,
4304 struct __kernel_old_timespec *stamp)
4305{
4306 struct timespec64 ts = ktime_to_timespec64(skb->tstamp);
4307
4308 stamp->tv_sec = ts.tv_sec;
4309 stamp->tv_nsec = ts.tv_nsec;
4310}
4311
4312static inline void skb_get_new_timestampns(const struct sk_buff *skb,
4313 struct __kernel_timespec *stamp)
4314{
4315 struct timespec64 ts = ktime_to_timespec64(skb->tstamp);
4316
4317 stamp->tv_sec = ts.tv_sec;
4318 stamp->tv_nsec = ts.tv_nsec;
4319}
4320
4321static inline void __net_timestamp(struct sk_buff *skb)
4322{
4323 skb->tstamp = ktime_get_real();
4324 skb->tstamp_type = SKB_CLOCK_REALTIME;
4325}
4326
4327static inline ktime_t net_timedelta(ktime_t t)
4328{
4329 return ktime_sub(ktime_get_real(), t);
4330}
4331
4332static inline void skb_set_delivery_time(struct sk_buff *skb, ktime_t kt,
4333 u8 tstamp_type)
4334{
4335 skb->tstamp = kt;
4336
4337 if (kt)
4338 skb->tstamp_type = tstamp_type;
4339 else
4340 skb->tstamp_type = SKB_CLOCK_REALTIME;
4341}
4342
4343static inline void skb_set_delivery_type_by_clockid(struct sk_buff *skb,
4344 ktime_t kt, clockid_t clockid)
4345{
4346 u8 tstamp_type = SKB_CLOCK_REALTIME;
4347
4348 switch (clockid) {
4349 case CLOCK_REALTIME:
4350 break;
4351 case CLOCK_MONOTONIC:
4352 tstamp_type = SKB_CLOCK_MONOTONIC;
4353 break;
4354 case CLOCK_TAI:
4355 tstamp_type = SKB_CLOCK_TAI;
4356 break;
4357 default:
4358 WARN_ON_ONCE(1);
4359 kt = 0;
4360 }
4361
4362 skb_set_delivery_time(skb, kt, tstamp_type);
4363}
4364
4365DECLARE_STATIC_KEY_FALSE(netstamp_needed_key);
4366
4367/* It is used in the ingress path to clear the delivery_time.
4368 * If needed, set the skb->tstamp to the (rcv) timestamp.
4369 */
4370static inline void skb_clear_delivery_time(struct sk_buff *skb)
4371{
4372 if (skb->tstamp_type) {
4373 skb->tstamp_type = SKB_CLOCK_REALTIME;
4374 if (static_branch_unlikely(&netstamp_needed_key))
4375 skb->tstamp = ktime_get_real();
4376 else
4377 skb->tstamp = 0;
4378 }
4379}
4380
4381static inline void skb_clear_tstamp(struct sk_buff *skb)
4382{
4383 if (skb->tstamp_type)
4384 return;
4385
4386 skb->tstamp = 0;
4387}
4388
4389static inline ktime_t skb_tstamp(const struct sk_buff *skb)
4390{
4391 if (skb->tstamp_type)
4392 return 0;
4393
4394 return skb->tstamp;
4395}
4396
4397static inline ktime_t skb_tstamp_cond(const struct sk_buff *skb, bool cond)
4398{
4399 if (skb->tstamp_type != SKB_CLOCK_MONOTONIC && skb->tstamp)
4400 return skb->tstamp;
4401
4402 if (static_branch_unlikely(&netstamp_needed_key) || cond)
4403 return ktime_get_real();
4404
4405 return 0;
4406}
4407
4408static inline u8 skb_metadata_len(const struct sk_buff *skb)
4409{
4410 return skb_shinfo(skb)->meta_len;
4411}
4412
4413static inline void *skb_metadata_end(const struct sk_buff *skb)
4414{
4415 return skb_mac_header(skb);
4416}
4417
4418static inline bool __skb_metadata_differs(const struct sk_buff *skb_a,
4419 const struct sk_buff *skb_b,
4420 u8 meta_len)
4421{
4422 const void *a = skb_metadata_end(skb: skb_a);
4423 const void *b = skb_metadata_end(skb: skb_b);
4424 u64 diffs = 0;
4425
4426 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) ||
4427 BITS_PER_LONG != 64)
4428 goto slow;
4429
4430 /* Using more efficient variant than plain call to memcmp(). */
4431 switch (meta_len) {
4432#define __it(x, op) (x -= sizeof(u##op))
4433#define __it_diff(a, b, op) (*(u##op *)__it(a, op)) ^ (*(u##op *)__it(b, op))
4434 case 32: diffs |= __it_diff(a, b, 64);
4435 fallthrough;
4436 case 24: diffs |= __it_diff(a, b, 64);
4437 fallthrough;
4438 case 16: diffs |= __it_diff(a, b, 64);
4439 fallthrough;
4440 case 8: diffs |= __it_diff(a, b, 64);
4441 break;
4442 case 28: diffs |= __it_diff(a, b, 64);
4443 fallthrough;
4444 case 20: diffs |= __it_diff(a, b, 64);
4445 fallthrough;
4446 case 12: diffs |= __it_diff(a, b, 64);
4447 fallthrough;
4448 case 4: diffs |= __it_diff(a, b, 32);
4449 break;
4450 default:
4451slow:
4452 return memcmp(p: a - meta_len, q: b - meta_len, size: meta_len);
4453 }
4454 return diffs;
4455}
4456
4457static inline bool skb_metadata_differs(const struct sk_buff *skb_a,
4458 const struct sk_buff *skb_b)
4459{
4460 u8 len_a = skb_metadata_len(skb: skb_a);
4461 u8 len_b = skb_metadata_len(skb: skb_b);
4462
4463 if (!(len_a | len_b))
4464 return false;
4465
4466 return len_a != len_b ?
4467 true : __skb_metadata_differs(skb_a, skb_b, meta_len: len_a);
4468}
4469
4470static inline void skb_metadata_set(struct sk_buff *skb, u8 meta_len)
4471{
4472 skb_shinfo(skb)->meta_len = meta_len;
4473}
4474
4475static inline void skb_metadata_clear(struct sk_buff *skb)
4476{
4477 skb_metadata_set(skb, meta_len: 0);
4478}
4479
4480struct sk_buff *skb_clone_sk(struct sk_buff *skb);
4481
4482#ifdef CONFIG_NETWORK_PHY_TIMESTAMPING
4483
4484void skb_clone_tx_timestamp(struct sk_buff *skb);
4485bool skb_defer_rx_timestamp(struct sk_buff *skb);
4486
4487#else /* CONFIG_NETWORK_PHY_TIMESTAMPING */
4488
4489static inline void skb_clone_tx_timestamp(struct sk_buff *skb)
4490{
4491}
4492
4493static inline bool skb_defer_rx_timestamp(struct sk_buff *skb)
4494{
4495 return false;
4496}
4497
4498#endif /* !CONFIG_NETWORK_PHY_TIMESTAMPING */
4499
4500/**
4501 * skb_complete_tx_timestamp() - deliver cloned skb with tx timestamps
4502 *
4503 * PHY drivers may accept clones of transmitted packets for
4504 * timestamping via their phy_driver.txtstamp method. These drivers
4505 * must call this function to return the skb back to the stack with a
4506 * timestamp.
4507 *
4508 * @skb: clone of the original outgoing packet
4509 * @hwtstamps: hardware time stamps
4510 *
4511 */
4512void skb_complete_tx_timestamp(struct sk_buff *skb,
4513 struct skb_shared_hwtstamps *hwtstamps);
4514
4515void __skb_tstamp_tx(struct sk_buff *orig_skb, const struct sk_buff *ack_skb,
4516 struct skb_shared_hwtstamps *hwtstamps,
4517 struct sock *sk, int tstype);
4518
4519/**
4520 * skb_tstamp_tx - queue clone of skb with send time stamps
4521 * @orig_skb: the original outgoing packet
4522 * @hwtstamps: hardware time stamps, may be NULL if not available
4523 *
4524 * If the skb has a socket associated, then this function clones the
4525 * skb (thus sharing the actual data and optional structures), stores
4526 * the optional hardware time stamping information (if non NULL) or
4527 * generates a software time stamp (otherwise), then queues the clone
4528 * to the error queue of the socket. Errors are silently ignored.
4529 */
4530void skb_tstamp_tx(struct sk_buff *orig_skb,
4531 struct skb_shared_hwtstamps *hwtstamps);
4532
4533/**
4534 * skb_tx_timestamp() - Driver hook for transmit timestamping
4535 *
4536 * Ethernet MAC Drivers should call this function in their hard_xmit()
4537 * function immediately before giving the sk_buff to the MAC hardware.
4538 *
4539 * Specifically, one should make absolutely sure that this function is
4540 * called before TX completion of this packet can trigger. Otherwise
4541 * the packet could potentially already be freed.
4542 *
4543 * @skb: A socket buffer.
4544 */
4545static inline void skb_tx_timestamp(struct sk_buff *skb)
4546{
4547 skb_clone_tx_timestamp(skb);
4548 if (skb_shinfo(skb)->tx_flags & (SKBTX_SW_TSTAMP | SKBTX_BPF))
4549 skb_tstamp_tx(orig_skb: skb, NULL);
4550}
4551
4552/**
4553 * skb_complete_wifi_ack - deliver skb with wifi status
4554 *
4555 * @skb: the original outgoing packet
4556 * @acked: ack status
4557 *
4558 */
4559void skb_complete_wifi_ack(struct sk_buff *skb, bool acked);
4560
4561__sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len);
4562__sum16 __skb_checksum_complete(struct sk_buff *skb);
4563
4564static inline int skb_csum_unnecessary(const struct sk_buff *skb)
4565{
4566 return ((skb->ip_summed == CHECKSUM_UNNECESSARY) ||
4567 skb->csum_valid ||
4568 (skb->ip_summed == CHECKSUM_PARTIAL &&
4569 skb_checksum_start_offset(skb) >= 0));
4570}
4571
4572/**
4573 * skb_checksum_complete - Calculate checksum of an entire packet
4574 * @skb: packet to process
4575 *
4576 * This function calculates the checksum over the entire packet plus
4577 * the value of skb->csum. The latter can be used to supply the
4578 * checksum of a pseudo header as used by TCP/UDP. It returns the
4579 * checksum.
4580 *
4581 * For protocols that contain complete checksums such as ICMP/TCP/UDP,
4582 * this function can be used to verify that checksum on received
4583 * packets. In that case the function should return zero if the
4584 * checksum is correct. In particular, this function will return zero
4585 * if skb->ip_summed is CHECKSUM_UNNECESSARY which indicates that the
4586 * hardware has already verified the correctness of the checksum.
4587 */
4588static inline __sum16 skb_checksum_complete(struct sk_buff *skb)
4589{
4590 return skb_csum_unnecessary(skb) ?
4591 0 : __skb_checksum_complete(skb);
4592}
4593
4594static inline void __skb_decr_checksum_unnecessary(struct sk_buff *skb)
4595{
4596 if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
4597 if (skb->csum_level == 0)
4598 skb->ip_summed = CHECKSUM_NONE;
4599 else
4600 skb->csum_level--;
4601 }
4602}
4603
4604static inline void __skb_incr_checksum_unnecessary(struct sk_buff *skb)
4605{
4606 if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
4607 if (skb->csum_level < SKB_MAX_CSUM_LEVEL)
4608 skb->csum_level++;
4609 } else if (skb->ip_summed == CHECKSUM_NONE) {
4610 skb->ip_summed = CHECKSUM_UNNECESSARY;
4611 skb->csum_level = 0;
4612 }
4613}
4614
4615static inline void __skb_reset_checksum_unnecessary(struct sk_buff *skb)
4616{
4617 if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
4618 skb->ip_summed = CHECKSUM_NONE;
4619 skb->csum_level = 0;
4620 }
4621}
4622
4623/* Check if we need to perform checksum complete validation.
4624 *
4625 * Returns: true if checksum complete is needed, false otherwise
4626 * (either checksum is unnecessary or zero checksum is allowed).
4627 */
4628static inline bool __skb_checksum_validate_needed(struct sk_buff *skb,
4629 bool zero_okay,
4630 __sum16 check)
4631{
4632 if (skb_csum_unnecessary(skb) || (zero_okay && !check)) {
4633 skb->csum_valid = 1;
4634 __skb_decr_checksum_unnecessary(skb);
4635 return false;
4636 }
4637
4638 return true;
4639}
4640
4641/* For small packets <= CHECKSUM_BREAK perform checksum complete directly
4642 * in checksum_init.
4643 */
4644#define CHECKSUM_BREAK 76
4645
4646/* Unset checksum-complete
4647 *
4648 * Unset checksum complete can be done when packet is being modified
4649 * (uncompressed for instance) and checksum-complete value is
4650 * invalidated.
4651 */
4652static inline void skb_checksum_complete_unset(struct sk_buff *skb)
4653{
4654 if (skb->ip_summed == CHECKSUM_COMPLETE)
4655 skb->ip_summed = CHECKSUM_NONE;
4656}
4657
4658/* Validate (init) checksum based on checksum complete.
4659 *
4660 * Return values:
4661 * 0: checksum is validated or try to in skb_checksum_complete. In the latter
4662 * case the ip_summed will not be CHECKSUM_UNNECESSARY and the pseudo
4663 * checksum is stored in skb->csum for use in __skb_checksum_complete
4664 * non-zero: value of invalid checksum
4665 *
4666 */
4667static inline __sum16 __skb_checksum_validate_complete(struct sk_buff *skb,
4668 bool complete,
4669 __wsum psum)
4670{
4671 if (skb->ip_summed == CHECKSUM_COMPLETE) {
4672 if (!csum_fold(csum: csum_add(csum: psum, addend: skb->csum))) {
4673 skb->csum_valid = 1;
4674 return 0;
4675 }
4676 }
4677
4678 skb->csum = psum;
4679
4680 if (complete || skb->len <= CHECKSUM_BREAK) {
4681 __sum16 csum;
4682
4683 csum = __skb_checksum_complete(skb);
4684 skb->csum_valid = !csum;
4685 return csum;
4686 }
4687
4688 return 0;
4689}
4690
4691static inline __wsum null_compute_pseudo(struct sk_buff *skb, int proto)
4692{
4693 return 0;
4694}
4695
4696/* Perform checksum validate (init). Note that this is a macro since we only
4697 * want to calculate the pseudo header which is an input function if necessary.
4698 * First we try to validate without any computation (checksum unnecessary) and
4699 * then calculate based on checksum complete calling the function to compute
4700 * pseudo header.
4701 *
4702 * Return values:
4703 * 0: checksum is validated or try to in skb_checksum_complete
4704 * non-zero: value of invalid checksum
4705 */
4706#define __skb_checksum_validate(skb, proto, complete, \
4707 zero_okay, check, compute_pseudo) \
4708({ \
4709 __sum16 __ret = 0; \
4710 skb->csum_valid = 0; \
4711 if (__skb_checksum_validate_needed(skb, zero_okay, check)) \
4712 __ret = __skb_checksum_validate_complete(skb, \
4713 complete, compute_pseudo(skb, proto)); \
4714 __ret; \
4715})
4716
4717#define skb_checksum_init(skb, proto, compute_pseudo) \
4718 __skb_checksum_validate(skb, proto, false, false, 0, compute_pseudo)
4719
4720#define skb_checksum_init_zero_check(skb, proto, check, compute_pseudo) \
4721 __skb_checksum_validate(skb, proto, false, true, check, compute_pseudo)
4722
4723#define skb_checksum_validate(skb, proto, compute_pseudo) \
4724 __skb_checksum_validate(skb, proto, true, false, 0, compute_pseudo)
4725
4726#define skb_checksum_validate_zero_check(skb, proto, check, \
4727 compute_pseudo) \
4728 __skb_checksum_validate(skb, proto, true, true, check, compute_pseudo)
4729
4730#define skb_checksum_simple_validate(skb) \
4731 __skb_checksum_validate(skb, 0, true, false, 0, null_compute_pseudo)
4732
4733static inline bool __skb_checksum_convert_check(struct sk_buff *skb)
4734{
4735 return (skb->ip_summed == CHECKSUM_NONE && skb->csum_valid);
4736}
4737
4738static inline void __skb_checksum_convert(struct sk_buff *skb, __wsum pseudo)
4739{
4740 skb->csum = ~pseudo;
4741 skb->ip_summed = CHECKSUM_COMPLETE;
4742}
4743
4744#define skb_checksum_try_convert(skb, proto, compute_pseudo) \
4745do { \
4746 if (__skb_checksum_convert_check(skb)) \
4747 __skb_checksum_convert(skb, compute_pseudo(skb, proto)); \
4748} while (0)
4749
4750static inline void skb_remcsum_adjust_partial(struct sk_buff *skb, void *ptr,
4751 u16 start, u16 offset)
4752{
4753 skb->ip_summed = CHECKSUM_PARTIAL;
4754 skb->csum_start = ((unsigned char *)ptr + start) - skb->head;
4755 skb->csum_offset = offset - start;
4756}
4757
4758/* Update skbuf and packet to reflect the remote checksum offload operation.
4759 * When called, ptr indicates the starting point for skb->csum when
4760 * ip_summed is CHECKSUM_COMPLETE. If we need create checksum complete
4761 * here, skb_postpull_rcsum is done so skb->csum start is ptr.
4762 */
4763static inline void skb_remcsum_process(struct sk_buff *skb, void *ptr,
4764 int start, int offset, bool nopartial)
4765{
4766 __wsum delta;
4767
4768 if (!nopartial) {
4769 skb_remcsum_adjust_partial(skb, ptr, start, offset);
4770 return;
4771 }
4772
4773 if (unlikely(skb->ip_summed != CHECKSUM_COMPLETE)) {
4774 __skb_checksum_complete(skb);
4775 skb_postpull_rcsum(skb, start: skb->data, len: ptr - (void *)skb->data);
4776 }
4777
4778 delta = remcsum_adjust(ptr, csum: skb->csum, start, offset);
4779
4780 /* Adjust skb->csum since we changed the packet */
4781 skb->csum = csum_add(csum: skb->csum, addend: delta);
4782}
4783
4784static inline struct nf_conntrack *skb_nfct(const struct sk_buff *skb)
4785{
4786#if IS_ENABLED(CONFIG_NF_CONNTRACK)
4787 return (void *)(skb->_nfct & NFCT_PTRMASK);
4788#else
4789 return NULL;
4790#endif
4791}
4792
4793static inline unsigned long skb_get_nfct(const struct sk_buff *skb)
4794{
4795#if IS_ENABLED(CONFIG_NF_CONNTRACK)
4796 return skb->_nfct;
4797#else
4798 return 0UL;
4799#endif
4800}
4801
4802static inline void skb_set_nfct(struct sk_buff *skb, unsigned long nfct)
4803{
4804#if IS_ENABLED(CONFIG_NF_CONNTRACK)
4805 skb->slow_gro |= !!nfct;
4806 skb->_nfct = nfct;
4807#endif
4808}
4809
4810#ifdef CONFIG_SKB_EXTENSIONS
4811enum skb_ext_id {
4812#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
4813 SKB_EXT_BRIDGE_NF,
4814#endif
4815#ifdef CONFIG_XFRM
4816 SKB_EXT_SEC_PATH,
4817#endif
4818#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
4819 TC_SKB_EXT,
4820#endif
4821#if IS_ENABLED(CONFIG_MPTCP)
4822 SKB_EXT_MPTCP,
4823#endif
4824#if IS_ENABLED(CONFIG_MCTP_FLOWS)
4825 SKB_EXT_MCTP,
4826#endif
4827 SKB_EXT_NUM, /* must be last */
4828};
4829
4830/**
4831 * struct skb_ext - sk_buff extensions
4832 * @refcnt: 1 on allocation, deallocated on 0
4833 * @offset: offset to add to @data to obtain extension address
4834 * @chunks: size currently allocated, stored in SKB_EXT_ALIGN_SHIFT units
4835 * @data: start of extension data, variable sized
4836 *
4837 * Note: offsets/lengths are stored in chunks of 8 bytes, this allows
4838 * to use 'u8' types while allowing up to 2kb worth of extension data.
4839 */
4840struct skb_ext {
4841 refcount_t refcnt;
4842 u8 offset[SKB_EXT_NUM]; /* in chunks of 8 bytes */
4843 u8 chunks; /* same */
4844 char data[] __aligned(8);
4845};
4846
4847struct skb_ext *__skb_ext_alloc(gfp_t flags);
4848void *__skb_ext_set(struct sk_buff *skb, enum skb_ext_id id,
4849 struct skb_ext *ext);
4850void *skb_ext_add(struct sk_buff *skb, enum skb_ext_id id);
4851void __skb_ext_del(struct sk_buff *skb, enum skb_ext_id id);
4852void __skb_ext_put(struct skb_ext *ext);
4853
4854static inline void skb_ext_put(struct sk_buff *skb)
4855{
4856 if (skb->active_extensions)
4857 __skb_ext_put(ext: skb->extensions);
4858}
4859
4860static inline void __skb_ext_copy(struct sk_buff *dst,
4861 const struct sk_buff *src)
4862{
4863 dst->active_extensions = src->active_extensions;
4864
4865 if (src->active_extensions) {
4866 struct skb_ext *ext = src->extensions;
4867
4868 refcount_inc(r: &ext->refcnt);
4869 dst->extensions = ext;
4870 }
4871}
4872
4873static inline void skb_ext_copy(struct sk_buff *dst, const struct sk_buff *src)
4874{
4875 skb_ext_put(skb: dst);
4876 __skb_ext_copy(dst, src);
4877}
4878
4879static inline bool __skb_ext_exist(const struct skb_ext *ext, enum skb_ext_id i)
4880{
4881 return !!ext->offset[i];
4882}
4883
4884static inline bool skb_ext_exist(const struct sk_buff *skb, enum skb_ext_id id)
4885{
4886 return skb->active_extensions & (1 << id);
4887}
4888
4889static inline void skb_ext_del(struct sk_buff *skb, enum skb_ext_id id)
4890{
4891 if (skb_ext_exist(skb, id))
4892 __skb_ext_del(skb, id);
4893}
4894
4895static inline void *skb_ext_find(const struct sk_buff *skb, enum skb_ext_id id)
4896{
4897 if (skb_ext_exist(skb, id)) {
4898 struct skb_ext *ext = skb->extensions;
4899
4900 return (void *)ext + (ext->offset[id] << 3);
4901 }
4902
4903 return NULL;
4904}
4905
4906static inline void skb_ext_reset(struct sk_buff *skb)
4907{
4908 if (unlikely(skb->active_extensions)) {
4909 __skb_ext_put(ext: skb->extensions);
4910 skb->active_extensions = 0;
4911 }
4912}
4913
4914static inline bool skb_has_extensions(struct sk_buff *skb)
4915{
4916 return unlikely(skb->active_extensions);
4917}
4918#else
4919static inline void skb_ext_put(struct sk_buff *skb) {}
4920static inline void skb_ext_reset(struct sk_buff *skb) {}
4921static inline void skb_ext_del(struct sk_buff *skb, int unused) {}
4922static inline void __skb_ext_copy(struct sk_buff *d, const struct sk_buff *s) {}
4923static inline void skb_ext_copy(struct sk_buff *dst, const struct sk_buff *s) {}
4924static inline bool skb_has_extensions(struct sk_buff *skb) { return false; }
4925#endif /* CONFIG_SKB_EXTENSIONS */
4926
4927static inline void nf_reset_ct(struct sk_buff *skb)
4928{
4929#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
4930 nf_conntrack_put(nfct: skb_nfct(skb));
4931 skb->_nfct = 0;
4932#endif
4933}
4934
4935static inline void nf_reset_trace(struct sk_buff *skb)
4936{
4937#if IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TRACE) || IS_ENABLED(CONFIG_NF_TABLES)
4938 skb->nf_trace = 0;
4939#endif
4940}
4941
4942static inline void ipvs_reset(struct sk_buff *skb)
4943{
4944#if IS_ENABLED(CONFIG_IP_VS)
4945 skb->ipvs_property = 0;
4946#endif
4947}
4948
4949/* Note: This doesn't put any conntrack info in dst. */
4950static inline void __nf_copy(struct sk_buff *dst, const struct sk_buff *src,
4951 bool copy)
4952{
4953#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
4954 dst->_nfct = src->_nfct;
4955 nf_conntrack_get(nfct: skb_nfct(skb: src));
4956#endif
4957#if IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TRACE) || IS_ENABLED(CONFIG_NF_TABLES)
4958 if (copy)
4959 dst->nf_trace = src->nf_trace;
4960#endif
4961}
4962
4963static inline void nf_copy(struct sk_buff *dst, const struct sk_buff *src)
4964{
4965#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
4966 nf_conntrack_put(nfct: skb_nfct(skb: dst));
4967#endif
4968 dst->slow_gro = src->slow_gro;
4969 __nf_copy(dst, src, copy: true);
4970}
4971
4972#ifdef CONFIG_NETWORK_SECMARK
4973static inline void skb_copy_secmark(struct sk_buff *to, const struct sk_buff *from)
4974{
4975 to->secmark = from->secmark;
4976}
4977
4978static inline void skb_init_secmark(struct sk_buff *skb)
4979{
4980 skb->secmark = 0;
4981}
4982#else
4983static inline void skb_copy_secmark(struct sk_buff *to, const struct sk_buff *from)
4984{ }
4985
4986static inline void skb_init_secmark(struct sk_buff *skb)
4987{ }
4988#endif
4989
4990static inline int secpath_exists(const struct sk_buff *skb)
4991{
4992#ifdef CONFIG_XFRM
4993 return skb_ext_exist(skb, id: SKB_EXT_SEC_PATH);
4994#else
4995 return 0;
4996#endif
4997}
4998
4999static inline bool skb_irq_freeable(const struct sk_buff *skb)
5000{
5001 return !skb->destructor &&
5002 !secpath_exists(skb) &&
5003 !skb_nfct(skb) &&
5004 !skb->_skb_refdst &&
5005 !skb_has_frag_list(skb);
5006}
5007
5008static inline void skb_set_queue_mapping(struct sk_buff *skb, u16 queue_mapping)
5009{
5010 skb->queue_mapping = queue_mapping;
5011}
5012
5013static inline u16 skb_get_queue_mapping(const struct sk_buff *skb)
5014{
5015 return skb->queue_mapping;
5016}
5017
5018static inline void skb_copy_queue_mapping(struct sk_buff *to, const struct sk_buff *from)
5019{
5020 to->queue_mapping = from->queue_mapping;
5021}
5022
5023static inline void skb_record_rx_queue(struct sk_buff *skb, u16 rx_queue)
5024{
5025 skb->queue_mapping = rx_queue + 1;
5026}
5027
5028static inline u16 skb_get_rx_queue(const struct sk_buff *skb)
5029{
5030 return skb->queue_mapping - 1;
5031}
5032
5033static inline bool skb_rx_queue_recorded(const struct sk_buff *skb)
5034{
5035 return skb->queue_mapping != 0;
5036}
5037
5038static inline void skb_set_dst_pending_confirm(struct sk_buff *skb, u32 val)
5039{
5040 skb->dst_pending_confirm = val;
5041}
5042
5043static inline bool skb_get_dst_pending_confirm(const struct sk_buff *skb)
5044{
5045 return skb->dst_pending_confirm != 0;
5046}
5047
5048static inline struct sec_path *skb_sec_path(const struct sk_buff *skb)
5049{
5050#ifdef CONFIG_XFRM
5051 return skb_ext_find(skb, id: SKB_EXT_SEC_PATH);
5052#else
5053 return NULL;
5054#endif
5055}
5056
5057static inline bool skb_is_gso(const struct sk_buff *skb)
5058{
5059 return skb_shinfo(skb)->gso_size;
5060}
5061
5062/* Note: Should be called only if skb_is_gso(skb) is true */
5063static inline bool skb_is_gso_v6(const struct sk_buff *skb)
5064{
5065 return skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6;
5066}
5067
5068/* Note: Should be called only if skb_is_gso(skb) is true */
5069static inline bool skb_is_gso_sctp(const struct sk_buff *skb)
5070{
5071 return skb_shinfo(skb)->gso_type & SKB_GSO_SCTP;
5072}
5073
5074/* Note: Should be called only if skb_is_gso(skb) is true */
5075static inline bool skb_is_gso_tcp(const struct sk_buff *skb)
5076{
5077 return skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6);
5078}
5079
5080static inline void skb_gso_reset(struct sk_buff *skb)
5081{
5082 skb_shinfo(skb)->gso_size = 0;
5083 skb_shinfo(skb)->gso_segs = 0;
5084 skb_shinfo(skb)->gso_type = 0;
5085}
5086
5087static inline void skb_increase_gso_size(struct skb_shared_info *shinfo,
5088 u16 increment)
5089{
5090 if (WARN_ON_ONCE(shinfo->gso_size == GSO_BY_FRAGS))
5091 return;
5092 shinfo->gso_size += increment;
5093}
5094
5095static inline void skb_decrease_gso_size(struct skb_shared_info *shinfo,
5096 u16 decrement)
5097{
5098 if (WARN_ON_ONCE(shinfo->gso_size == GSO_BY_FRAGS))
5099 return;
5100 shinfo->gso_size -= decrement;
5101}
5102
5103void __skb_warn_lro_forwarding(const struct sk_buff *skb);
5104
5105static inline bool skb_warn_if_lro(const struct sk_buff *skb)
5106{
5107 /* LRO sets gso_size but not gso_type, whereas if GSO is really
5108 * wanted then gso_type will be set. */
5109 const struct skb_shared_info *shinfo = skb_shinfo(skb);
5110
5111 if (skb_is_nonlinear(skb) && shinfo->gso_size != 0 &&
5112 unlikely(shinfo->gso_type == 0)) {
5113 __skb_warn_lro_forwarding(skb);
5114 return true;
5115 }
5116 return false;
5117}
5118
5119static inline void skb_forward_csum(struct sk_buff *skb)
5120{
5121 /* Unfortunately we don't support this one. Any brave souls? */
5122 if (skb->ip_summed == CHECKSUM_COMPLETE)
5123 skb->ip_summed = CHECKSUM_NONE;
5124}
5125
5126/**
5127 * skb_checksum_none_assert - make sure skb ip_summed is CHECKSUM_NONE
5128 * @skb: skb to check
5129 *
5130 * fresh skbs have their ip_summed set to CHECKSUM_NONE.
5131 * Instead of forcing ip_summed to CHECKSUM_NONE, we can
5132 * use this helper, to document places where we make this assertion.
5133 */
5134static inline void skb_checksum_none_assert(const struct sk_buff *skb)
5135{
5136 DEBUG_NET_WARN_ON_ONCE(skb->ip_summed != CHECKSUM_NONE);
5137}
5138
5139bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off);
5140
5141int skb_checksum_setup(struct sk_buff *skb, bool recalculate);
5142struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb,
5143 unsigned int transport_len,
5144 __sum16(*skb_chkf)(struct sk_buff *skb));
5145
5146/**
5147 * skb_head_is_locked - Determine if the skb->head is locked down
5148 * @skb: skb to check
5149 *
5150 * The head on skbs build around a head frag can be removed if they are
5151 * not cloned. This function returns true if the skb head is locked down
5152 * due to either being allocated via kmalloc, or by being a clone with
5153 * multiple references to the head.
5154 */
5155static inline bool skb_head_is_locked(const struct sk_buff *skb)
5156{
5157 return !skb->head_frag || skb_cloned(skb);
5158}
5159
5160/* Local Checksum Offload.
5161 * Compute outer checksum based on the assumption that the
5162 * inner checksum will be offloaded later.
5163 * See Documentation/networking/checksum-offloads.rst for
5164 * explanation of how this works.
5165 * Fill in outer checksum adjustment (e.g. with sum of outer
5166 * pseudo-header) before calling.
5167 * Also ensure that inner checksum is in linear data area.
5168 */
5169static inline __wsum lco_csum(struct sk_buff *skb)
5170{
5171 unsigned char *csum_start = skb_checksum_start(skb);
5172 unsigned char *l4_hdr = skb_transport_header(skb);
5173 __wsum partial;
5174
5175 /* Start with complement of inner checksum adjustment */
5176 partial = ~csum_unfold(n: *(__force __sum16 *)(csum_start +
5177 skb->csum_offset));
5178
5179 /* Add in checksum of our headers (incl. outer checksum
5180 * adjustment filled in by caller) and return result.
5181 */
5182 return csum_partial(buff: l4_hdr, len: csum_start - l4_hdr, sum: partial);
5183}
5184
5185static inline bool skb_is_redirected(const struct sk_buff *skb)
5186{
5187 return skb->redirected;
5188}
5189
5190static inline void skb_set_redirected(struct sk_buff *skb, bool from_ingress)
5191{
5192 skb->redirected = 1;
5193#ifdef CONFIG_NET_REDIRECT
5194 skb->from_ingress = from_ingress;
5195 if (skb->from_ingress)
5196 skb_clear_tstamp(skb);
5197#endif
5198}
5199
5200static inline void skb_reset_redirect(struct sk_buff *skb)
5201{
5202 skb->redirected = 0;
5203}
5204
5205static inline void skb_set_redirected_noclear(struct sk_buff *skb,
5206 bool from_ingress)
5207{
5208 skb->redirected = 1;
5209#ifdef CONFIG_NET_REDIRECT
5210 skb->from_ingress = from_ingress;
5211#endif
5212}
5213
5214static inline bool skb_csum_is_sctp(struct sk_buff *skb)
5215{
5216#if IS_ENABLED(CONFIG_IP_SCTP)
5217 return skb->csum_not_inet;
5218#else
5219 return 0;
5220#endif
5221}
5222
5223static inline void skb_reset_csum_not_inet(struct sk_buff *skb)
5224{
5225 skb->ip_summed = CHECKSUM_NONE;
5226#if IS_ENABLED(CONFIG_IP_SCTP)
5227 skb->csum_not_inet = 0;
5228#endif
5229}
5230
5231static inline void skb_set_kcov_handle(struct sk_buff *skb,
5232 const u64 kcov_handle)
5233{
5234#ifdef CONFIG_KCOV
5235 skb->kcov_handle = kcov_handle;
5236#endif
5237}
5238
5239static inline u64 skb_get_kcov_handle(struct sk_buff *skb)
5240{
5241#ifdef CONFIG_KCOV
5242 return skb->kcov_handle;
5243#else
5244 return 0;
5245#endif
5246}
5247
5248static inline void skb_mark_for_recycle(struct sk_buff *skb)
5249{
5250#ifdef CONFIG_PAGE_POOL
5251 skb->pp_recycle = 1;
5252#endif
5253}
5254
5255ssize_t skb_splice_from_iter(struct sk_buff *skb, struct iov_iter *iter,
5256 ssize_t maxsize, gfp_t gfp);
5257
5258#endif /* __KERNEL__ */
5259#endif /* _LINUX_SKBUFF_H */
5260

source code of linux/include/linux/skbuff.h