1/* SPDX-License-Identifier: GPL-2.0 */
2
3#ifndef _NET_PAGE_POOL_TYPES_H
4#define _NET_PAGE_POOL_TYPES_H
5
6#include <linux/dma-direction.h>
7#include <linux/ptr_ring.h>
8#include <linux/types.h>
9
10#define PP_FLAG_DMA_MAP BIT(0) /* Should page_pool do the DMA
11 * map/unmap
12 */
13#define PP_FLAG_DMA_SYNC_DEV BIT(1) /* If set all pages that the driver gets
14 * from page_pool will be
15 * DMA-synced-for-device according to
16 * the length provided by the device
17 * driver.
18 * Please note DMA-sync-for-CPU is still
19 * device driver responsibility
20 */
21#define PP_FLAG_SYSTEM_POOL BIT(2) /* Global system page_pool */
22#define PP_FLAG_ALL (PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV | \
23 PP_FLAG_SYSTEM_POOL)
24
25/*
26 * Fast allocation side cache array/stack
27 *
28 * The cache size and refill watermark is related to the network
29 * use-case. The NAPI budget is 64 packets. After a NAPI poll the RX
30 * ring is usually refilled and the max consumed elements will be 64,
31 * thus a natural max size of objects needed in the cache.
32 *
33 * Keeping room for more objects, is due to XDP_DROP use-case. As
34 * XDP_DROP allows the opportunity to recycle objects directly into
35 * this array, as it shares the same softirq/NAPI protection. If
36 * cache is already full (or partly full) then the XDP_DROP recycles
37 * would have to take a slower code path.
38 */
39#define PP_ALLOC_CACHE_SIZE 128
40#define PP_ALLOC_CACHE_REFILL 64
41struct pp_alloc_cache {
42 u32 count;
43 struct page *cache[PP_ALLOC_CACHE_SIZE];
44};
45
46/**
47 * struct page_pool_params - page pool parameters
48 * @flags: PP_FLAG_DMA_MAP, PP_FLAG_DMA_SYNC_DEV
49 * @order: 2^order pages on allocation
50 * @pool_size: size of the ptr_ring
51 * @nid: NUMA node id to allocate from pages from
52 * @dev: device, for DMA pre-mapping purposes
53 * @netdev: netdev this pool will serve (leave as NULL if none or multiple)
54 * @napi: NAPI which is the sole consumer of pages, otherwise NULL
55 * @dma_dir: DMA mapping direction
56 * @max_len: max DMA sync memory size for PP_FLAG_DMA_SYNC_DEV
57 * @offset: DMA sync address offset for PP_FLAG_DMA_SYNC_DEV
58 */
59struct page_pool_params {
60 struct_group_tagged(page_pool_params_fast, fast,
61 unsigned int flags;
62 unsigned int order;
63 unsigned int pool_size;
64 int nid;
65 struct device *dev;
66 struct napi_struct *napi;
67 enum dma_data_direction dma_dir;
68 unsigned int max_len;
69 unsigned int offset;
70 );
71 struct_group_tagged(page_pool_params_slow, slow,
72 struct net_device *netdev;
73/* private: used by test code only */
74 void (*init_callback)(struct page *page, void *arg);
75 void *init_arg;
76 );
77};
78
79#ifdef CONFIG_PAGE_POOL_STATS
80/**
81 * struct page_pool_alloc_stats - allocation statistics
82 * @fast: successful fast path allocations
83 * @slow: slow path order-0 allocations
84 * @slow_high_order: slow path high order allocations
85 * @empty: ptr ring is empty, so a slow path allocation was forced
86 * @refill: an allocation which triggered a refill of the cache
87 * @waive: pages obtained from the ptr ring that cannot be added to
88 * the cache due to a NUMA mismatch
89 */
90struct page_pool_alloc_stats {
91 u64 fast;
92 u64 slow;
93 u64 slow_high_order;
94 u64 empty;
95 u64 refill;
96 u64 waive;
97};
98
99/**
100 * struct page_pool_recycle_stats - recycling (freeing) statistics
101 * @cached: recycling placed page in the page pool cache
102 * @cache_full: page pool cache was full
103 * @ring: page placed into the ptr ring
104 * @ring_full: page released from page pool because the ptr ring was full
105 * @released_refcnt: page released (and not recycled) because refcnt > 1
106 */
107struct page_pool_recycle_stats {
108 u64 cached;
109 u64 cache_full;
110 u64 ring;
111 u64 ring_full;
112 u64 released_refcnt;
113};
114
115/**
116 * struct page_pool_stats - combined page pool use statistics
117 * @alloc_stats: see struct page_pool_alloc_stats
118 * @recycle_stats: see struct page_pool_recycle_stats
119 *
120 * Wrapper struct for combining page pool stats with different storage
121 * requirements.
122 */
123struct page_pool_stats {
124 struct page_pool_alloc_stats alloc_stats;
125 struct page_pool_recycle_stats recycle_stats;
126};
127#endif
128
129struct page_pool {
130 struct page_pool_params_fast p;
131
132 int cpuid;
133 bool has_init_callback;
134
135 long frag_users;
136 struct page *frag_page;
137 unsigned int frag_offset;
138 u32 pages_state_hold_cnt;
139
140 struct delayed_work release_dw;
141 void (*disconnect)(void *pool);
142 unsigned long defer_start;
143 unsigned long defer_warn;
144
145#ifdef CONFIG_PAGE_POOL_STATS
146 /* these stats are incremented while in softirq context */
147 struct page_pool_alloc_stats alloc_stats;
148#endif
149 u32 xdp_mem_id;
150
151 /*
152 * Data structure for allocation side
153 *
154 * Drivers allocation side usually already perform some kind
155 * of resource protection. Piggyback on this protection, and
156 * require driver to protect allocation side.
157 *
158 * For NIC drivers this means, allocate a page_pool per
159 * RX-queue. As the RX-queue is already protected by
160 * Softirq/BH scheduling and napi_schedule. NAPI schedule
161 * guarantee that a single napi_struct will only be scheduled
162 * on a single CPU (see napi_schedule).
163 */
164 struct pp_alloc_cache alloc ____cacheline_aligned_in_smp;
165
166 /* Data structure for storing recycled pages.
167 *
168 * Returning/freeing pages is more complicated synchronization
169 * wise, because free's can happen on remote CPUs, with no
170 * association with allocation resource.
171 *
172 * Use ptr_ring, as it separates consumer and producer
173 * efficiently, it a way that doesn't bounce cache-lines.
174 *
175 * TODO: Implement bulk return pages into this structure.
176 */
177 struct ptr_ring ring;
178
179#ifdef CONFIG_PAGE_POOL_STATS
180 /* recycle stats are per-cpu to avoid locking */
181 struct page_pool_recycle_stats __percpu *recycle_stats;
182#endif
183 atomic_t pages_state_release_cnt;
184
185 /* A page_pool is strictly tied to a single RX-queue being
186 * protected by NAPI, due to above pp_alloc_cache. This
187 * refcnt serves purpose is to simplify drivers error handling.
188 */
189 refcount_t user_cnt;
190
191 u64 destroy_cnt;
192
193 /* Slow/Control-path information follows */
194 struct page_pool_params_slow slow;
195 /* User-facing fields, protected by page_pools_lock */
196 struct {
197 struct hlist_node list;
198 u64 detach_time;
199 u32 napi_id;
200 u32 id;
201 } user;
202};
203
204struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp);
205struct page *page_pool_alloc_frag(struct page_pool *pool, unsigned int *offset,
206 unsigned int size, gfp_t gfp);
207struct page_pool *page_pool_create(const struct page_pool_params *params);
208struct page_pool *page_pool_create_percpu(const struct page_pool_params *params,
209 int cpuid);
210
211struct xdp_mem_info;
212
213#ifdef CONFIG_PAGE_POOL
214void page_pool_destroy(struct page_pool *pool);
215void page_pool_use_xdp_mem(struct page_pool *pool, void (*disconnect)(void *),
216 struct xdp_mem_info *mem);
217void page_pool_put_page_bulk(struct page_pool *pool, void **data,
218 int count);
219#else
220static inline void page_pool_destroy(struct page_pool *pool)
221{
222}
223
224static inline void page_pool_use_xdp_mem(struct page_pool *pool,
225 void (*disconnect)(void *),
226 struct xdp_mem_info *mem)
227{
228}
229
230static inline void page_pool_put_page_bulk(struct page_pool *pool, void **data,
231 int count)
232{
233}
234#endif
235
236void page_pool_put_unrefed_page(struct page_pool *pool, struct page *page,
237 unsigned int dma_sync_size,
238 bool allow_direct);
239
240static inline bool is_page_pool_compiled_in(void)
241{
242#ifdef CONFIG_PAGE_POOL
243 return true;
244#else
245 return false;
246#endif
247}
248
249/* Caller must provide appropriate safe context, e.g. NAPI. */
250void page_pool_update_nid(struct page_pool *pool, int new_nid);
251
252#endif /* _NET_PAGE_POOL_H */
253

source code of linux/include/net/page_pool/types.h