1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | /* Interface for implementing AF_XDP zero-copy support in drivers. |
3 | * Copyright(c) 2020 Intel Corporation. |
4 | */ |
5 | |
6 | #ifndef _LINUX_XDP_SOCK_DRV_H |
7 | #define _LINUX_XDP_SOCK_DRV_H |
8 | |
9 | #include <net/xdp_sock.h> |
10 | #include <net/xsk_buff_pool.h> |
11 | |
12 | #define XDP_UMEM_MIN_CHUNK_SHIFT 11 |
13 | #define XDP_UMEM_MIN_CHUNK_SIZE (1 << XDP_UMEM_MIN_CHUNK_SHIFT) |
14 | |
15 | #ifdef CONFIG_XDP_SOCKETS |
16 | |
17 | void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries); |
18 | bool xsk_tx_peek_desc(struct xsk_buff_pool *pool, struct xdp_desc *desc); |
19 | u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 max); |
20 | void xsk_tx_release(struct xsk_buff_pool *pool); |
21 | struct xsk_buff_pool *xsk_get_pool_from_qid(struct net_device *dev, |
22 | u16 queue_id); |
23 | void xsk_set_rx_need_wakeup(struct xsk_buff_pool *pool); |
24 | void xsk_set_tx_need_wakeup(struct xsk_buff_pool *pool); |
25 | void xsk_clear_rx_need_wakeup(struct xsk_buff_pool *pool); |
26 | void xsk_clear_tx_need_wakeup(struct xsk_buff_pool *pool); |
27 | bool xsk_uses_need_wakeup(struct xsk_buff_pool *pool); |
28 | |
29 | static inline u32 xsk_pool_get_headroom(struct xsk_buff_pool *pool) |
30 | { |
31 | return XDP_PACKET_HEADROOM + pool->headroom; |
32 | } |
33 | |
34 | static inline u32 xsk_pool_get_chunk_size(struct xsk_buff_pool *pool) |
35 | { |
36 | return pool->chunk_size; |
37 | } |
38 | |
39 | static inline u32 xsk_pool_get_rx_frame_size(struct xsk_buff_pool *pool) |
40 | { |
41 | return xsk_pool_get_chunk_size(pool) - xsk_pool_get_headroom(pool); |
42 | } |
43 | |
44 | static inline void xsk_pool_set_rxq_info(struct xsk_buff_pool *pool, |
45 | struct xdp_rxq_info *rxq) |
46 | { |
47 | xp_set_rxq_info(pool, rxq); |
48 | } |
49 | |
50 | static inline unsigned int xsk_pool_get_napi_id(struct xsk_buff_pool *pool) |
51 | { |
52 | #ifdef CONFIG_NET_RX_BUSY_POLL |
53 | return pool->heads[0].xdp.rxq->napi_id; |
54 | #else |
55 | return 0; |
56 | #endif |
57 | } |
58 | |
59 | static inline void xsk_pool_dma_unmap(struct xsk_buff_pool *pool, |
60 | unsigned long attrs) |
61 | { |
62 | xp_dma_unmap(pool, attrs); |
63 | } |
64 | |
65 | static inline int xsk_pool_dma_map(struct xsk_buff_pool *pool, |
66 | struct device *dev, unsigned long attrs) |
67 | { |
68 | struct xdp_umem *umem = pool->umem; |
69 | |
70 | return xp_dma_map(pool, dev, attrs, pages: umem->pgs, nr_pages: umem->npgs); |
71 | } |
72 | |
73 | static inline dma_addr_t xsk_buff_xdp_get_dma(struct xdp_buff *xdp) |
74 | { |
75 | struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); |
76 | |
77 | return xp_get_dma(xskb); |
78 | } |
79 | |
80 | static inline dma_addr_t xsk_buff_xdp_get_frame_dma(struct xdp_buff *xdp) |
81 | { |
82 | struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); |
83 | |
84 | return xp_get_frame_dma(xskb); |
85 | } |
86 | |
87 | static inline struct xdp_buff *xsk_buff_alloc(struct xsk_buff_pool *pool) |
88 | { |
89 | return xp_alloc(pool); |
90 | } |
91 | |
92 | static inline bool xsk_is_eop_desc(struct xdp_desc *desc) |
93 | { |
94 | return !xp_mb_desc(desc); |
95 | } |
96 | |
97 | /* Returns as many entries as possible up to max. 0 <= N <= max. */ |
98 | static inline u32 xsk_buff_alloc_batch(struct xsk_buff_pool *pool, struct xdp_buff **xdp, u32 max) |
99 | { |
100 | return xp_alloc_batch(pool, xdp, max); |
101 | } |
102 | |
103 | static inline bool xsk_buff_can_alloc(struct xsk_buff_pool *pool, u32 count) |
104 | { |
105 | return xp_can_alloc(pool, count); |
106 | } |
107 | |
108 | static inline void xsk_buff_free(struct xdp_buff *xdp) |
109 | { |
110 | struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); |
111 | struct list_head *xskb_list = &xskb->pool->xskb_list; |
112 | struct xdp_buff_xsk *pos, *tmp; |
113 | |
114 | if (likely(!xdp_buff_has_frags(xdp))) |
115 | goto out; |
116 | |
117 | list_for_each_entry_safe(pos, tmp, xskb_list, xskb_list_node) { |
118 | list_del(entry: &pos->xskb_list_node); |
119 | xp_free(xskb: pos); |
120 | } |
121 | |
122 | xdp_get_shared_info_from_buff(xdp)->nr_frags = 0; |
123 | out: |
124 | xp_free(xskb); |
125 | } |
126 | |
127 | static inline void xsk_buff_add_frag(struct xdp_buff *xdp) |
128 | { |
129 | struct xdp_buff_xsk *frag = container_of(xdp, struct xdp_buff_xsk, xdp); |
130 | |
131 | list_add_tail(new: &frag->xskb_list_node, head: &frag->pool->xskb_list); |
132 | } |
133 | |
134 | static inline struct xdp_buff *xsk_buff_get_frag(struct xdp_buff *first) |
135 | { |
136 | struct xdp_buff_xsk *xskb = container_of(first, struct xdp_buff_xsk, xdp); |
137 | struct xdp_buff *ret = NULL; |
138 | struct xdp_buff_xsk *frag; |
139 | |
140 | frag = list_first_entry_or_null(&xskb->pool->xskb_list, |
141 | struct xdp_buff_xsk, xskb_list_node); |
142 | if (frag) { |
143 | list_del(entry: &frag->xskb_list_node); |
144 | ret = &frag->xdp; |
145 | } |
146 | |
147 | return ret; |
148 | } |
149 | |
150 | static inline void xsk_buff_set_size(struct xdp_buff *xdp, u32 size) |
151 | { |
152 | xdp->data = xdp->data_hard_start + XDP_PACKET_HEADROOM; |
153 | xdp->data_meta = xdp->data; |
154 | xdp->data_end = xdp->data + size; |
155 | } |
156 | |
157 | static inline dma_addr_t xsk_buff_raw_get_dma(struct xsk_buff_pool *pool, |
158 | u64 addr) |
159 | { |
160 | return xp_raw_get_dma(pool, addr); |
161 | } |
162 | |
163 | static inline void *xsk_buff_raw_get_data(struct xsk_buff_pool *pool, u64 addr) |
164 | { |
165 | return xp_raw_get_data(pool, addr); |
166 | } |
167 | |
168 | static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp, struct xsk_buff_pool *pool) |
169 | { |
170 | struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); |
171 | |
172 | if (!pool->dma_need_sync) |
173 | return; |
174 | |
175 | xp_dma_sync_for_cpu(xskb); |
176 | } |
177 | |
178 | static inline void xsk_buff_raw_dma_sync_for_device(struct xsk_buff_pool *pool, |
179 | dma_addr_t dma, |
180 | size_t size) |
181 | { |
182 | xp_dma_sync_for_device(pool, dma, size); |
183 | } |
184 | |
185 | #else |
186 | |
187 | static inline void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries) |
188 | { |
189 | } |
190 | |
191 | static inline bool xsk_tx_peek_desc(struct xsk_buff_pool *pool, |
192 | struct xdp_desc *desc) |
193 | { |
194 | return false; |
195 | } |
196 | |
197 | static inline u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 max) |
198 | { |
199 | return 0; |
200 | } |
201 | |
202 | static inline void xsk_tx_release(struct xsk_buff_pool *pool) |
203 | { |
204 | } |
205 | |
206 | static inline struct xsk_buff_pool * |
207 | xsk_get_pool_from_qid(struct net_device *dev, u16 queue_id) |
208 | { |
209 | return NULL; |
210 | } |
211 | |
212 | static inline void xsk_set_rx_need_wakeup(struct xsk_buff_pool *pool) |
213 | { |
214 | } |
215 | |
216 | static inline void xsk_set_tx_need_wakeup(struct xsk_buff_pool *pool) |
217 | { |
218 | } |
219 | |
220 | static inline void xsk_clear_rx_need_wakeup(struct xsk_buff_pool *pool) |
221 | { |
222 | } |
223 | |
224 | static inline void xsk_clear_tx_need_wakeup(struct xsk_buff_pool *pool) |
225 | { |
226 | } |
227 | |
228 | static inline bool xsk_uses_need_wakeup(struct xsk_buff_pool *pool) |
229 | { |
230 | return false; |
231 | } |
232 | |
233 | static inline u32 xsk_pool_get_headroom(struct xsk_buff_pool *pool) |
234 | { |
235 | return 0; |
236 | } |
237 | |
238 | static inline u32 xsk_pool_get_chunk_size(struct xsk_buff_pool *pool) |
239 | { |
240 | return 0; |
241 | } |
242 | |
243 | static inline u32 xsk_pool_get_rx_frame_size(struct xsk_buff_pool *pool) |
244 | { |
245 | return 0; |
246 | } |
247 | |
248 | static inline void xsk_pool_set_rxq_info(struct xsk_buff_pool *pool, |
249 | struct xdp_rxq_info *rxq) |
250 | { |
251 | } |
252 | |
253 | static inline unsigned int xsk_pool_get_napi_id(struct xsk_buff_pool *pool) |
254 | { |
255 | return 0; |
256 | } |
257 | |
258 | static inline void xsk_pool_dma_unmap(struct xsk_buff_pool *pool, |
259 | unsigned long attrs) |
260 | { |
261 | } |
262 | |
263 | static inline int xsk_pool_dma_map(struct xsk_buff_pool *pool, |
264 | struct device *dev, unsigned long attrs) |
265 | { |
266 | return 0; |
267 | } |
268 | |
269 | static inline dma_addr_t xsk_buff_xdp_get_dma(struct xdp_buff *xdp) |
270 | { |
271 | return 0; |
272 | } |
273 | |
274 | static inline dma_addr_t xsk_buff_xdp_get_frame_dma(struct xdp_buff *xdp) |
275 | { |
276 | return 0; |
277 | } |
278 | |
279 | static inline struct xdp_buff *xsk_buff_alloc(struct xsk_buff_pool *pool) |
280 | { |
281 | return NULL; |
282 | } |
283 | |
284 | static inline bool xsk_is_eop_desc(struct xdp_desc *desc) |
285 | { |
286 | return false; |
287 | } |
288 | |
289 | static inline u32 xsk_buff_alloc_batch(struct xsk_buff_pool *pool, struct xdp_buff **xdp, u32 max) |
290 | { |
291 | return 0; |
292 | } |
293 | |
294 | static inline bool xsk_buff_can_alloc(struct xsk_buff_pool *pool, u32 count) |
295 | { |
296 | return false; |
297 | } |
298 | |
299 | static inline void xsk_buff_free(struct xdp_buff *xdp) |
300 | { |
301 | } |
302 | |
303 | static inline void xsk_buff_add_frag(struct xdp_buff *xdp) |
304 | { |
305 | } |
306 | |
307 | static inline struct xdp_buff *xsk_buff_get_frag(struct xdp_buff *first) |
308 | { |
309 | return NULL; |
310 | } |
311 | |
312 | static inline void xsk_buff_set_size(struct xdp_buff *xdp, u32 size) |
313 | { |
314 | } |
315 | |
316 | static inline dma_addr_t xsk_buff_raw_get_dma(struct xsk_buff_pool *pool, |
317 | u64 addr) |
318 | { |
319 | return 0; |
320 | } |
321 | |
322 | static inline void *xsk_buff_raw_get_data(struct xsk_buff_pool *pool, u64 addr) |
323 | { |
324 | return NULL; |
325 | } |
326 | |
327 | static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp, struct xsk_buff_pool *pool) |
328 | { |
329 | } |
330 | |
331 | static inline void xsk_buff_raw_dma_sync_for_device(struct xsk_buff_pool *pool, |
332 | dma_addr_t dma, |
333 | size_t size) |
334 | { |
335 | } |
336 | |
337 | #endif /* CONFIG_XDP_SOCKETS */ |
338 | |
339 | #endif /* _LINUX_XDP_SOCK_DRV_H */ |
340 | |