1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | /* Copyright(c) 2020 Intel Corporation. */ |
3 | |
4 | #ifndef XSK_BUFF_POOL_H_ |
5 | #define XSK_BUFF_POOL_H_ |
6 | |
7 | #include <linux/if_xdp.h> |
8 | #include <linux/types.h> |
9 | #include <linux/dma-mapping.h> |
10 | #include <linux/bpf.h> |
11 | #include <net/xdp.h> |
12 | |
13 | struct xsk_buff_pool; |
14 | struct xdp_rxq_info; |
15 | struct xsk_queue; |
16 | struct xdp_desc; |
17 | struct xdp_umem; |
18 | struct xdp_sock; |
19 | struct device; |
20 | struct page; |
21 | |
22 | #define XSK_PRIV_MAX 24 |
23 | |
24 | struct xdp_buff_xsk { |
25 | struct xdp_buff xdp; |
26 | u8 cb[XSK_PRIV_MAX]; |
27 | dma_addr_t dma; |
28 | dma_addr_t frame_dma; |
29 | struct xsk_buff_pool *pool; |
30 | u64 orig_addr; |
31 | struct list_head free_list_node; |
32 | struct list_head xskb_list_node; |
33 | }; |
34 | |
35 | #define XSK_CHECK_PRIV_TYPE(t) BUILD_BUG_ON(sizeof(t) > offsetofend(struct xdp_buff_xsk, cb)) |
36 | |
37 | struct xsk_dma_map { |
38 | dma_addr_t *dma_pages; |
39 | struct device *dev; |
40 | struct net_device *netdev; |
41 | refcount_t users; |
42 | struct list_head list; /* Protected by the RTNL_LOCK */ |
43 | u32 dma_pages_cnt; |
44 | bool dma_need_sync; |
45 | }; |
46 | |
47 | struct xsk_buff_pool { |
48 | /* Members only used in the control path first. */ |
49 | struct device *dev; |
50 | struct net_device *netdev; |
51 | struct list_head xsk_tx_list; |
52 | /* Protects modifications to the xsk_tx_list */ |
53 | spinlock_t xsk_tx_list_lock; |
54 | refcount_t users; |
55 | struct xdp_umem *umem; |
56 | struct work_struct work; |
57 | struct list_head free_list; |
58 | struct list_head xskb_list; |
59 | u32 heads_cnt; |
60 | u16 queue_id; |
61 | |
62 | /* Data path members as close to free_heads at the end as possible. */ |
63 | struct xsk_queue *fq ____cacheline_aligned_in_smp; |
64 | struct xsk_queue *cq; |
65 | /* For performance reasons, each buff pool has its own array of dma_pages |
66 | * even when they are identical. |
67 | */ |
68 | dma_addr_t *dma_pages; |
69 | struct xdp_buff_xsk *heads; |
70 | struct xdp_desc *tx_descs; |
71 | u64 chunk_mask; |
72 | u64 addrs_cnt; |
73 | u32 free_list_cnt; |
74 | u32 dma_pages_cnt; |
75 | u32 free_heads_cnt; |
76 | u32 headroom; |
77 | u32 chunk_size; |
78 | u32 chunk_shift; |
79 | u32 frame_len; |
80 | u8 cached_need_wakeup; |
81 | bool uses_need_wakeup; |
82 | bool dma_need_sync; |
83 | bool unaligned; |
84 | void *addrs; |
85 | /* Mutual exclusion of the completion ring in the SKB mode. Two cases to protect: |
86 | * NAPI TX thread and sendmsg error paths in the SKB destructor callback and when |
87 | * sockets share a single cq when the same netdev and queue id is shared. |
88 | */ |
89 | spinlock_t cq_lock; |
90 | struct xdp_buff_xsk *free_heads[]; |
91 | }; |
92 | |
93 | /* Masks for xdp_umem_page flags. |
94 | * The low 12-bits of the addr will be 0 since this is the page address, so we |
95 | * can use them for flags. |
96 | */ |
97 | #define XSK_NEXT_PG_CONTIG_SHIFT 0 |
98 | #define XSK_NEXT_PG_CONTIG_MASK BIT_ULL(XSK_NEXT_PG_CONTIG_SHIFT) |
99 | |
100 | /* AF_XDP core. */ |
101 | struct xsk_buff_pool *xp_create_and_assign_umem(struct xdp_sock *xs, |
102 | struct xdp_umem *umem); |
103 | int xp_assign_dev(struct xsk_buff_pool *pool, struct net_device *dev, |
104 | u16 queue_id, u16 flags); |
105 | int xp_assign_dev_shared(struct xsk_buff_pool *pool, struct xdp_sock *umem_xs, |
106 | struct net_device *dev, u16 queue_id); |
107 | int xp_alloc_tx_descs(struct xsk_buff_pool *pool, struct xdp_sock *xs); |
108 | void xp_destroy(struct xsk_buff_pool *pool); |
109 | void xp_get_pool(struct xsk_buff_pool *pool); |
110 | bool xp_put_pool(struct xsk_buff_pool *pool); |
111 | void xp_clear_dev(struct xsk_buff_pool *pool); |
112 | void xp_add_xsk(struct xsk_buff_pool *pool, struct xdp_sock *xs); |
113 | void xp_del_xsk(struct xsk_buff_pool *pool, struct xdp_sock *xs); |
114 | |
115 | /* AF_XDP, and XDP core. */ |
116 | void xp_free(struct xdp_buff_xsk *xskb); |
117 | |
118 | static inline void xp_init_xskb_addr(struct xdp_buff_xsk *xskb, struct xsk_buff_pool *pool, |
119 | u64 addr) |
120 | { |
121 | xskb->orig_addr = addr; |
122 | xskb->xdp.data_hard_start = pool->addrs + addr + pool->headroom; |
123 | } |
124 | |
125 | static inline void xp_init_xskb_dma(struct xdp_buff_xsk *xskb, struct xsk_buff_pool *pool, |
126 | dma_addr_t *dma_pages, u64 addr) |
127 | { |
128 | xskb->frame_dma = (dma_pages[addr >> PAGE_SHIFT] & ~XSK_NEXT_PG_CONTIG_MASK) + |
129 | (addr & ~PAGE_MASK); |
130 | xskb->dma = xskb->frame_dma + pool->headroom + XDP_PACKET_HEADROOM; |
131 | } |
132 | |
133 | /* AF_XDP ZC drivers, via xdp_sock_buff.h */ |
134 | void xp_set_rxq_info(struct xsk_buff_pool *pool, struct xdp_rxq_info *rxq); |
135 | int xp_dma_map(struct xsk_buff_pool *pool, struct device *dev, |
136 | unsigned long attrs, struct page **pages, u32 nr_pages); |
137 | void xp_dma_unmap(struct xsk_buff_pool *pool, unsigned long attrs); |
138 | struct xdp_buff *xp_alloc(struct xsk_buff_pool *pool); |
139 | u32 xp_alloc_batch(struct xsk_buff_pool *pool, struct xdp_buff **xdp, u32 max); |
140 | bool xp_can_alloc(struct xsk_buff_pool *pool, u32 count); |
141 | void *xp_raw_get_data(struct xsk_buff_pool *pool, u64 addr); |
142 | dma_addr_t xp_raw_get_dma(struct xsk_buff_pool *pool, u64 addr); |
143 | static inline dma_addr_t xp_get_dma(struct xdp_buff_xsk *xskb) |
144 | { |
145 | return xskb->dma; |
146 | } |
147 | |
148 | static inline dma_addr_t xp_get_frame_dma(struct xdp_buff_xsk *xskb) |
149 | { |
150 | return xskb->frame_dma; |
151 | } |
152 | |
153 | void xp_dma_sync_for_cpu_slow(struct xdp_buff_xsk *xskb); |
154 | static inline void xp_dma_sync_for_cpu(struct xdp_buff_xsk *xskb) |
155 | { |
156 | xp_dma_sync_for_cpu_slow(xskb); |
157 | } |
158 | |
159 | void xp_dma_sync_for_device_slow(struct xsk_buff_pool *pool, dma_addr_t dma, |
160 | size_t size); |
161 | static inline void xp_dma_sync_for_device(struct xsk_buff_pool *pool, |
162 | dma_addr_t dma, size_t size) |
163 | { |
164 | if (!pool->dma_need_sync) |
165 | return; |
166 | |
167 | xp_dma_sync_for_device_slow(pool, dma, size); |
168 | } |
169 | |
170 | /* Masks for xdp_umem_page flags. |
171 | * The low 12-bits of the addr will be 0 since this is the page address, so we |
172 | * can use them for flags. |
173 | */ |
174 | #define XSK_NEXT_PG_CONTIG_SHIFT 0 |
175 | #define XSK_NEXT_PG_CONTIG_MASK BIT_ULL(XSK_NEXT_PG_CONTIG_SHIFT) |
176 | |
177 | static inline bool xp_desc_crosses_non_contig_pg(struct xsk_buff_pool *pool, |
178 | u64 addr, u32 len) |
179 | { |
180 | bool cross_pg = (addr & (PAGE_SIZE - 1)) + len > PAGE_SIZE; |
181 | |
182 | if (likely(!cross_pg)) |
183 | return false; |
184 | |
185 | return pool->dma_pages && |
186 | !(pool->dma_pages[addr >> PAGE_SHIFT] & XSK_NEXT_PG_CONTIG_MASK); |
187 | } |
188 | |
189 | static inline bool xp_mb_desc(struct xdp_desc *desc) |
190 | { |
191 | return desc->options & XDP_PKT_CONTD; |
192 | } |
193 | |
194 | static inline u64 (struct xsk_buff_pool *pool, u64 addr) |
195 | { |
196 | return addr & pool->chunk_mask; |
197 | } |
198 | |
199 | static inline u64 (u64 addr) |
200 | { |
201 | return addr & XSK_UNALIGNED_BUF_ADDR_MASK; |
202 | } |
203 | |
204 | static inline u64 (u64 addr) |
205 | { |
206 | return addr >> XSK_UNALIGNED_BUF_OFFSET_SHIFT; |
207 | } |
208 | |
209 | static inline u64 xp_unaligned_add_offset_to_addr(u64 addr) |
210 | { |
211 | return xp_unaligned_extract_addr(addr) + |
212 | xp_unaligned_extract_offset(addr); |
213 | } |
214 | |
215 | static inline u32 (struct xsk_buff_pool *pool, u64 addr) |
216 | { |
217 | return xp_aligned_extract_addr(pool, addr) >> pool->chunk_shift; |
218 | } |
219 | |
220 | static inline void xp_release(struct xdp_buff_xsk *xskb) |
221 | { |
222 | if (xskb->pool->unaligned) |
223 | xskb->pool->free_heads[xskb->pool->free_heads_cnt++] = xskb; |
224 | } |
225 | |
226 | static inline u64 xp_get_handle(struct xdp_buff_xsk *xskb) |
227 | { |
228 | u64 offset = xskb->xdp.data - xskb->xdp.data_hard_start; |
229 | |
230 | offset += xskb->pool->headroom; |
231 | if (!xskb->pool->unaligned) |
232 | return xskb->orig_addr + offset; |
233 | return xskb->orig_addr + (offset << XSK_UNALIGNED_BUF_OFFSET_SHIFT); |
234 | } |
235 | |
236 | #endif /* XSK_BUFF_POOL_H_ */ |
237 | |