1// SPDX-License-Identifier: GPL-2.0
2
3#include <linux/mutex.h>
4#include <linux/netdevice.h>
5#include <linux/xarray.h>
6#include <net/busy_poll.h>
7#include <net/net_debug.h>
8#include <net/netdev_rx_queue.h>
9#include <net/page_pool/helpers.h>
10#include <net/page_pool/types.h>
11#include <net/page_pool/memory_provider.h>
12#include <net/sock.h>
13
14#include "page_pool_priv.h"
15#include "netdev-genl-gen.h"
16
17static DEFINE_XARRAY_FLAGS(page_pools, XA_FLAGS_ALLOC1);
18/* Protects: page_pools, netdevice->page_pools, pool->p.napi, pool->slow.netdev,
19 * pool->user.
20 * Ordering: inside rtnl_lock
21 */
22DEFINE_MUTEX(page_pools_lock);
23
24/* Page pools are only reachable from user space (via netlink) if they are
25 * linked to a netdev at creation time. Following page pool "visibility"
26 * states are possible:
27 * - normal
28 * - user.list: linked to real netdev, netdev: real netdev
29 * - orphaned - real netdev has disappeared
30 * - user.list: linked to lo, netdev: lo
31 * - invisible - either (a) created without netdev linking, (b) unlisted due
32 * to error, or (c) the entire namespace which owned this pool disappeared
33 * - user.list: unhashed, netdev: unknown
34 */
35
36typedef int (*pp_nl_fill_cb)(struct sk_buff *rsp, const struct page_pool *pool,
37 const struct genl_info *info);
38
39static int
40netdev_nl_page_pool_get_do(struct genl_info *info, u32 id, pp_nl_fill_cb fill)
41{
42 struct page_pool *pool;
43 struct sk_buff *rsp;
44 int err;
45
46 mutex_lock(&page_pools_lock);
47 pool = xa_load(&page_pools, index: id);
48 if (!pool || hlist_unhashed(h: &pool->user.list) ||
49 !net_eq(net1: dev_net(dev: pool->slow.netdev), net2: genl_info_net(info))) {
50 err = -ENOENT;
51 goto err_unlock;
52 }
53
54 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
55 if (!rsp) {
56 err = -ENOMEM;
57 goto err_unlock;
58 }
59
60 err = fill(rsp, pool, info);
61 if (err)
62 goto err_free_msg;
63
64 mutex_unlock(lock: &page_pools_lock);
65
66 return genlmsg_reply(skb: rsp, info);
67
68err_free_msg:
69 nlmsg_free(skb: rsp);
70err_unlock:
71 mutex_unlock(lock: &page_pools_lock);
72 return err;
73}
74
75struct page_pool_dump_cb {
76 unsigned long ifindex;
77 u32 pp_id;
78};
79
80static int
81netdev_nl_page_pool_get_dump(struct sk_buff *skb, struct netlink_callback *cb,
82 pp_nl_fill_cb fill)
83{
84 struct page_pool_dump_cb *state = (void *)cb->ctx;
85 const struct genl_info *info = genl_info_dump(cb);
86 struct net *net = sock_net(sk: skb->sk);
87 struct net_device *netdev;
88 struct page_pool *pool;
89 int err = 0;
90
91 rtnl_lock();
92 mutex_lock(&page_pools_lock);
93 for_each_netdev_dump(net, netdev, state->ifindex) {
94 hlist_for_each_entry(pool, &netdev->page_pools, user.list) {
95 if (state->pp_id && state->pp_id < pool->user.id)
96 continue;
97
98 state->pp_id = pool->user.id;
99 err = fill(skb, pool, info);
100 if (err)
101 goto out;
102 }
103
104 state->pp_id = 0;
105 }
106out:
107 mutex_unlock(lock: &page_pools_lock);
108 rtnl_unlock();
109
110 return err;
111}
112
113static int
114page_pool_nl_stats_fill(struct sk_buff *rsp, const struct page_pool *pool,
115 const struct genl_info *info)
116{
117#ifdef CONFIG_PAGE_POOL_STATS
118 struct page_pool_stats stats = {};
119 struct nlattr *nest;
120 void *hdr;
121
122 if (!page_pool_get_stats(pool, stats: &stats))
123 return 0;
124
125 hdr = genlmsg_iput(skb: rsp, info);
126 if (!hdr)
127 return -EMSGSIZE;
128
129 nest = nla_nest_start(skb: rsp, attrtype: NETDEV_A_PAGE_POOL_STATS_INFO);
130
131 if (nla_put_uint(skb: rsp, attrtype: NETDEV_A_PAGE_POOL_ID, value: pool->user.id) ||
132 (pool->slow.netdev->ifindex != LOOPBACK_IFINDEX &&
133 nla_put_u32(skb: rsp, attrtype: NETDEV_A_PAGE_POOL_IFINDEX,
134 value: pool->slow.netdev->ifindex)))
135 goto err_cancel_nest;
136
137 nla_nest_end(skb: rsp, start: nest);
138
139 if (nla_put_uint(skb: rsp, attrtype: NETDEV_A_PAGE_POOL_STATS_ALLOC_FAST,
140 value: stats.alloc_stats.fast) ||
141 nla_put_uint(skb: rsp, attrtype: NETDEV_A_PAGE_POOL_STATS_ALLOC_SLOW,
142 value: stats.alloc_stats.slow) ||
143 nla_put_uint(skb: rsp, attrtype: NETDEV_A_PAGE_POOL_STATS_ALLOC_SLOW_HIGH_ORDER,
144 value: stats.alloc_stats.slow_high_order) ||
145 nla_put_uint(skb: rsp, attrtype: NETDEV_A_PAGE_POOL_STATS_ALLOC_EMPTY,
146 value: stats.alloc_stats.empty) ||
147 nla_put_uint(skb: rsp, attrtype: NETDEV_A_PAGE_POOL_STATS_ALLOC_REFILL,
148 value: stats.alloc_stats.refill) ||
149 nla_put_uint(skb: rsp, attrtype: NETDEV_A_PAGE_POOL_STATS_ALLOC_WAIVE,
150 value: stats.alloc_stats.waive) ||
151 nla_put_uint(skb: rsp, attrtype: NETDEV_A_PAGE_POOL_STATS_RECYCLE_CACHED,
152 value: stats.recycle_stats.cached) ||
153 nla_put_uint(skb: rsp, attrtype: NETDEV_A_PAGE_POOL_STATS_RECYCLE_CACHE_FULL,
154 value: stats.recycle_stats.cache_full) ||
155 nla_put_uint(skb: rsp, attrtype: NETDEV_A_PAGE_POOL_STATS_RECYCLE_RING,
156 value: stats.recycle_stats.ring) ||
157 nla_put_uint(skb: rsp, attrtype: NETDEV_A_PAGE_POOL_STATS_RECYCLE_RING_FULL,
158 value: stats.recycle_stats.ring_full) ||
159 nla_put_uint(skb: rsp, attrtype: NETDEV_A_PAGE_POOL_STATS_RECYCLE_RELEASED_REFCNT,
160 value: stats.recycle_stats.released_refcnt))
161 goto err_cancel_msg;
162
163 genlmsg_end(skb: rsp, hdr);
164
165 return 0;
166err_cancel_nest:
167 nla_nest_cancel(skb: rsp, start: nest);
168err_cancel_msg:
169 genlmsg_cancel(skb: rsp, hdr);
170 return -EMSGSIZE;
171#else
172 GENL_SET_ERR_MSG(info, "kernel built without CONFIG_PAGE_POOL_STATS");
173 return -EOPNOTSUPP;
174#endif
175}
176
177int netdev_nl_page_pool_stats_get_doit(struct sk_buff *skb,
178 struct genl_info *info)
179{
180 struct nlattr *tb[ARRAY_SIZE(netdev_page_pool_info_nl_policy)];
181 struct nlattr *nest;
182 int err;
183 u32 id;
184
185 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_PAGE_POOL_STATS_INFO))
186 return -EINVAL;
187
188 nest = info->attrs[NETDEV_A_PAGE_POOL_STATS_INFO];
189 err = nla_parse_nested(tb, ARRAY_SIZE(tb) - 1, nla: nest,
190 policy: netdev_page_pool_info_nl_policy,
191 extack: info->extack);
192 if (err)
193 return err;
194
195 if (NL_REQ_ATTR_CHECK(info->extack, nest, tb, NETDEV_A_PAGE_POOL_ID))
196 return -EINVAL;
197 if (tb[NETDEV_A_PAGE_POOL_IFINDEX]) {
198 NL_SET_ERR_MSG_ATTR(info->extack,
199 tb[NETDEV_A_PAGE_POOL_IFINDEX],
200 "selecting by ifindex not supported");
201 return -EINVAL;
202 }
203
204 id = nla_get_uint(nla: tb[NETDEV_A_PAGE_POOL_ID]);
205
206 return netdev_nl_page_pool_get_do(info, id, fill: page_pool_nl_stats_fill);
207}
208
209int netdev_nl_page_pool_stats_get_dumpit(struct sk_buff *skb,
210 struct netlink_callback *cb)
211{
212 return netdev_nl_page_pool_get_dump(skb, cb, fill: page_pool_nl_stats_fill);
213}
214
215static int
216page_pool_nl_fill(struct sk_buff *rsp, const struct page_pool *pool,
217 const struct genl_info *info)
218{
219 size_t inflight, refsz;
220 unsigned int napi_id;
221 void *hdr;
222
223 hdr = genlmsg_iput(skb: rsp, info);
224 if (!hdr)
225 return -EMSGSIZE;
226
227 if (nla_put_uint(skb: rsp, attrtype: NETDEV_A_PAGE_POOL_ID, value: pool->user.id))
228 goto err_cancel;
229
230 if (pool->slow.netdev->ifindex != LOOPBACK_IFINDEX &&
231 nla_put_u32(skb: rsp, attrtype: NETDEV_A_PAGE_POOL_IFINDEX,
232 value: pool->slow.netdev->ifindex))
233 goto err_cancel;
234
235 napi_id = pool->p.napi ? READ_ONCE(pool->p.napi->napi_id) : 0;
236 if (napi_id_valid(napi_id) &&
237 nla_put_uint(skb: rsp, attrtype: NETDEV_A_PAGE_POOL_NAPI_ID, value: napi_id))
238 goto err_cancel;
239
240 inflight = page_pool_inflight(pool, strict: false);
241 refsz = PAGE_SIZE << pool->p.order;
242 if (nla_put_uint(skb: rsp, attrtype: NETDEV_A_PAGE_POOL_INFLIGHT, value: inflight) ||
243 nla_put_uint(skb: rsp, attrtype: NETDEV_A_PAGE_POOL_INFLIGHT_MEM,
244 value: inflight * refsz))
245 goto err_cancel;
246 if (pool->user.detach_time &&
247 nla_put_uint(skb: rsp, attrtype: NETDEV_A_PAGE_POOL_DETACH_TIME,
248 value: pool->user.detach_time))
249 goto err_cancel;
250
251 if (pool->mp_ops && pool->mp_ops->nl_fill(pool->mp_priv, rsp, NULL))
252 goto err_cancel;
253
254 genlmsg_end(skb: rsp, hdr);
255
256 return 0;
257err_cancel:
258 genlmsg_cancel(skb: rsp, hdr);
259 return -EMSGSIZE;
260}
261
262static void netdev_nl_page_pool_event(const struct page_pool *pool, u32 cmd)
263{
264 struct genl_info info;
265 struct sk_buff *ntf;
266 struct net *net;
267
268 lockdep_assert_held(&page_pools_lock);
269
270 /* 'invisible' page pools don't matter */
271 if (hlist_unhashed(h: &pool->user.list))
272 return;
273 net = dev_net(dev: pool->slow.netdev);
274
275 if (!genl_has_listeners(family: &netdev_nl_family, net, group: NETDEV_NLGRP_PAGE_POOL))
276 return;
277
278 genl_info_init_ntf(info: &info, family: &netdev_nl_family, cmd);
279
280 ntf = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
281 if (!ntf)
282 return;
283
284 if (page_pool_nl_fill(rsp: ntf, pool, info: &info)) {
285 nlmsg_free(skb: ntf);
286 return;
287 }
288
289 genlmsg_multicast_netns(family: &netdev_nl_family, net, skb: ntf,
290 portid: 0, group: NETDEV_NLGRP_PAGE_POOL, GFP_KERNEL);
291}
292
293int netdev_nl_page_pool_get_doit(struct sk_buff *skb, struct genl_info *info)
294{
295 u32 id;
296
297 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_PAGE_POOL_ID))
298 return -EINVAL;
299
300 id = nla_get_uint(nla: info->attrs[NETDEV_A_PAGE_POOL_ID]);
301
302 return netdev_nl_page_pool_get_do(info, id, fill: page_pool_nl_fill);
303}
304
305int netdev_nl_page_pool_get_dumpit(struct sk_buff *skb,
306 struct netlink_callback *cb)
307{
308 return netdev_nl_page_pool_get_dump(skb, cb, fill: page_pool_nl_fill);
309}
310
311int page_pool_list(struct page_pool *pool)
312{
313 static u32 id_alloc_next;
314 int err;
315
316 mutex_lock(&page_pools_lock);
317 err = xa_alloc_cyclic(xa: &page_pools, id: &pool->user.id, entry: pool, xa_limit_32b,
318 next: &id_alloc_next, GFP_KERNEL);
319 if (err < 0)
320 goto err_unlock;
321
322 INIT_HLIST_NODE(h: &pool->user.list);
323 if (pool->slow.netdev) {
324 hlist_add_head(n: &pool->user.list,
325 h: &pool->slow.netdev->page_pools);
326 netdev_nl_page_pool_event(pool, cmd: NETDEV_CMD_PAGE_POOL_ADD_NTF);
327 }
328
329 mutex_unlock(lock: &page_pools_lock);
330 return 0;
331
332err_unlock:
333 mutex_unlock(lock: &page_pools_lock);
334 return err;
335}
336
337void page_pool_detached(struct page_pool *pool)
338{
339 mutex_lock(&page_pools_lock);
340 pool->user.detach_time = ktime_get_boottime_seconds();
341 netdev_nl_page_pool_event(pool, cmd: NETDEV_CMD_PAGE_POOL_CHANGE_NTF);
342 mutex_unlock(lock: &page_pools_lock);
343}
344
345void page_pool_unlist(struct page_pool *pool)
346{
347 mutex_lock(&page_pools_lock);
348 netdev_nl_page_pool_event(pool, cmd: NETDEV_CMD_PAGE_POOL_DEL_NTF);
349 xa_erase(&page_pools, index: pool->user.id);
350 if (!hlist_unhashed(h: &pool->user.list))
351 hlist_del(n: &pool->user.list);
352 mutex_unlock(lock: &page_pools_lock);
353}
354
355int page_pool_check_memory_provider(struct net_device *dev,
356 struct netdev_rx_queue *rxq)
357{
358 void *binding = rxq->mp_params.mp_priv;
359 struct page_pool *pool;
360 struct hlist_node *n;
361
362 if (!binding)
363 return 0;
364
365 mutex_lock(&page_pools_lock);
366 hlist_for_each_entry_safe(pool, n, &dev->page_pools, user.list) {
367 if (pool->mp_priv != binding)
368 continue;
369
370 if (pool->slow.queue_idx == get_netdev_rx_queue_index(queue: rxq)) {
371 mutex_unlock(lock: &page_pools_lock);
372 return 0;
373 }
374 }
375 mutex_unlock(lock: &page_pools_lock);
376 return -ENODATA;
377}
378
379static void page_pool_unreg_netdev_wipe(struct net_device *netdev)
380{
381 struct page_pool *pool;
382 struct hlist_node *n;
383
384 mutex_lock(&page_pools_lock);
385 hlist_for_each_entry_safe(pool, n, &netdev->page_pools, user.list) {
386 hlist_del_init(n: &pool->user.list);
387 pool->slow.netdev = NET_PTR_POISON;
388 }
389 mutex_unlock(lock: &page_pools_lock);
390}
391
392static void page_pool_unreg_netdev(struct net_device *netdev)
393{
394 struct page_pool *pool, *last;
395 struct net_device *lo;
396
397 lo = dev_net(dev: netdev)->loopback_dev;
398
399 mutex_lock(&page_pools_lock);
400 last = NULL;
401 hlist_for_each_entry(pool, &netdev->page_pools, user.list) {
402 pool->slow.netdev = lo;
403 netdev_nl_page_pool_event(pool,
404 cmd: NETDEV_CMD_PAGE_POOL_CHANGE_NTF);
405 last = pool;
406 }
407 if (last)
408 hlist_splice_init(from: &netdev->page_pools, last: &last->user.list,
409 to: &lo->page_pools);
410 mutex_unlock(lock: &page_pools_lock);
411}
412
413static int
414page_pool_netdevice_event(struct notifier_block *nb,
415 unsigned long event, void *ptr)
416{
417 struct net_device *netdev = netdev_notifier_info_to_dev(info: ptr);
418
419 if (event != NETDEV_UNREGISTER)
420 return NOTIFY_DONE;
421
422 if (hlist_empty(h: &netdev->page_pools))
423 return NOTIFY_OK;
424
425 if (netdev->ifindex != LOOPBACK_IFINDEX)
426 page_pool_unreg_netdev(netdev);
427 else
428 page_pool_unreg_netdev_wipe(netdev);
429 return NOTIFY_OK;
430}
431
432static struct notifier_block page_pool_netdevice_nb = {
433 .notifier_call = page_pool_netdevice_event,
434};
435
436static int __init page_pool_user_init(void)
437{
438 return register_netdevice_notifier(nb: &page_pool_netdevice_nb);
439}
440
441subsys_initcall(page_pool_user_init);
442

source code of linux/net/core/page_pool_user.c