1// SPDX-License-Identifier: GPL-2.0-or-later
2
3#include <linux/ethtool_netlink.h>
4#include <linux/netdevice.h>
5#include <net/netdev_lock.h>
6#include <net/netdev_queues.h>
7#include <net/netdev_rx_queue.h>
8#include <net/page_pool/memory_provider.h>
9
10#include "page_pool_priv.h"
11
12/* See also page_pool_is_unreadable() */
13bool netif_rxq_has_unreadable_mp(struct net_device *dev, int idx)
14{
15 struct netdev_rx_queue *rxq = __netif_get_rx_queue(dev, rxq: idx);
16
17 return !!rxq->mp_params.mp_ops;
18}
19EXPORT_SYMBOL(netif_rxq_has_unreadable_mp);
20
21int netdev_rx_queue_restart(struct net_device *dev, unsigned int rxq_idx)
22{
23 struct netdev_rx_queue *rxq = __netif_get_rx_queue(dev, rxq: rxq_idx);
24 const struct netdev_queue_mgmt_ops *qops = dev->queue_mgmt_ops;
25 void *new_mem, *old_mem;
26 int err;
27
28 if (!qops || !qops->ndo_queue_stop || !qops->ndo_queue_mem_free ||
29 !qops->ndo_queue_mem_alloc || !qops->ndo_queue_start)
30 return -EOPNOTSUPP;
31
32 netdev_assert_locked(dev);
33
34 new_mem = kvzalloc(qops->ndo_queue_mem_size, GFP_KERNEL);
35 if (!new_mem)
36 return -ENOMEM;
37
38 old_mem = kvzalloc(qops->ndo_queue_mem_size, GFP_KERNEL);
39 if (!old_mem) {
40 err = -ENOMEM;
41 goto err_free_new_mem;
42 }
43
44 err = qops->ndo_queue_mem_alloc(dev, new_mem, rxq_idx);
45 if (err)
46 goto err_free_old_mem;
47
48 err = page_pool_check_memory_provider(dev, rxq);
49 if (err)
50 goto err_free_new_queue_mem;
51
52 if (netif_running(dev)) {
53 err = qops->ndo_queue_stop(dev, old_mem, rxq_idx);
54 if (err)
55 goto err_free_new_queue_mem;
56
57 err = qops->ndo_queue_start(dev, new_mem, rxq_idx);
58 if (err)
59 goto err_start_queue;
60 } else {
61 swap(new_mem, old_mem);
62 }
63
64 qops->ndo_queue_mem_free(dev, old_mem);
65
66 kvfree(addr: old_mem);
67 kvfree(addr: new_mem);
68
69 return 0;
70
71err_start_queue:
72 /* Restarting the queue with old_mem should be successful as we haven't
73 * changed any of the queue configuration, and there is not much we can
74 * do to recover from a failure here.
75 *
76 * WARN if we fail to recover the old rx queue, and at least free
77 * old_mem so we don't also leak that.
78 */
79 if (qops->ndo_queue_start(dev, old_mem, rxq_idx)) {
80 WARN(1,
81 "Failed to restart old queue in error path. RX queue %d may be unhealthy.",
82 rxq_idx);
83 qops->ndo_queue_mem_free(dev, old_mem);
84 }
85
86err_free_new_queue_mem:
87 qops->ndo_queue_mem_free(dev, new_mem);
88
89err_free_old_mem:
90 kvfree(addr: old_mem);
91
92err_free_new_mem:
93 kvfree(addr: new_mem);
94
95 return err;
96}
97EXPORT_SYMBOL_NS_GPL(netdev_rx_queue_restart, "NETDEV_INTERNAL");
98
99int __net_mp_open_rxq(struct net_device *dev, unsigned int rxq_idx,
100 const struct pp_memory_provider_params *p,
101 struct netlink_ext_ack *extack)
102{
103 struct netdev_rx_queue *rxq;
104 int ret;
105
106 if (!netdev_need_ops_lock(dev))
107 return -EOPNOTSUPP;
108
109 if (rxq_idx >= dev->real_num_rx_queues) {
110 NL_SET_ERR_MSG(extack, "rx queue index out of range");
111 return -ERANGE;
112 }
113 rxq_idx = array_index_nospec(rxq_idx, dev->real_num_rx_queues);
114
115 if (dev->cfg->hds_config != ETHTOOL_TCP_DATA_SPLIT_ENABLED) {
116 NL_SET_ERR_MSG(extack, "tcp-data-split is disabled");
117 return -EINVAL;
118 }
119 if (dev->cfg->hds_thresh) {
120 NL_SET_ERR_MSG(extack, "hds-thresh is not zero");
121 return -EINVAL;
122 }
123 if (dev_xdp_prog_count(dev)) {
124 NL_SET_ERR_MSG(extack, "unable to custom memory provider to device with XDP program attached");
125 return -EEXIST;
126 }
127
128 rxq = __netif_get_rx_queue(dev, rxq: rxq_idx);
129 if (rxq->mp_params.mp_ops) {
130 NL_SET_ERR_MSG(extack, "designated queue already memory provider bound");
131 return -EEXIST;
132 }
133#ifdef CONFIG_XDP_SOCKETS
134 if (rxq->pool) {
135 NL_SET_ERR_MSG(extack, "designated queue already in use by AF_XDP");
136 return -EBUSY;
137 }
138#endif
139
140 rxq->mp_params = *p;
141 ret = netdev_rx_queue_restart(dev, rxq_idx);
142 if (ret) {
143 rxq->mp_params.mp_ops = NULL;
144 rxq->mp_params.mp_priv = NULL;
145 }
146 return ret;
147}
148
149int net_mp_open_rxq(struct net_device *dev, unsigned int rxq_idx,
150 struct pp_memory_provider_params *p)
151{
152 int ret;
153
154 netdev_lock(dev);
155 ret = __net_mp_open_rxq(dev, rxq_idx, p, NULL);
156 netdev_unlock(dev);
157 return ret;
158}
159
160void __net_mp_close_rxq(struct net_device *dev, unsigned int ifq_idx,
161 const struct pp_memory_provider_params *old_p)
162{
163 struct netdev_rx_queue *rxq;
164 int err;
165
166 if (WARN_ON_ONCE(ifq_idx >= dev->real_num_rx_queues))
167 return;
168
169 rxq = __netif_get_rx_queue(dev, rxq: ifq_idx);
170
171 /* Callers holding a netdev ref may get here after we already
172 * went thru shutdown via dev_memory_provider_uninstall().
173 */
174 if (dev->reg_state > NETREG_REGISTERED &&
175 !rxq->mp_params.mp_ops)
176 return;
177
178 if (WARN_ON_ONCE(rxq->mp_params.mp_ops != old_p->mp_ops ||
179 rxq->mp_params.mp_priv != old_p->mp_priv))
180 return;
181
182 rxq->mp_params.mp_ops = NULL;
183 rxq->mp_params.mp_priv = NULL;
184 err = netdev_rx_queue_restart(dev, ifq_idx);
185 WARN_ON(err && err != -ENETDOWN);
186}
187
188void net_mp_close_rxq(struct net_device *dev, unsigned ifq_idx,
189 struct pp_memory_provider_params *old_p)
190{
191 netdev_lock(dev);
192 __net_mp_close_rxq(dev, ifq_idx, old_p);
193 netdev_unlock(dev);
194}
195

source code of linux/net/core/netdev_rx_queue.c