1/*
2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33#include <linux/dim.h>
34#include <linux/debugfs.h>
35#include <linux/mlx5/fs.h>
36#include <net/netdev_lock.h>
37#include <net/switchdev.h>
38#include <net/pkt_cls.h>
39#include <net/act_api.h>
40#include <net/devlink.h>
41#include <net/ipv6_stubs.h>
42
43#include "eswitch.h"
44#include "en.h"
45#include "en/dim.h"
46#include "en_rep.h"
47#include "en/params.h"
48#include "en/txrx.h"
49#include "en_tc.h"
50#include "en/rep/tc.h"
51#include "en/rep/neigh.h"
52#include "en/rep/bridge.h"
53#include "en/devlink.h"
54#include "fs_core.h"
55#include "lib/mlx5.h"
56#include "lib/devcom.h"
57#include "lib/vxlan.h"
58#define CREATE_TRACE_POINTS
59#include "diag/en_rep_tracepoint.h"
60#include "diag/reporter_vnic.h"
61#include "en_accel/ipsec.h"
62#include "en/tc/int_port.h"
63#include "en/ptp.h"
64#include "en/fs_ethtool.h"
65
66#define MLX5E_REP_PARAMS_DEF_LOG_SQ_SIZE \
67 max(0x7, MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE)
68#define MLX5E_REP_PARAMS_DEF_NUM_CHANNELS 1
69#define MLX5E_REP_PARAMS_DEF_LOG_RQ_SIZE 0x8
70
71static const char mlx5e_rep_driver_name[] = "mlx5e_rep";
72
73static void mlx5e_rep_get_drvinfo(struct net_device *dev,
74 struct ethtool_drvinfo *drvinfo)
75{
76 struct mlx5e_priv *priv = netdev_priv(dev);
77 struct mlx5_core_dev *mdev = priv->mdev;
78 int count;
79
80 strscpy(drvinfo->driver, mlx5e_rep_driver_name,
81 sizeof(drvinfo->driver));
82 count = snprintf(buf: drvinfo->fw_version, size: sizeof(drvinfo->fw_version),
83 fmt: "%d.%d.%04d (%.16s)", fw_rev_maj(dev: mdev),
84 fw_rev_min(dev: mdev), fw_rev_sub(dev: mdev), mdev->board_id);
85 if (count >= sizeof(drvinfo->fw_version))
86 snprintf(buf: drvinfo->fw_version, size: sizeof(drvinfo->fw_version),
87 fmt: "%d.%d.%04d", fw_rev_maj(dev: mdev),
88 fw_rev_min(dev: mdev), fw_rev_sub(dev: mdev));
89}
90
91static const struct counter_desc sw_rep_stats_desc[] = {
92 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_packets) },
93 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_bytes) },
94 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_packets) },
95 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_bytes) },
96};
97
98static const struct counter_desc vport_rep_stats_desc[] = {
99 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, vport_rx_packets) },
100 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, vport_rx_bytes) },
101 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, vport_tx_packets) },
102 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, vport_tx_bytes) },
103 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats,
104 rx_vport_rdma_unicast_packets) },
105 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, rx_vport_rdma_unicast_bytes) },
106 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats,
107 tx_vport_rdma_unicast_packets) },
108 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, tx_vport_rdma_unicast_bytes) },
109 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats,
110 rx_vport_rdma_multicast_packets) },
111 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats,
112 rx_vport_rdma_multicast_bytes) },
113 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats,
114 tx_vport_rdma_multicast_packets) },
115 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats,
116 tx_vport_rdma_multicast_bytes) },
117};
118
119static const struct counter_desc vport_rep_loopback_stats_desc[] = {
120 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats,
121 vport_loopback_packets) },
122 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats,
123 vport_loopback_bytes) },
124};
125
126#define NUM_VPORT_REP_SW_COUNTERS ARRAY_SIZE(sw_rep_stats_desc)
127#define NUM_VPORT_REP_HW_COUNTERS ARRAY_SIZE(vport_rep_stats_desc)
128#define NUM_VPORT_REP_LOOPBACK_COUNTERS(dev) \
129 (MLX5_CAP_GEN(dev, vport_counter_local_loopback) ? \
130 ARRAY_SIZE(vport_rep_loopback_stats_desc) : 0)
131
132static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(sw_rep)
133{
134 return NUM_VPORT_REP_SW_COUNTERS;
135}
136
137static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(sw_rep)
138{
139 int i;
140
141 for (i = 0; i < NUM_VPORT_REP_SW_COUNTERS; i++)
142 ethtool_puts(data, str: sw_rep_stats_desc[i].format);
143}
144
145static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(sw_rep)
146{
147 int i;
148
149 for (i = 0; i < NUM_VPORT_REP_SW_COUNTERS; i++)
150 mlx5e_ethtool_put_stat(
151 data, MLX5E_READ_CTR64_CPU(&priv->stats.sw,
152 sw_rep_stats_desc, i));
153}
154
155static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(sw_rep)
156{
157 struct mlx5e_sw_stats *s = &priv->stats.sw;
158 struct rtnl_link_stats64 stats64 = {};
159
160 memset(s, 0, sizeof(*s));
161 mlx5e_fold_sw_stats64(priv, s: &stats64);
162
163 s->rx_packets = stats64.rx_packets;
164 s->rx_bytes = stats64.rx_bytes;
165 s->tx_packets = stats64.tx_packets;
166 s->tx_bytes = stats64.tx_bytes;
167 s->tx_queue_dropped = stats64.tx_dropped;
168}
169
170static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(vport_rep)
171{
172 return NUM_VPORT_REP_HW_COUNTERS +
173 NUM_VPORT_REP_LOOPBACK_COUNTERS(priv->mdev);
174}
175
176static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(vport_rep)
177{
178 int i;
179
180 for (i = 0; i < NUM_VPORT_REP_HW_COUNTERS; i++)
181 ethtool_puts(data, str: vport_rep_stats_desc[i].format);
182 for (i = 0; i < NUM_VPORT_REP_LOOPBACK_COUNTERS(priv->mdev); i++)
183 ethtool_puts(data, str: vport_rep_loopback_stats_desc[i].format);
184}
185
186static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(vport_rep)
187{
188 int i;
189
190 for (i = 0; i < NUM_VPORT_REP_HW_COUNTERS; i++)
191 mlx5e_ethtool_put_stat(
192 data, MLX5E_READ_CTR64_CPU(&priv->stats.rep_stats,
193 vport_rep_stats_desc, i));
194 for (i = 0; i < NUM_VPORT_REP_LOOPBACK_COUNTERS(priv->mdev); i++)
195 mlx5e_ethtool_put_stat(
196 data,
197 MLX5E_READ_CTR64_CPU(&priv->stats.rep_stats,
198 vport_rep_loopback_stats_desc, i));
199}
200
201static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(vport_rep)
202{
203 struct mlx5e_rep_stats *rep_stats = &priv->stats.rep_stats;
204 int outlen = MLX5_ST_SZ_BYTES(query_vport_counter_out);
205 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
206 struct mlx5e_rep_priv *rpriv = priv->ppriv;
207 struct mlx5_eswitch_rep *rep = rpriv->rep;
208 u32 *out;
209 int err;
210
211 out = kvzalloc(outlen, GFP_KERNEL);
212 if (!out)
213 return;
214
215 err = mlx5_core_query_vport_counter(dev: esw->dev, other_vport: 1, vf: rep->vport - 1, port_num: 0, out);
216 if (err) {
217 netdev_warn(dev: priv->netdev, format: "vport %d error %d reading stats\n",
218 rep->vport, err);
219 goto out;
220 }
221
222 #define MLX5_GET_CTR(p, x) \
223 MLX5_GET64(query_vport_counter_out, p, x)
224 /* flip tx/rx as we are reporting the counters for the switch vport */
225 rep_stats->vport_rx_packets =
226 MLX5_GET_CTR(out, transmitted_ib_unicast.packets) +
227 MLX5_GET_CTR(out, transmitted_eth_unicast.packets) +
228 MLX5_GET_CTR(out, transmitted_ib_multicast.packets) +
229 MLX5_GET_CTR(out, transmitted_eth_multicast.packets) +
230 MLX5_GET_CTR(out, transmitted_eth_broadcast.packets);
231
232 rep_stats->vport_tx_packets =
233 MLX5_GET_CTR(out, received_ib_unicast.packets) +
234 MLX5_GET_CTR(out, received_eth_unicast.packets) +
235 MLX5_GET_CTR(out, received_ib_multicast.packets) +
236 MLX5_GET_CTR(out, received_eth_multicast.packets) +
237 MLX5_GET_CTR(out, received_eth_broadcast.packets);
238
239 rep_stats->vport_rx_bytes =
240 MLX5_GET_CTR(out, transmitted_ib_unicast.octets) +
241 MLX5_GET_CTR(out, transmitted_eth_unicast.octets) +
242 MLX5_GET_CTR(out, transmitted_ib_multicast.octets) +
243 MLX5_GET_CTR(out, transmitted_eth_broadcast.octets);
244
245 rep_stats->vport_tx_bytes =
246 MLX5_GET_CTR(out, received_ib_unicast.octets) +
247 MLX5_GET_CTR(out, received_eth_unicast.octets) +
248 MLX5_GET_CTR(out, received_ib_multicast.octets) +
249 MLX5_GET_CTR(out, received_eth_multicast.octets) +
250 MLX5_GET_CTR(out, received_eth_broadcast.octets);
251
252 rep_stats->rx_vport_rdma_unicast_packets =
253 MLX5_GET_CTR(out, transmitted_ib_unicast.packets);
254 rep_stats->tx_vport_rdma_unicast_packets =
255 MLX5_GET_CTR(out, received_ib_unicast.packets);
256 rep_stats->rx_vport_rdma_unicast_bytes =
257 MLX5_GET_CTR(out, transmitted_ib_unicast.octets);
258 rep_stats->tx_vport_rdma_unicast_bytes =
259 MLX5_GET_CTR(out, received_ib_unicast.octets);
260 rep_stats->rx_vport_rdma_multicast_packets =
261 MLX5_GET_CTR(out, transmitted_ib_multicast.packets);
262 rep_stats->tx_vport_rdma_multicast_packets =
263 MLX5_GET_CTR(out, received_ib_multicast.packets);
264 rep_stats->rx_vport_rdma_multicast_bytes =
265 MLX5_GET_CTR(out, transmitted_ib_multicast.octets);
266 rep_stats->tx_vport_rdma_multicast_bytes =
267 MLX5_GET_CTR(out, received_ib_multicast.octets);
268
269 if (MLX5_CAP_GEN(priv->mdev, vport_counter_local_loopback)) {
270 rep_stats->vport_loopback_packets =
271 MLX5_GET_CTR(out, local_loopback.packets);
272 rep_stats->vport_loopback_bytes =
273 MLX5_GET_CTR(out, local_loopback.octets);
274 }
275
276out:
277 kvfree(addr: out);
278}
279
280static int mlx5e_rep_query_aggr_q_counter(struct mlx5_core_dev *dev, int vport, void *out)
281{
282 u32 in[MLX5_ST_SZ_DW(query_q_counter_in)] = {};
283
284 MLX5_SET(query_q_counter_in, in, opcode, MLX5_CMD_OP_QUERY_Q_COUNTER);
285 MLX5_SET(query_q_counter_in, in, other_vport, 1);
286 MLX5_SET(query_q_counter_in, in, vport_number, vport);
287 MLX5_SET(query_q_counter_in, in, aggregate, 1);
288
289 return mlx5_cmd_exec_inout(dev, query_q_counter, in, out);
290}
291
292static void mlx5e_rep_update_vport_q_counter(struct mlx5e_priv *priv)
293{
294 struct mlx5e_rep_stats *rep_stats = &priv->stats.rep_stats;
295 u32 out[MLX5_ST_SZ_DW(query_q_counter_out)] = {};
296 struct mlx5e_rep_priv *rpriv = priv->ppriv;
297 struct mlx5_eswitch_rep *rep = rpriv->rep;
298 int err;
299
300 if (!MLX5_CAP_GEN(priv->mdev, q_counter_other_vport) ||
301 !MLX5_CAP_GEN(priv->mdev, q_counter_aggregation))
302 return;
303
304 err = mlx5e_rep_query_aggr_q_counter(dev: priv->mdev, vport: rep->vport, out);
305 if (err) {
306 netdev_warn(dev: priv->netdev, format: "failed reading stats on vport %d, error %d\n",
307 rep->vport, err);
308 return;
309 }
310
311 rep_stats->rx_vport_out_of_buffer = MLX5_GET(query_q_counter_out, out, out_of_buffer);
312}
313
314static void mlx5e_rep_get_strings(struct net_device *dev,
315 u32 stringset, u8 *data)
316{
317 struct mlx5e_priv *priv = netdev_priv(dev);
318
319 switch (stringset) {
320 case ETH_SS_STATS:
321 mlx5e_stats_fill_strings(priv, data);
322 break;
323 }
324}
325
326static void mlx5e_rep_get_ethtool_stats(struct net_device *dev,
327 struct ethtool_stats *stats, u64 *data)
328{
329 struct mlx5e_priv *priv = netdev_priv(dev);
330
331 mlx5e_ethtool_get_ethtool_stats(priv, stats, data);
332}
333
334static int mlx5e_rep_get_sset_count(struct net_device *dev, int sset)
335{
336 struct mlx5e_priv *priv = netdev_priv(dev);
337
338 switch (sset) {
339 case ETH_SS_STATS:
340 return mlx5e_stats_total_num(priv);
341 default:
342 return -EOPNOTSUPP;
343 }
344}
345
346static void
347mlx5e_rep_get_ringparam(struct net_device *dev,
348 struct ethtool_ringparam *param,
349 struct kernel_ethtool_ringparam *kernel_param,
350 struct netlink_ext_ack *extack)
351{
352 struct mlx5e_priv *priv = netdev_priv(dev);
353
354 mlx5e_ethtool_get_ringparam(priv, param, kernel_param);
355}
356
357static int
358mlx5e_rep_set_ringparam(struct net_device *dev,
359 struct ethtool_ringparam *param,
360 struct kernel_ethtool_ringparam *kernel_param,
361 struct netlink_ext_ack *extack)
362{
363 struct mlx5e_priv *priv = netdev_priv(dev);
364
365 return mlx5e_ethtool_set_ringparam(priv, param, extack);
366}
367
368static void mlx5e_rep_get_channels(struct net_device *dev,
369 struct ethtool_channels *ch)
370{
371 struct mlx5e_priv *priv = netdev_priv(dev);
372
373 mlx5e_ethtool_get_channels(priv, ch);
374}
375
376static int mlx5e_rep_set_channels(struct net_device *dev,
377 struct ethtool_channels *ch)
378{
379 struct mlx5e_priv *priv = netdev_priv(dev);
380
381 return mlx5e_ethtool_set_channels(priv, ch);
382}
383
384static int mlx5e_rep_get_coalesce(struct net_device *netdev,
385 struct ethtool_coalesce *coal,
386 struct kernel_ethtool_coalesce *kernel_coal,
387 struct netlink_ext_ack *extack)
388{
389 struct mlx5e_priv *priv = netdev_priv(dev: netdev);
390
391 return mlx5e_ethtool_get_coalesce(priv, coal, kernel_coal, extack);
392}
393
394static int mlx5e_rep_set_coalesce(struct net_device *netdev,
395 struct ethtool_coalesce *coal,
396 struct kernel_ethtool_coalesce *kernel_coal,
397 struct netlink_ext_ack *extack)
398{
399 struct mlx5e_priv *priv = netdev_priv(dev: netdev);
400
401 return mlx5e_ethtool_set_coalesce(priv, coal, kernel_coal, extack);
402}
403
404static u32 mlx5e_rep_get_rxfh_key_size(struct net_device *netdev)
405{
406 struct mlx5e_priv *priv = netdev_priv(dev: netdev);
407
408 return mlx5e_ethtool_get_rxfh_key_size(priv);
409}
410
411static u32 mlx5e_rep_get_rxfh_indir_size(struct net_device *netdev)
412{
413 struct mlx5e_priv *priv = netdev_priv(dev: netdev);
414
415 return mlx5e_ethtool_get_rxfh_indir_size(priv);
416}
417
418static const struct ethtool_ops mlx5e_rep_ethtool_ops = {
419 .supported_coalesce_params = ETHTOOL_COALESCE_USECS |
420 ETHTOOL_COALESCE_MAX_FRAMES |
421 ETHTOOL_COALESCE_USE_ADAPTIVE,
422 .get_drvinfo = mlx5e_rep_get_drvinfo,
423 .get_link = ethtool_op_get_link,
424 .get_strings = mlx5e_rep_get_strings,
425 .get_sset_count = mlx5e_rep_get_sset_count,
426 .get_ethtool_stats = mlx5e_rep_get_ethtool_stats,
427 .get_ringparam = mlx5e_rep_get_ringparam,
428 .set_ringparam = mlx5e_rep_set_ringparam,
429 .get_channels = mlx5e_rep_get_channels,
430 .set_channels = mlx5e_rep_set_channels,
431 .get_coalesce = mlx5e_rep_get_coalesce,
432 .set_coalesce = mlx5e_rep_set_coalesce,
433 .get_per_queue_coalesce = mlx5e_get_per_queue_coalesce,
434 .set_per_queue_coalesce = mlx5e_set_per_queue_coalesce,
435 .get_rxfh_key_size = mlx5e_rep_get_rxfh_key_size,
436 .get_rxfh_indir_size = mlx5e_rep_get_rxfh_indir_size,
437};
438
439static void mlx5e_sqs2vport_stop(struct mlx5_eswitch *esw,
440 struct mlx5_eswitch_rep *rep)
441{
442 struct mlx5e_rep_sq *rep_sq, *tmp;
443 struct mlx5e_rep_sq_peer *sq_peer;
444 struct mlx5e_rep_priv *rpriv;
445 unsigned long i;
446
447 if (esw->mode != MLX5_ESWITCH_OFFLOADS)
448 return;
449
450 rpriv = mlx5e_rep_to_rep_priv(rep);
451 list_for_each_entry_safe(rep_sq, tmp, &rpriv->vport_sqs_list, list) {
452 mlx5_eswitch_del_send_to_vport_rule(rule: rep_sq->send_to_vport_rule);
453 xa_for_each(&rep_sq->sq_peer, i, sq_peer) {
454 if (sq_peer->rule)
455 mlx5_eswitch_del_send_to_vport_rule(rule: sq_peer->rule);
456
457 xa_erase(&rep_sq->sq_peer, index: i);
458 kfree(objp: sq_peer);
459 }
460
461 xa_destroy(&rep_sq->sq_peer);
462 list_del(entry: &rep_sq->list);
463 kfree(objp: rep_sq);
464 }
465}
466
467static int mlx5e_sqs2vport_add_peers_rules(struct mlx5_eswitch *esw, struct mlx5_eswitch_rep *rep,
468 struct mlx5e_rep_sq *rep_sq, int i)
469{
470 struct mlx5_flow_handle *flow_rule;
471 struct mlx5_devcom_comp_dev *tmp;
472 struct mlx5_eswitch *peer_esw;
473
474 mlx5_devcom_for_each_peer_entry(esw->devcom, peer_esw, tmp) {
475 u16 peer_rule_idx = MLX5_CAP_GEN(peer_esw->dev, vhca_id);
476 struct mlx5e_rep_sq_peer *sq_peer;
477 int err;
478
479 sq_peer = kzalloc(sizeof(*sq_peer), GFP_KERNEL);
480 if (!sq_peer)
481 return -ENOMEM;
482
483 flow_rule = mlx5_eswitch_add_send_to_vport_rule(on_esw: peer_esw, from_esw: esw,
484 rep, sqn: rep_sq->sqn);
485 if (IS_ERR(ptr: flow_rule)) {
486 kfree(objp: sq_peer);
487 return PTR_ERR(ptr: flow_rule);
488 }
489
490 sq_peer->rule = flow_rule;
491 sq_peer->peer = peer_esw;
492 err = xa_insert(xa: &rep_sq->sq_peer, index: peer_rule_idx, entry: sq_peer, GFP_KERNEL);
493 if (err) {
494 kfree(objp: sq_peer);
495 mlx5_eswitch_del_send_to_vport_rule(rule: flow_rule);
496 return err;
497 }
498 }
499
500 return 0;
501}
502
503static int mlx5e_sqs2vport_start(struct mlx5_eswitch *esw,
504 struct mlx5_eswitch_rep *rep,
505 u32 *sqns_array, int sqns_num)
506{
507 struct mlx5_flow_handle *flow_rule;
508 struct mlx5e_rep_priv *rpriv;
509 struct mlx5e_rep_sq *rep_sq;
510 bool devcom_locked = false;
511 int err;
512 int i;
513
514 if (esw->mode != MLX5_ESWITCH_OFFLOADS)
515 return 0;
516
517 rpriv = mlx5e_rep_to_rep_priv(rep);
518
519 if (mlx5_devcom_comp_is_ready(devcom: esw->devcom) &&
520 mlx5_devcom_for_each_peer_begin(devcom: esw->devcom))
521 devcom_locked = true;
522
523 for (i = 0; i < sqns_num; i++) {
524 rep_sq = kzalloc(sizeof(*rep_sq), GFP_KERNEL);
525 if (!rep_sq) {
526 err = -ENOMEM;
527 goto out_err;
528 }
529
530 /* Add re-inject rule to the PF/representor sqs */
531 flow_rule = mlx5_eswitch_add_send_to_vport_rule(on_esw: esw, from_esw: esw, rep,
532 sqn: sqns_array[i]);
533 if (IS_ERR(ptr: flow_rule)) {
534 err = PTR_ERR(ptr: flow_rule);
535 kfree(objp: rep_sq);
536 goto out_err;
537 }
538 rep_sq->send_to_vport_rule = flow_rule;
539 rep_sq->sqn = sqns_array[i];
540
541 xa_init(xa: &rep_sq->sq_peer);
542 if (devcom_locked) {
543 err = mlx5e_sqs2vport_add_peers_rules(esw, rep, rep_sq, i);
544 if (err) {
545 mlx5_eswitch_del_send_to_vport_rule(rule: rep_sq->send_to_vport_rule);
546 xa_destroy(&rep_sq->sq_peer);
547 kfree(objp: rep_sq);
548 goto out_err;
549 }
550 }
551
552 list_add(new: &rep_sq->list, head: &rpriv->vport_sqs_list);
553 }
554
555 if (devcom_locked)
556 mlx5_devcom_for_each_peer_end(devcom: esw->devcom);
557
558 return 0;
559
560out_err:
561 mlx5e_sqs2vport_stop(esw, rep);
562
563 if (devcom_locked)
564 mlx5_devcom_for_each_peer_end(devcom: esw->devcom);
565
566 return err;
567}
568
569static int
570mlx5e_add_sqs_fwd_rules(struct mlx5e_priv *priv)
571{
572 int sqs_per_channel = mlx5e_get_dcb_num_tc(params: &priv->channels.params);
573 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
574 bool is_uplink_rep = mlx5e_is_uplink_rep(priv);
575 struct mlx5e_rep_priv *rpriv = priv->ppriv;
576 struct mlx5_eswitch_rep *rep = rpriv->rep;
577 int n, tc, nch, num_sqs = 0;
578 struct mlx5e_channel *c;
579 int err = -ENOMEM;
580 bool ptp_sq;
581 u32 *sqs;
582
583 ptp_sq = !!(priv->channels.ptp &&
584 MLX5E_GET_PFLAG(&priv->channels.params, MLX5E_PFLAG_TX_PORT_TS));
585 nch = priv->channels.num + ptp_sq;
586 /* +2 for xdpsqs, they don't exist on the ptp channel but will not be
587 * counted for by num_sqs.
588 */
589 if (is_uplink_rep)
590 sqs_per_channel += 2;
591
592 sqs = kvcalloc(nch * sqs_per_channel, sizeof(*sqs), GFP_KERNEL);
593 if (!sqs)
594 goto out;
595
596 for (n = 0; n < priv->channels.num; n++) {
597 c = priv->channels.c[n];
598 for (tc = 0; tc < c->num_tc; tc++)
599 sqs[num_sqs++] = c->sq[tc].sqn;
600
601 if (is_uplink_rep) {
602 if (c->xdp)
603 sqs[num_sqs++] = c->rq_xdpsq.sqn;
604
605 if (c->xdpsq)
606 sqs[num_sqs++] = c->xdpsq->sqn;
607 }
608 }
609 if (ptp_sq) {
610 struct mlx5e_ptp *ptp_ch = priv->channels.ptp;
611
612 for (tc = 0; tc < ptp_ch->num_tc; tc++)
613 sqs[num_sqs++] = ptp_ch->ptpsq[tc].txqsq.sqn;
614 }
615
616 err = mlx5e_sqs2vport_start(esw, rep, sqns_array: sqs, sqns_num: num_sqs);
617 kvfree(addr: sqs);
618
619out:
620 if (err)
621 netdev_warn(dev: priv->netdev, format: "Failed to add SQs FWD rules %d\n", err);
622 return err;
623}
624
625static void
626mlx5e_remove_sqs_fwd_rules(struct mlx5e_priv *priv)
627{
628 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
629 struct mlx5e_rep_priv *rpriv = priv->ppriv;
630 struct mlx5_eswitch_rep *rep = rpriv->rep;
631
632 mlx5e_sqs2vport_stop(esw, rep);
633}
634
635static int
636mlx5e_rep_add_meta_tunnel_rule(struct mlx5e_priv *priv)
637{
638 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
639 struct mlx5e_rep_priv *rpriv = priv->ppriv;
640 struct mlx5_eswitch_rep *rep = rpriv->rep;
641 struct mlx5_flow_handle *flow_rule;
642 struct mlx5_flow_group *g;
643
644 g = esw->fdb_table.offloads.send_to_vport_meta_grp;
645 if (!g)
646 return 0;
647
648 flow_rule = mlx5_eswitch_add_send_to_vport_meta_rule(esw, vport_num: rep->vport);
649 if (IS_ERR(ptr: flow_rule))
650 return PTR_ERR(ptr: flow_rule);
651
652 rpriv->send_to_vport_meta_rule = flow_rule;
653
654 return 0;
655}
656
657static void
658mlx5e_rep_del_meta_tunnel_rule(struct mlx5e_priv *priv)
659{
660 struct mlx5e_rep_priv *rpriv = priv->ppriv;
661
662 if (rpriv->send_to_vport_meta_rule)
663 mlx5_eswitch_del_send_to_vport_meta_rule(rule: rpriv->send_to_vport_meta_rule);
664}
665
666void mlx5e_rep_activate_channels(struct mlx5e_priv *priv)
667{
668 mlx5e_add_sqs_fwd_rules(priv);
669 mlx5e_rep_add_meta_tunnel_rule(priv);
670}
671
672void mlx5e_rep_deactivate_channels(struct mlx5e_priv *priv)
673{
674 mlx5e_rep_del_meta_tunnel_rule(priv);
675 mlx5e_remove_sqs_fwd_rules(priv);
676}
677
678static int mlx5e_rep_open(struct net_device *dev)
679{
680 struct mlx5e_priv *priv = netdev_priv(dev);
681 struct mlx5e_rep_priv *rpriv = priv->ppriv;
682 struct mlx5_eswitch_rep *rep = rpriv->rep;
683 int err;
684
685 mutex_lock(&priv->state_lock);
686 err = mlx5e_open_locked(netdev: dev);
687 if (err)
688 goto unlock;
689
690 if (!mlx5_modify_vport_admin_state(mdev: priv->mdev,
691 opmod: MLX5_VPORT_STATE_OP_MOD_ESW_VPORT,
692 vport: rep->vport, other_vport: 1,
693 state: MLX5_VPORT_ADMIN_STATE_UP))
694 netif_carrier_on(dev);
695
696unlock:
697 mutex_unlock(lock: &priv->state_lock);
698 return err;
699}
700
701static int mlx5e_rep_close(struct net_device *dev)
702{
703 struct mlx5e_priv *priv = netdev_priv(dev);
704 struct mlx5e_rep_priv *rpriv = priv->ppriv;
705 struct mlx5_eswitch_rep *rep = rpriv->rep;
706 int ret;
707
708 mutex_lock(&priv->state_lock);
709 mlx5_modify_vport_admin_state(mdev: priv->mdev,
710 opmod: MLX5_VPORT_STATE_OP_MOD_ESW_VPORT,
711 vport: rep->vport, other_vport: 1,
712 state: MLX5_VPORT_ADMIN_STATE_DOWN);
713 ret = mlx5e_close_locked(netdev: dev);
714 mutex_unlock(lock: &priv->state_lock);
715 return ret;
716}
717
718bool mlx5e_is_uplink_rep(struct mlx5e_priv *priv)
719{
720 struct mlx5e_rep_priv *rpriv = priv->ppriv;
721 struct mlx5_eswitch_rep *rep;
722
723 if (!MLX5_ESWITCH_MANAGER(priv->mdev))
724 return false;
725
726 if (!rpriv) /* non vport rep mlx5e instances don't use this field */
727 return false;
728
729 rep = rpriv->rep;
730 return (rep->vport == MLX5_VPORT_UPLINK);
731}
732
733bool mlx5e_rep_has_offload_stats(const struct net_device *dev, int attr_id)
734{
735 switch (attr_id) {
736 case IFLA_OFFLOAD_XSTATS_CPU_HIT:
737 return true;
738 }
739
740 return false;
741}
742
743static int
744mlx5e_get_sw_stats64(const struct net_device *dev,
745 struct rtnl_link_stats64 *stats)
746{
747 struct mlx5e_priv *priv = netdev_priv(dev);
748
749 mlx5e_fold_sw_stats64(priv, s: stats);
750 return 0;
751}
752
753int mlx5e_rep_get_offload_stats(int attr_id, const struct net_device *dev,
754 void *sp)
755{
756 switch (attr_id) {
757 case IFLA_OFFLOAD_XSTATS_CPU_HIT:
758 return mlx5e_get_sw_stats64(dev, stats: sp);
759 }
760
761 return -EINVAL;
762}
763
764static void
765mlx5e_rep_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats)
766{
767 struct mlx5e_priv *priv = netdev_priv(dev);
768
769 /* update HW stats in background for next time */
770 mlx5e_queue_update_stats(priv);
771 mlx5e_stats_copy_rep_stats(vf_vport: stats, rep_stats: &priv->stats.rep_stats);
772}
773
774static int mlx5e_rep_change_mtu(struct net_device *netdev, int new_mtu)
775{
776 return mlx5e_change_mtu(netdev, new_mtu, NULL);
777}
778
779static int mlx5e_rep_change_carrier(struct net_device *dev, bool new_carrier)
780{
781 struct mlx5e_priv *priv = netdev_priv(dev);
782 struct mlx5e_rep_priv *rpriv = priv->ppriv;
783 struct mlx5_eswitch_rep *rep = rpriv->rep;
784 int err;
785
786 if (new_carrier) {
787 err = mlx5_modify_vport_admin_state(mdev: priv->mdev, opmod: MLX5_VPORT_STATE_OP_MOD_ESW_VPORT,
788 vport: rep->vport, other_vport: 1, state: MLX5_VPORT_ADMIN_STATE_UP);
789 if (err)
790 return err;
791 netif_carrier_on(dev);
792 } else {
793 err = mlx5_modify_vport_admin_state(mdev: priv->mdev, opmod: MLX5_VPORT_STATE_OP_MOD_ESW_VPORT,
794 vport: rep->vport, other_vport: 1, state: MLX5_VPORT_ADMIN_STATE_DOWN);
795 if (err)
796 return err;
797 netif_carrier_off(dev);
798 }
799 return 0;
800}
801
802static const struct net_device_ops mlx5e_netdev_ops_rep = {
803 .ndo_open = mlx5e_rep_open,
804 .ndo_stop = mlx5e_rep_close,
805 .ndo_start_xmit = mlx5e_xmit,
806 .ndo_setup_tc = mlx5e_rep_setup_tc,
807 .ndo_set_mac_address = eth_mac_addr,
808 .ndo_get_stats64 = mlx5e_rep_get_stats,
809 .ndo_has_offload_stats = mlx5e_rep_has_offload_stats,
810 .ndo_get_offload_stats = mlx5e_rep_get_offload_stats,
811 .ndo_change_mtu = mlx5e_rep_change_mtu,
812 .ndo_change_carrier = mlx5e_rep_change_carrier,
813};
814
815bool mlx5e_eswitch_uplink_rep(const struct net_device *netdev)
816{
817 return netdev->netdev_ops == &mlx5e_netdev_ops &&
818 mlx5e_is_uplink_rep(priv: netdev_priv(dev: netdev));
819}
820
821bool mlx5e_eswitch_vf_rep(const struct net_device *netdev)
822{
823 return netdev->netdev_ops == &mlx5e_netdev_ops_rep;
824}
825
826/* One indirect TIR set for outer. Inner not supported in reps. */
827#define REP_NUM_INDIR_TIRS MLX5E_NUM_INDIR_TIRS
828
829static int mlx5e_rep_max_nch_limit(struct mlx5_core_dev *mdev)
830{
831 int max_tir_num = 1 << MLX5_CAP_GEN(mdev, log_max_tir);
832 int num_vports = mlx5_eswitch_get_total_vports(dev: mdev);
833
834 return (max_tir_num - mlx5e_get_pf_num_tirs(mdev)
835 - (num_vports * REP_NUM_INDIR_TIRS)) / num_vports;
836}
837
838static void mlx5e_build_rep_params(struct net_device *netdev)
839{
840 const bool take_rtnl = netdev->reg_state == NETREG_REGISTERED;
841 struct mlx5e_priv *priv = netdev_priv(dev: netdev);
842 struct mlx5e_rep_priv *rpriv = priv->ppriv;
843 struct mlx5_eswitch_rep *rep = rpriv->rep;
844 struct mlx5_core_dev *mdev = priv->mdev;
845 struct mlx5e_params *params;
846
847 params = &priv->channels.params;
848
849 params->num_channels = MLX5E_REP_PARAMS_DEF_NUM_CHANNELS;
850 params->hard_mtu = MLX5E_ETH_HARD_MTU;
851 params->sw_mtu = netdev->mtu;
852
853 /* SQ */
854 if (rep->vport == MLX5_VPORT_UPLINK)
855 params->log_sq_size = MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE;
856 else
857 params->log_sq_size = MLX5E_REP_PARAMS_DEF_LOG_SQ_SIZE;
858
859 /* RQ */
860 mlx5e_build_rq_params(mdev, params);
861 if (!mlx5e_is_uplink_rep(priv) && mlx5_core_is_ecpf(dev: mdev))
862 params->log_rq_mtu_frames = MLX5E_REP_PARAMS_DEF_LOG_RQ_SIZE;
863
864 /* If netdev is already registered (e.g. move from nic profile to uplink,
865 * RTNL lock must be held before triggering netdev notifiers.
866 */
867 if (take_rtnl)
868 rtnl_lock();
869 /* update XDP supported features */
870 mlx5e_set_xdp_feature(priv);
871 if (take_rtnl)
872 rtnl_unlock();
873
874 /* CQ moderation params */
875 params->rx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
876 params->rx_moder_use_cqe_mode = !!MLX5_CAP_GEN(mdev, cq_period_start_from_cqe);
877
878 params->mqprio.num_tc = 1;
879 if (rep->vport != MLX5_VPORT_UPLINK)
880 params->vlan_strip_disable = true;
881
882 mlx5_query_min_inline(mdev, min_inline: &params->tx_min_inline_mode);
883}
884
885static void mlx5e_build_rep_netdev(struct net_device *netdev,
886 struct mlx5_core_dev *mdev)
887{
888 SET_NETDEV_DEV(netdev, mdev->device);
889 netdev->netdev_ops = &mlx5e_netdev_ops_rep;
890 netdev->request_ops_lock = true;
891 netdev_lockdep_set_classes(netdev);
892 eth_hw_addr_random(dev: netdev);
893 netdev->ethtool_ops = &mlx5e_rep_ethtool_ops;
894
895 netdev->watchdog_timeo = 15 * HZ;
896 if (mlx5_core_is_ecpf(dev: mdev))
897 netdev->tx_queue_len = 1 << MLX5E_REP_PARAMS_DEF_LOG_SQ_SIZE;
898
899#if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
900 netdev->hw_features |= NETIF_F_HW_TC;
901#endif
902 netdev->hw_features |= NETIF_F_SG;
903 netdev->hw_features |= NETIF_F_IP_CSUM;
904 netdev->hw_features |= NETIF_F_IPV6_CSUM;
905 netdev->hw_features |= NETIF_F_GRO;
906 netdev->hw_features |= NETIF_F_TSO;
907 netdev->hw_features |= NETIF_F_TSO6;
908 netdev->hw_features |= NETIF_F_RXCSUM;
909
910 netdev->features |= netdev->hw_features;
911
912 netdev->netns_immutable = true;
913}
914
915static int mlx5e_init_rep(struct mlx5_core_dev *mdev,
916 struct net_device *netdev)
917{
918 struct mlx5e_priv *priv = netdev_priv(dev: netdev);
919
920 priv->fs =
921 mlx5e_fs_init(profile: priv->profile, mdev,
922 state_destroy: !test_bit(MLX5E_STATE_DESTROYING, &priv->state),
923 dfs_root: priv->dfs_root);
924 if (!priv->fs) {
925 netdev_err(dev: priv->netdev, format: "FS allocation failed\n");
926 return -ENOMEM;
927 }
928
929 mlx5e_build_rep_params(netdev);
930 mlx5e_timestamp_init(priv);
931
932 return 0;
933}
934
935static int mlx5e_init_ul_rep(struct mlx5_core_dev *mdev,
936 struct net_device *netdev)
937{
938 struct mlx5e_priv *priv = netdev_priv(dev: netdev);
939
940 priv->dfs_root = debugfs_create_dir(name: "nic",
941 parent: mlx5_debugfs_get_dev_root(dev: mdev));
942
943 priv->fs = mlx5e_fs_init(profile: priv->profile, mdev,
944 state_destroy: !test_bit(MLX5E_STATE_DESTROYING, &priv->state),
945 dfs_root: priv->dfs_root);
946 if (!priv->fs) {
947 netdev_err(dev: priv->netdev, format: "FS allocation failed\n");
948 debugfs_remove_recursive(dentry: priv->dfs_root);
949 return -ENOMEM;
950 }
951
952 mlx5e_vxlan_set_netdev_info(priv);
953 mlx5e_build_rep_params(netdev);
954 mlx5e_timestamp_init(priv);
955 return 0;
956}
957
958static void mlx5e_cleanup_rep(struct mlx5e_priv *priv)
959{
960 mlx5e_fs_cleanup(fs: priv->fs);
961 debugfs_remove_recursive(dentry: priv->dfs_root);
962 priv->fs = NULL;
963}
964
965static int mlx5e_create_rep_ttc_table(struct mlx5e_priv *priv)
966{
967 struct mlx5e_rep_priv *rpriv = priv->ppriv;
968 struct mlx5_eswitch_rep *rep = rpriv->rep;
969 struct ttc_params ttc_params = {};
970 int err;
971
972 mlx5e_fs_set_ns(fs: priv->fs,
973 ns: mlx5_get_flow_namespace(dev: priv->mdev,
974 type: MLX5_FLOW_NAMESPACE_KERNEL), egress: false);
975
976 /* The inner_ttc in the ttc params is intentionally not set */
977 mlx5e_set_ttc_params(fs: priv->fs, rx_res: priv->rx_res, ttc_params: &ttc_params, tunnel: false, ipsec_rss: false);
978
979 if (rep->vport != MLX5_VPORT_UPLINK)
980 /* To give uplik rep TTC a lower level for chaining from root ft */
981 ttc_params.ft_attr.level = MLX5E_TTC_FT_LEVEL + 1;
982
983 mlx5e_fs_set_ttc(fs: priv->fs, ttc: mlx5_create_ttc_table(dev: priv->mdev, params: &ttc_params), inner: false);
984 if (IS_ERR(ptr: mlx5e_fs_get_ttc(fs: priv->fs, inner: false))) {
985 err = PTR_ERR(ptr: mlx5e_fs_get_ttc(fs: priv->fs, inner: false));
986 netdev_err(dev: priv->netdev, format: "Failed to create rep ttc table, err=%d\n",
987 err);
988 return err;
989 }
990 return 0;
991}
992
993static int mlx5e_create_rep_root_ft(struct mlx5e_priv *priv)
994{
995 struct mlx5e_rep_priv *rpriv = priv->ppriv;
996 struct mlx5_eswitch_rep *rep = rpriv->rep;
997 struct mlx5_flow_table_attr ft_attr = {};
998 struct mlx5_flow_namespace *ns;
999 int err = 0;
1000
1001 if (rep->vport != MLX5_VPORT_UPLINK) {
1002 /* non uplik reps will skip any bypass tables and go directly to
1003 * their own ttc
1004 */
1005 rpriv->root_ft = mlx5_get_ttc_flow_table(ttc: mlx5e_fs_get_ttc(fs: priv->fs, inner: false));
1006 return 0;
1007 }
1008
1009 /* uplink root ft will be used to auto chain, to ethtool or ttc tables */
1010 ns = mlx5_get_flow_namespace(dev: priv->mdev, type: MLX5_FLOW_NAMESPACE_OFFLOADS);
1011 if (!ns) {
1012 netdev_err(dev: priv->netdev, format: "Failed to get reps offloads namespace\n");
1013 return -EOPNOTSUPP;
1014 }
1015
1016 ft_attr.max_fte = 0; /* Empty table, miss rule will always point to next table */
1017 ft_attr.prio = 1;
1018 ft_attr.level = 1;
1019
1020 rpriv->root_ft = mlx5_create_flow_table(ns, ft_attr: &ft_attr);
1021 if (IS_ERR(ptr: rpriv->root_ft)) {
1022 err = PTR_ERR(ptr: rpriv->root_ft);
1023 rpriv->root_ft = NULL;
1024 }
1025
1026 return err;
1027}
1028
1029static void mlx5e_destroy_rep_root_ft(struct mlx5e_priv *priv)
1030{
1031 struct mlx5e_rep_priv *rpriv = priv->ppriv;
1032 struct mlx5_eswitch_rep *rep = rpriv->rep;
1033
1034 if (rep->vport != MLX5_VPORT_UPLINK)
1035 return;
1036 mlx5_destroy_flow_table(ft: rpriv->root_ft);
1037}
1038
1039static int mlx5e_create_rep_vport_rx_rule(struct mlx5e_priv *priv)
1040{
1041 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
1042 struct mlx5e_rep_priv *rpriv = priv->ppriv;
1043 struct mlx5_eswitch_rep *rep = rpriv->rep;
1044 struct mlx5_flow_handle *flow_rule;
1045 struct mlx5_flow_destination dest;
1046
1047 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
1048 dest.ft = rpriv->root_ft;
1049
1050 flow_rule = mlx5_eswitch_create_vport_rx_rule(esw, vport: rep->vport, dest: &dest);
1051 if (IS_ERR(ptr: flow_rule))
1052 return PTR_ERR(ptr: flow_rule);
1053 rpriv->vport_rx_rule = flow_rule;
1054 return 0;
1055}
1056
1057static void rep_vport_rx_rule_destroy(struct mlx5e_priv *priv)
1058{
1059 struct mlx5e_rep_priv *rpriv = priv->ppriv;
1060
1061 if (!rpriv->vport_rx_rule)
1062 return;
1063
1064 mlx5_del_flow_rules(fr: rpriv->vport_rx_rule);
1065 rpriv->vport_rx_rule = NULL;
1066}
1067
1068int mlx5e_rep_bond_update(struct mlx5e_priv *priv, bool cleanup)
1069{
1070 rep_vport_rx_rule_destroy(priv);
1071
1072 return cleanup ? 0 : mlx5e_create_rep_vport_rx_rule(priv);
1073}
1074
1075static int mlx5e_init_rep_rx(struct mlx5e_priv *priv)
1076{
1077 struct mlx5_core_dev *mdev = priv->mdev;
1078 int err;
1079
1080 mlx5e_fs_init_l2_addr(fs: priv->fs, netdev: priv->netdev);
1081
1082 err = mlx5e_open_drop_rq(priv, drop_rq: &priv->drop_rq);
1083 if (err) {
1084 mlx5_core_err(mdev, "open drop rq failed, %d\n", err);
1085 goto err_free_fs;
1086 }
1087
1088 priv->rx_res = mlx5e_rx_res_create(mdev: priv->mdev, features: 0, max_nch: priv->max_nch, drop_rqn: priv->drop_rq.rqn,
1089 pkt_merge_param: &priv->channels.params.packet_merge,
1090 init_nch: priv->channels.params.num_channels);
1091 if (IS_ERR(ptr: priv->rx_res)) {
1092 err = PTR_ERR(ptr: priv->rx_res);
1093 mlx5_core_err(mdev, "Create rx resources failed, err=%d\n", err);
1094 goto err_close_drop_rq;
1095 }
1096
1097 err = mlx5e_create_rep_ttc_table(priv);
1098 if (err)
1099 goto err_destroy_rx_res;
1100
1101 err = mlx5e_create_rep_root_ft(priv);
1102 if (err)
1103 goto err_destroy_ttc_table;
1104
1105 err = mlx5e_create_rep_vport_rx_rule(priv);
1106 if (err)
1107 goto err_destroy_root_ft;
1108
1109 mlx5e_ethtool_init_steering(fs: priv->fs);
1110
1111 return 0;
1112
1113err_destroy_root_ft:
1114 mlx5e_destroy_rep_root_ft(priv);
1115err_destroy_ttc_table:
1116 mlx5_destroy_ttc_table(ttc: mlx5e_fs_get_ttc(fs: priv->fs, inner: false));
1117err_destroy_rx_res:
1118 mlx5e_rx_res_destroy(res: priv->rx_res);
1119 priv->rx_res = ERR_PTR(error: -EINVAL);
1120err_close_drop_rq:
1121 mlx5e_close_drop_rq(drop_rq: &priv->drop_rq);
1122err_free_fs:
1123 mlx5e_fs_cleanup(fs: priv->fs);
1124 priv->fs = NULL;
1125 return err;
1126}
1127
1128static void mlx5e_cleanup_rep_rx(struct mlx5e_priv *priv)
1129{
1130 mlx5e_ethtool_cleanup_steering(fs: priv->fs);
1131 rep_vport_rx_rule_destroy(priv);
1132 mlx5e_destroy_rep_root_ft(priv);
1133 mlx5_destroy_ttc_table(ttc: mlx5e_fs_get_ttc(fs: priv->fs, inner: false));
1134 mlx5e_rx_res_destroy(res: priv->rx_res);
1135 priv->rx_res = ERR_PTR(error: -EINVAL);
1136 mlx5e_close_drop_rq(drop_rq: &priv->drop_rq);
1137}
1138
1139static void mlx5e_rep_mpesw_work(struct work_struct *work)
1140{
1141 struct mlx5_rep_uplink_priv *uplink_priv =
1142 container_of(work, struct mlx5_rep_uplink_priv,
1143 mpesw_work);
1144 struct mlx5e_rep_priv *rpriv =
1145 container_of(uplink_priv, struct mlx5e_rep_priv,
1146 uplink_priv);
1147 struct mlx5e_priv *priv = netdev_priv(dev: rpriv->netdev);
1148
1149 rep_vport_rx_rule_destroy(priv);
1150 mlx5e_create_rep_vport_rx_rule(priv);
1151}
1152
1153static int mlx5e_init_ul_rep_rx(struct mlx5e_priv *priv)
1154{
1155 struct mlx5e_rep_priv *rpriv = priv->ppriv;
1156 int err;
1157
1158 mlx5e_create_q_counters(priv);
1159 err = mlx5e_init_rep_rx(priv);
1160 if (err)
1161 goto out;
1162
1163 mlx5e_tc_int_port_init_rep_rx(priv);
1164
1165 INIT_WORK(&rpriv->uplink_priv.mpesw_work, mlx5e_rep_mpesw_work);
1166
1167out:
1168 return err;
1169}
1170
1171static void mlx5e_cleanup_ul_rep_rx(struct mlx5e_priv *priv)
1172{
1173 struct mlx5e_rep_priv *rpriv = priv->ppriv;
1174
1175 cancel_work_sync(work: &rpriv->uplink_priv.mpesw_work);
1176 mlx5e_tc_int_port_cleanup_rep_rx(priv);
1177 mlx5e_cleanup_rep_rx(priv);
1178 mlx5e_destroy_q_counters(priv);
1179}
1180
1181static int mlx5e_init_uplink_rep_tx(struct mlx5e_rep_priv *rpriv)
1182{
1183 struct mlx5_rep_uplink_priv *uplink_priv;
1184 struct net_device *netdev;
1185 struct mlx5e_priv *priv;
1186 int err;
1187
1188 netdev = rpriv->netdev;
1189 priv = netdev_priv(dev: netdev);
1190 uplink_priv = &rpriv->uplink_priv;
1191
1192 err = mlx5e_rep_tc_init(rpriv);
1193 if (err)
1194 return err;
1195
1196 mlx5_init_port_tun_entropy(tun_entropy: &uplink_priv->tun_entropy, mdev: priv->mdev);
1197
1198 mlx5e_rep_bond_init(rpriv);
1199 err = mlx5e_rep_tc_netdevice_event_register(rpriv);
1200 if (err) {
1201 mlx5_core_err(priv->mdev, "Failed to register netdev notifier, err: %d\n",
1202 err);
1203 goto err_event_reg;
1204 }
1205
1206 return 0;
1207
1208err_event_reg:
1209 mlx5e_rep_bond_cleanup(rpriv);
1210 mlx5e_rep_tc_cleanup(rpriv);
1211 return err;
1212}
1213
1214static void mlx5e_cleanup_uplink_rep_tx(struct mlx5e_rep_priv *rpriv)
1215{
1216 mlx5e_rep_tc_netdevice_event_unregister(rpriv);
1217 mlx5e_rep_bond_cleanup(rpriv);
1218 mlx5e_rep_tc_cleanup(rpriv);
1219}
1220
1221static int mlx5e_init_rep_tx(struct mlx5e_priv *priv)
1222{
1223 struct mlx5e_rep_priv *rpriv = priv->ppriv;
1224 int err;
1225
1226 err = mlx5e_rep_neigh_init(rpriv);
1227 if (err)
1228 goto err_neigh_init;
1229
1230 if (rpriv->rep->vport == MLX5_VPORT_UPLINK) {
1231 err = mlx5e_init_uplink_rep_tx(rpriv);
1232 if (err)
1233 goto err_init_tx;
1234 }
1235
1236 err = mlx5e_tc_ht_init(tc_ht: &rpriv->tc_ht);
1237 if (err)
1238 goto err_ht_init;
1239
1240 return 0;
1241
1242err_ht_init:
1243 if (rpriv->rep->vport == MLX5_VPORT_UPLINK)
1244 mlx5e_cleanup_uplink_rep_tx(rpriv);
1245err_init_tx:
1246 mlx5e_rep_neigh_cleanup(rpriv);
1247err_neigh_init:
1248 return err;
1249}
1250
1251static void mlx5e_cleanup_rep_tx(struct mlx5e_priv *priv)
1252{
1253 struct mlx5e_rep_priv *rpriv = priv->ppriv;
1254
1255 mlx5e_tc_ht_cleanup(tc_ht: &rpriv->tc_ht);
1256
1257 if (rpriv->rep->vport == MLX5_VPORT_UPLINK)
1258 mlx5e_cleanup_uplink_rep_tx(rpriv);
1259
1260 mlx5e_rep_neigh_cleanup(rpriv);
1261}
1262
1263static void mlx5e_rep_enable(struct mlx5e_priv *priv)
1264{
1265 mlx5e_set_netdev_mtu_boundaries(priv);
1266}
1267
1268static void mlx5e_rep_disable(struct mlx5e_priv *priv)
1269{
1270}
1271
1272static int mlx5e_update_rep_rx(struct mlx5e_priv *priv)
1273{
1274 return 0;
1275}
1276
1277static void mlx5e_rep_stats_update_ndo_stats(struct mlx5e_priv *priv)
1278{
1279 mlx5e_stats_update_ndo_stats(priv);
1280 mlx5e_rep_update_vport_q_counter(priv);
1281}
1282
1283static int mlx5e_rep_event_mpesw(struct mlx5e_priv *priv)
1284{
1285 struct mlx5e_rep_priv *rpriv = priv->ppriv;
1286 struct mlx5_eswitch_rep *rep = rpriv->rep;
1287
1288 if (rep->vport != MLX5_VPORT_UPLINK)
1289 return NOTIFY_DONE;
1290
1291 queue_work(wq: priv->wq, work: &rpriv->uplink_priv.mpesw_work);
1292
1293 return NOTIFY_OK;
1294}
1295
1296static int uplink_rep_async_event(struct notifier_block *nb, unsigned long event, void *data)
1297{
1298 struct mlx5e_priv *priv = container_of(nb, struct mlx5e_priv, events_nb);
1299
1300 if (event == MLX5_EVENT_TYPE_PORT_CHANGE) {
1301 struct mlx5_eqe *eqe = data;
1302
1303 switch (eqe->sub_type) {
1304 case MLX5_PORT_CHANGE_SUBTYPE_DOWN:
1305 case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE:
1306 queue_work(wq: priv->wq, work: &priv->update_carrier_work);
1307 break;
1308 default:
1309 return NOTIFY_DONE;
1310 }
1311
1312 return NOTIFY_OK;
1313 }
1314
1315 if (event == MLX5_DEV_EVENT_PORT_AFFINITY)
1316 return mlx5e_rep_tc_event_port_affinity(priv);
1317 else if (event == MLX5_DEV_EVENT_MULTIPORT_ESW)
1318 return mlx5e_rep_event_mpesw(priv);
1319
1320 return NOTIFY_DONE;
1321}
1322
1323static void mlx5e_uplink_rep_enable(struct mlx5e_priv *priv)
1324{
1325 struct net_device *netdev = priv->netdev;
1326 struct mlx5_core_dev *mdev = priv->mdev;
1327 u16 max_mtu;
1328
1329 mlx5e_ipsec_init(priv);
1330
1331 netdev->min_mtu = ETH_MIN_MTU;
1332 mlx5_query_port_max_mtu(dev: priv->mdev, max_mtu: &max_mtu, port: 1);
1333 netdev->max_mtu = MLX5E_HW2SW_MTU(&priv->channels.params, max_mtu);
1334 mlx5e_set_dev_port_mtu(priv);
1335
1336 mlx5e_rep_tc_enable(priv);
1337
1338 if (MLX5_CAP_GEN(mdev, uplink_follow))
1339 mlx5_modify_vport_admin_state(mdev, opmod: MLX5_VPORT_STATE_OP_MOD_UPLINK,
1340 vport: 0, other_vport: 0, state: MLX5_VPORT_ADMIN_STATE_AUTO);
1341 mlx5_lag_add_netdev(dev: mdev, netdev);
1342 priv->events_nb.notifier_call = uplink_rep_async_event;
1343 mlx5_notifier_register(dev: mdev, nb: &priv->events_nb);
1344 mlx5e_dcbnl_initialize(priv);
1345 mlx5e_dcbnl_init_app(priv);
1346 mlx5e_rep_bridge_init(priv);
1347
1348 netdev->wanted_features |= NETIF_F_HW_TC;
1349
1350 rtnl_lock();
1351 netdev_lock(dev: netdev);
1352 if (netif_running(dev: netdev))
1353 mlx5e_open(netdev);
1354 udp_tunnel_nic_reset_ntf(dev: priv->netdev);
1355 netdev_unlock(dev: netdev);
1356 netif_device_attach(dev: netdev);
1357 rtnl_unlock();
1358}
1359
1360static void mlx5e_uplink_rep_disable(struct mlx5e_priv *priv)
1361{
1362 struct mlx5_core_dev *mdev = priv->mdev;
1363
1364 rtnl_lock();
1365 netdev_lock(dev: priv->netdev);
1366 if (netif_running(dev: priv->netdev))
1367 mlx5e_close(netdev: priv->netdev);
1368 netif_device_detach(dev: priv->netdev);
1369 netdev_unlock(dev: priv->netdev);
1370 rtnl_unlock();
1371
1372 mlx5e_rep_bridge_cleanup(priv);
1373 mlx5e_dcbnl_delete_app(priv);
1374 mlx5_notifier_unregister(dev: mdev, nb: &priv->events_nb);
1375 mlx5e_rep_tc_disable(priv);
1376 mlx5_lag_remove_netdev(dev: mdev, netdev: priv->netdev);
1377 mlx5_vxlan_reset_to_default(vxlan: mdev->vxlan);
1378
1379 mlx5e_ipsec_cleanup(priv);
1380}
1381
1382static MLX5E_DEFINE_STATS_GRP(sw_rep, 0);
1383static MLX5E_DEFINE_STATS_GRP(vport_rep, MLX5E_NDO_UPDATE_STATS);
1384
1385/* The stats groups order is opposite to the update_stats() order calls */
1386static mlx5e_stats_grp_t mlx5e_rep_stats_grps[] = {
1387 &MLX5E_STATS_GRP(sw_rep),
1388 &MLX5E_STATS_GRP(vport_rep),
1389};
1390
1391static unsigned int mlx5e_rep_stats_grps_num(struct mlx5e_priv *priv)
1392{
1393 return ARRAY_SIZE(mlx5e_rep_stats_grps);
1394}
1395
1396/* The stats groups order is opposite to the update_stats() order calls */
1397static mlx5e_stats_grp_t mlx5e_ul_rep_stats_grps[] = {
1398 &MLX5E_STATS_GRP(sw),
1399 &MLX5E_STATS_GRP(qcnt),
1400 &MLX5E_STATS_GRP(vnic_env),
1401 &MLX5E_STATS_GRP(vport),
1402 &MLX5E_STATS_GRP(802_3),
1403 &MLX5E_STATS_GRP(2863),
1404 &MLX5E_STATS_GRP(2819),
1405 &MLX5E_STATS_GRP(phy),
1406 &MLX5E_STATS_GRP(eth_ext),
1407 &MLX5E_STATS_GRP(pcie),
1408 &MLX5E_STATS_GRP(per_prio),
1409 &MLX5E_STATS_GRP(pme),
1410 &MLX5E_STATS_GRP(channels),
1411 &MLX5E_STATS_GRP(per_port_buff_congest),
1412#ifdef CONFIG_MLX5_EN_IPSEC
1413 &MLX5E_STATS_GRP(ipsec_hw),
1414 &MLX5E_STATS_GRP(ipsec_sw),
1415#endif
1416 &MLX5E_STATS_GRP(ptp),
1417};
1418
1419static unsigned int mlx5e_ul_rep_stats_grps_num(struct mlx5e_priv *priv)
1420{
1421 return ARRAY_SIZE(mlx5e_ul_rep_stats_grps);
1422}
1423
1424static int
1425mlx5e_rep_vnic_reporter_diagnose(struct devlink_health_reporter *reporter,
1426 struct devlink_fmsg *fmsg,
1427 struct netlink_ext_ack *extack)
1428{
1429 struct mlx5e_rep_priv *rpriv = devlink_health_reporter_priv(reporter);
1430 struct mlx5_eswitch_rep *rep = rpriv->rep;
1431
1432 mlx5_reporter_vnic_diagnose_counters(dev: rep->esw->dev, fmsg, vport_num: rep->vport,
1433 other_vport: true);
1434 return 0;
1435}
1436
1437static const struct devlink_health_reporter_ops mlx5_rep_vnic_reporter_ops = {
1438 .name = "vnic",
1439 .diagnose = mlx5e_rep_vnic_reporter_diagnose,
1440};
1441
1442static void mlx5e_rep_vnic_reporter_create(struct mlx5e_priv *priv,
1443 struct devlink_port *dl_port)
1444{
1445 struct mlx5e_rep_priv *rpriv = priv->ppriv;
1446 struct devlink_health_reporter *reporter;
1447
1448 reporter = devl_port_health_reporter_create(port: dl_port,
1449 ops: &mlx5_rep_vnic_reporter_ops,
1450 priv: rpriv);
1451 if (IS_ERR(ptr: reporter)) {
1452 mlx5_core_err(priv->mdev,
1453 "Failed to create representor vnic reporter, err = %pe\n",
1454 reporter);
1455 return;
1456 }
1457
1458 rpriv->rep_vnic_reporter = reporter;
1459}
1460
1461static void mlx5e_rep_vnic_reporter_destroy(struct mlx5e_priv *priv)
1462{
1463 struct mlx5e_rep_priv *rpriv = priv->ppriv;
1464
1465 if (!IS_ERR_OR_NULL(ptr: rpriv->rep_vnic_reporter))
1466 devl_health_reporter_destroy(reporter: rpriv->rep_vnic_reporter);
1467}
1468
1469static const struct mlx5e_profile mlx5e_rep_profile = {
1470 .init = mlx5e_init_rep,
1471 .cleanup = mlx5e_cleanup_rep,
1472 .init_rx = mlx5e_init_rep_rx,
1473 .cleanup_rx = mlx5e_cleanup_rep_rx,
1474 .init_tx = mlx5e_init_rep_tx,
1475 .cleanup_tx = mlx5e_cleanup_rep_tx,
1476 .enable = mlx5e_rep_enable,
1477 .disable = mlx5e_rep_disable,
1478 .update_rx = mlx5e_update_rep_rx,
1479 .update_stats = mlx5e_rep_stats_update_ndo_stats,
1480 .rx_handlers = &mlx5e_rx_handlers_rep,
1481 .max_tc = 1,
1482 .stats_grps = mlx5e_rep_stats_grps,
1483 .stats_grps_num = mlx5e_rep_stats_grps_num,
1484 .max_nch_limit = mlx5e_rep_max_nch_limit,
1485};
1486
1487static const struct mlx5e_profile mlx5e_uplink_rep_profile = {
1488 .init = mlx5e_init_ul_rep,
1489 .cleanup = mlx5e_cleanup_rep,
1490 .init_rx = mlx5e_init_ul_rep_rx,
1491 .cleanup_rx = mlx5e_cleanup_ul_rep_rx,
1492 .init_tx = mlx5e_init_rep_tx,
1493 .cleanup_tx = mlx5e_cleanup_rep_tx,
1494 .enable = mlx5e_uplink_rep_enable,
1495 .disable = mlx5e_uplink_rep_disable,
1496 .update_rx = mlx5e_update_rep_rx,
1497 .update_stats = mlx5e_stats_update_ndo_stats,
1498 .update_carrier = mlx5e_update_carrier,
1499 .rx_handlers = &mlx5e_rx_handlers_rep,
1500 .max_tc = MLX5_MAX_NUM_TC,
1501 .stats_grps = mlx5e_ul_rep_stats_grps,
1502 .stats_grps_num = mlx5e_ul_rep_stats_grps_num,
1503};
1504
1505/* e-Switch vport representors */
1506static int
1507mlx5e_vport_uplink_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep)
1508{
1509 struct mlx5e_rep_priv *rpriv = mlx5e_rep_to_rep_priv(rep);
1510 struct net_device *netdev;
1511 int err;
1512
1513 netdev = mlx5_uplink_netdev_get(mdev: dev);
1514 if (!netdev)
1515 return 0;
1516
1517 /* must not use netdev_priv(netdev), it might not be initialized yet */
1518 rpriv->netdev = netdev;
1519 err = mlx5e_netdev_change_profile(netdev, mdev: dev,
1520 new_profile: &mlx5e_uplink_rep_profile, new_ppriv: rpriv);
1521 mlx5_uplink_netdev_put(mdev: dev, netdev);
1522 return err;
1523}
1524
1525static void
1526mlx5e_vport_uplink_rep_unload(struct mlx5e_rep_priv *rpriv)
1527{
1528 struct net_device *netdev = rpriv->netdev;
1529 struct mlx5e_priv *priv;
1530
1531 priv = netdev_priv(dev: netdev);
1532
1533 /* This bit is set when using devlink to change eswitch mode from
1534 * switchdev to legacy. As need to keep uplink netdev ifindex, we
1535 * detach uplink representor profile and attach NIC profile only.
1536 * The netdev will be unregistered later when unload NIC auxiliary
1537 * driver for this case.
1538 * We explicitly block devlink eswitch mode change if any IPSec rules
1539 * offloaded, but can't block other cases, such as driver unload
1540 * and devlink reload. We have to unregister netdev before profile
1541 * change for those cases. This is to avoid resource leak because
1542 * the offloaded rules don't have the chance to be unoffloaded before
1543 * cleanup which is triggered by detach uplink representor profile.
1544 */
1545 if (!(priv->mdev->priv.flags & MLX5_PRIV_FLAGS_SWITCH_LEGACY))
1546 unregister_netdev(dev: netdev);
1547
1548 mlx5e_netdev_attach_nic_profile(netdev, mdev: priv->mdev);
1549}
1550
1551static int
1552mlx5e_vport_vf_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep)
1553{
1554 struct mlx5e_rep_priv *rpriv = mlx5e_rep_to_rep_priv(rep);
1555 const struct mlx5e_profile *profile;
1556 struct devlink_port *dl_port;
1557 struct net_device *netdev;
1558 struct mlx5e_priv *priv;
1559 int err;
1560
1561 profile = &mlx5e_rep_profile;
1562 netdev = mlx5e_create_netdev(mdev: dev, profile);
1563 if (!netdev) {
1564 mlx5_core_warn(dev,
1565 "Failed to create representor netdev for vport %d\n",
1566 rep->vport);
1567 return -EINVAL;
1568 }
1569
1570 mlx5e_build_rep_netdev(netdev, mdev: dev);
1571 rpriv->netdev = netdev;
1572
1573 priv = netdev_priv(dev: netdev);
1574 priv->profile = profile;
1575 priv->ppriv = rpriv;
1576 err = profile->init(dev, netdev);
1577 if (err) {
1578 netdev_warn(dev: netdev, format: "rep profile init failed, %d\n", err);
1579 goto err_destroy_netdev;
1580 }
1581
1582 err = mlx5e_attach_netdev(priv: netdev_priv(dev: netdev));
1583 if (err) {
1584 netdev_warn(dev: netdev,
1585 format: "Failed to attach representor netdev for vport %d\n",
1586 rep->vport);
1587 goto err_cleanup_profile;
1588 }
1589
1590 dl_port = mlx5_esw_offloads_devlink_port(esw: dev->priv.eswitch,
1591 vport_num: rpriv->rep->vport);
1592 if (!IS_ERR(ptr: dl_port)) {
1593 SET_NETDEV_DEVLINK_PORT(netdev, dl_port);
1594 mlx5e_rep_vnic_reporter_create(priv, dl_port);
1595 }
1596
1597 err = register_netdev(dev: netdev);
1598 if (err) {
1599 netdev_warn(dev: netdev,
1600 format: "Failed to register representor netdev for vport %d\n",
1601 rep->vport);
1602 goto err_detach_netdev;
1603 }
1604
1605 return 0;
1606
1607err_detach_netdev:
1608 mlx5e_rep_vnic_reporter_destroy(priv);
1609 mlx5e_detach_netdev(priv: netdev_priv(dev: netdev));
1610err_cleanup_profile:
1611 priv->profile->cleanup(priv);
1612
1613err_destroy_netdev:
1614 mlx5e_destroy_netdev(netdev);
1615 return err;
1616}
1617
1618static int
1619mlx5e_vport_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep)
1620{
1621 struct mlx5e_rep_priv *rpriv;
1622 int err;
1623
1624 rpriv = kvzalloc(sizeof(*rpriv), GFP_KERNEL);
1625 if (!rpriv)
1626 return -ENOMEM;
1627
1628 /* rpriv->rep to be looked up when profile->init() is called */
1629 rpriv->rep = rep;
1630 rep->rep_data[REP_ETH].priv = rpriv;
1631 INIT_LIST_HEAD(list: &rpriv->vport_sqs_list);
1632
1633 if (rep->vport == MLX5_VPORT_UPLINK)
1634 err = mlx5e_vport_uplink_rep_load(dev, rep);
1635 else
1636 err = mlx5e_vport_vf_rep_load(dev, rep);
1637
1638 if (err)
1639 kvfree(addr: rpriv);
1640
1641 return err;
1642}
1643
1644static void
1645mlx5e_vport_rep_unload(struct mlx5_eswitch_rep *rep)
1646{
1647 struct mlx5e_rep_priv *rpriv = mlx5e_rep_to_rep_priv(rep);
1648 struct net_device *netdev = rpriv->netdev;
1649 struct mlx5e_priv *priv;
1650 void *ppriv;
1651
1652 if (!netdev) {
1653 ppriv = rpriv;
1654 goto free_ppriv;
1655 }
1656
1657 priv = netdev_priv(dev: netdev);
1658 ppriv = priv->ppriv;
1659
1660 if (rep->vport == MLX5_VPORT_UPLINK) {
1661 mlx5e_vport_uplink_rep_unload(rpriv);
1662 goto free_ppriv;
1663 }
1664
1665 unregister_netdev(dev: netdev);
1666 mlx5e_rep_vnic_reporter_destroy(priv);
1667 mlx5e_detach_netdev(priv);
1668 priv->profile->cleanup(priv);
1669 mlx5e_destroy_netdev(netdev);
1670free_ppriv:
1671 kvfree(addr: ppriv); /* mlx5e_rep_priv */
1672}
1673
1674static void *mlx5e_vport_rep_get_proto_dev(struct mlx5_eswitch_rep *rep)
1675{
1676 struct mlx5e_rep_priv *rpriv;
1677
1678 rpriv = mlx5e_rep_to_rep_priv(rep);
1679
1680 return rpriv->netdev;
1681}
1682
1683static void mlx5e_vport_rep_event_unpair(struct mlx5_eswitch_rep *rep,
1684 struct mlx5_eswitch *peer_esw)
1685{
1686 u16 i = MLX5_CAP_GEN(peer_esw->dev, vhca_id);
1687 struct mlx5e_rep_priv *rpriv;
1688 struct mlx5e_rep_sq *rep_sq;
1689
1690 WARN_ON_ONCE(!peer_esw);
1691 rpriv = mlx5e_rep_to_rep_priv(rep);
1692 list_for_each_entry(rep_sq, &rpriv->vport_sqs_list, list) {
1693 struct mlx5e_rep_sq_peer *sq_peer = xa_load(&rep_sq->sq_peer, index: i);
1694
1695 if (!sq_peer || sq_peer->peer != peer_esw)
1696 continue;
1697
1698 mlx5_eswitch_del_send_to_vport_rule(rule: sq_peer->rule);
1699 xa_erase(&rep_sq->sq_peer, index: i);
1700 kfree(objp: sq_peer);
1701 }
1702}
1703
1704static int mlx5e_vport_rep_event_pair(struct mlx5_eswitch *esw,
1705 struct mlx5_eswitch_rep *rep,
1706 struct mlx5_eswitch *peer_esw)
1707{
1708 u16 i = MLX5_CAP_GEN(peer_esw->dev, vhca_id);
1709 struct mlx5_flow_handle *flow_rule;
1710 struct mlx5e_rep_sq_peer *sq_peer;
1711 struct mlx5e_rep_priv *rpriv;
1712 struct mlx5e_rep_sq *rep_sq;
1713 int err;
1714
1715 rpriv = mlx5e_rep_to_rep_priv(rep);
1716 list_for_each_entry(rep_sq, &rpriv->vport_sqs_list, list) {
1717 sq_peer = xa_load(&rep_sq->sq_peer, index: i);
1718
1719 if (sq_peer && sq_peer->peer)
1720 continue;
1721
1722 flow_rule = mlx5_eswitch_add_send_to_vport_rule(on_esw: peer_esw, from_esw: esw, rep,
1723 sqn: rep_sq->sqn);
1724 if (IS_ERR(ptr: flow_rule)) {
1725 err = PTR_ERR(ptr: flow_rule);
1726 goto err_out;
1727 }
1728
1729 if (sq_peer) {
1730 sq_peer->rule = flow_rule;
1731 sq_peer->peer = peer_esw;
1732 continue;
1733 }
1734 sq_peer = kzalloc(sizeof(*sq_peer), GFP_KERNEL);
1735 if (!sq_peer) {
1736 err = -ENOMEM;
1737 goto err_sq_alloc;
1738 }
1739 err = xa_insert(xa: &rep_sq->sq_peer, index: i, entry: sq_peer, GFP_KERNEL);
1740 if (err)
1741 goto err_xa;
1742 sq_peer->rule = flow_rule;
1743 sq_peer->peer = peer_esw;
1744 }
1745
1746 return 0;
1747err_xa:
1748 kfree(objp: sq_peer);
1749err_sq_alloc:
1750 mlx5_eswitch_del_send_to_vport_rule(rule: flow_rule);
1751err_out:
1752 mlx5e_vport_rep_event_unpair(rep, peer_esw);
1753 return err;
1754}
1755
1756static int mlx5e_vport_rep_event(struct mlx5_eswitch *esw,
1757 struct mlx5_eswitch_rep *rep,
1758 enum mlx5_switchdev_event event,
1759 void *data)
1760{
1761 int err = 0;
1762
1763 if (event == MLX5_SWITCHDEV_EVENT_PAIR)
1764 err = mlx5e_vport_rep_event_pair(esw, rep, peer_esw: data);
1765 else if (event == MLX5_SWITCHDEV_EVENT_UNPAIR)
1766 mlx5e_vport_rep_event_unpair(rep, peer_esw: data);
1767
1768 return err;
1769}
1770
1771static const struct mlx5_eswitch_rep_ops rep_ops = {
1772 .load = mlx5e_vport_rep_load,
1773 .unload = mlx5e_vport_rep_unload,
1774 .get_proto_dev = mlx5e_vport_rep_get_proto_dev,
1775 .event = mlx5e_vport_rep_event,
1776};
1777
1778static int mlx5e_rep_probe(struct auxiliary_device *adev,
1779 const struct auxiliary_device_id *id)
1780{
1781 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev);
1782 struct mlx5_core_dev *mdev = edev->mdev;
1783 struct mlx5_eswitch *esw;
1784
1785 esw = mdev->priv.eswitch;
1786 mlx5_eswitch_register_vport_reps(esw, ops: &rep_ops, rep_type: REP_ETH);
1787 return 0;
1788}
1789
1790static void mlx5e_rep_remove(struct auxiliary_device *adev)
1791{
1792 struct mlx5_adev *vdev = container_of(adev, struct mlx5_adev, adev);
1793 struct mlx5_core_dev *mdev = vdev->mdev;
1794 struct mlx5_eswitch *esw;
1795
1796 esw = mdev->priv.eswitch;
1797 mlx5_eswitch_unregister_vport_reps(esw, rep_type: REP_ETH);
1798}
1799
1800static const struct auxiliary_device_id mlx5e_rep_id_table[] = {
1801 { .name = MLX5_ADEV_NAME ".eth-rep", },
1802 {},
1803};
1804
1805MODULE_DEVICE_TABLE(auxiliary, mlx5e_rep_id_table);
1806
1807static struct auxiliary_driver mlx5e_rep_driver = {
1808 .name = "eth-rep",
1809 .probe = mlx5e_rep_probe,
1810 .remove = mlx5e_rep_remove,
1811 .id_table = mlx5e_rep_id_table,
1812};
1813
1814int mlx5e_rep_init(void)
1815{
1816 return auxiliary_driver_register(&mlx5e_rep_driver);
1817}
1818
1819void mlx5e_rep_cleanup(void)
1820{
1821 auxiliary_driver_unregister(auxdrv: &mlx5e_rep_driver);
1822}
1823

source code of linux/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c