1// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
2/* QLogic qede NIC Driver
3 * Copyright (c) 2015-2017 QLogic Corporation
4 * Copyright (c) 2019-2020 Marvell International Ltd.
5 */
6
7#include <linux/crash_dump.h>
8#include <linux/module.h>
9#include <linux/pci.h>
10#include <linux/device.h>
11#include <linux/netdevice.h>
12#include <linux/etherdevice.h>
13#include <linux/skbuff.h>
14#include <linux/errno.h>
15#include <linux/list.h>
16#include <linux/string.h>
17#include <linux/dma-mapping.h>
18#include <linux/interrupt.h>
19#include <asm/byteorder.h>
20#include <asm/param.h>
21#include <linux/io.h>
22#include <linux/netdev_features.h>
23#include <linux/udp.h>
24#include <linux/tcp.h>
25#include <net/udp_tunnel.h>
26#include <linux/ip.h>
27#include <net/ipv6.h>
28#include <net/tcp.h>
29#include <linux/if_ether.h>
30#include <linux/if_vlan.h>
31#include <linux/pkt_sched.h>
32#include <linux/ethtool.h>
33#include <linux/in.h>
34#include <linux/random.h>
35#include <net/ip6_checksum.h>
36#include <linux/bitops.h>
37#include <linux/vmalloc.h>
38#include "qede.h"
39#include "qede_ptp.h"
40
41MODULE_DESCRIPTION("QLogic FastLinQ 4xxxx Ethernet Driver");
42MODULE_LICENSE("GPL");
43
44static uint debug;
45module_param(debug, uint, 0);
46MODULE_PARM_DESC(debug, " Default debug msglevel");
47
48static const struct qed_eth_ops *qed_ops;
49
50#define CHIP_NUM_57980S_40 0x1634
51#define CHIP_NUM_57980S_10 0x1666
52#define CHIP_NUM_57980S_MF 0x1636
53#define CHIP_NUM_57980S_100 0x1644
54#define CHIP_NUM_57980S_50 0x1654
55#define CHIP_NUM_57980S_25 0x1656
56#define CHIP_NUM_57980S_IOV 0x1664
57#define CHIP_NUM_AH 0x8070
58#define CHIP_NUM_AH_IOV 0x8090
59
60#ifndef PCI_DEVICE_ID_NX2_57980E
61#define PCI_DEVICE_ID_57980S_40 CHIP_NUM_57980S_40
62#define PCI_DEVICE_ID_57980S_10 CHIP_NUM_57980S_10
63#define PCI_DEVICE_ID_57980S_MF CHIP_NUM_57980S_MF
64#define PCI_DEVICE_ID_57980S_100 CHIP_NUM_57980S_100
65#define PCI_DEVICE_ID_57980S_50 CHIP_NUM_57980S_50
66#define PCI_DEVICE_ID_57980S_25 CHIP_NUM_57980S_25
67#define PCI_DEVICE_ID_57980S_IOV CHIP_NUM_57980S_IOV
68#define PCI_DEVICE_ID_AH CHIP_NUM_AH
69#define PCI_DEVICE_ID_AH_IOV CHIP_NUM_AH_IOV
70
71#endif
72
73enum qede_pci_private {
74 QEDE_PRIVATE_PF,
75 QEDE_PRIVATE_VF
76};
77
78static const struct pci_device_id qede_pci_tbl[] = {
79 {PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_57980S_40), QEDE_PRIVATE_PF},
80 {PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_57980S_10), QEDE_PRIVATE_PF},
81 {PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_57980S_MF), QEDE_PRIVATE_PF},
82 {PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_57980S_100), QEDE_PRIVATE_PF},
83 {PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_57980S_50), QEDE_PRIVATE_PF},
84 {PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_57980S_25), QEDE_PRIVATE_PF},
85#ifdef CONFIG_QED_SRIOV
86 {PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_57980S_IOV), QEDE_PRIVATE_VF},
87#endif
88 {PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_AH), QEDE_PRIVATE_PF},
89#ifdef CONFIG_QED_SRIOV
90 {PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_AH_IOV), QEDE_PRIVATE_VF},
91#endif
92 { 0 }
93};
94
95MODULE_DEVICE_TABLE(pci, qede_pci_tbl);
96
97static int qede_probe(struct pci_dev *pdev, const struct pci_device_id *id);
98static pci_ers_result_t
99qede_io_error_detected(struct pci_dev *pdev, pci_channel_state_t state);
100
101#define TX_TIMEOUT (5 * HZ)
102
103/* Utilize last protocol index for XDP */
104#define XDP_PI 11
105
106static void qede_remove(struct pci_dev *pdev);
107static void qede_shutdown(struct pci_dev *pdev);
108static void qede_link_update(void *dev, struct qed_link_output *link);
109static void qede_schedule_recovery_handler(void *dev);
110static void qede_recovery_handler(struct qede_dev *edev);
111static void qede_schedule_hw_err_handler(void *dev,
112 enum qed_hw_err_type err_type);
113static void qede_get_eth_tlv_data(void *edev, void *data);
114static void qede_get_generic_tlv_data(void *edev,
115 struct qed_generic_tlvs *data);
116static void qede_generic_hw_err_handler(struct qede_dev *edev);
117#ifdef CONFIG_QED_SRIOV
118static int qede_set_vf_vlan(struct net_device *ndev, int vf, u16 vlan, u8 qos,
119 __be16 vlan_proto)
120{
121 struct qede_dev *edev = netdev_priv(dev: ndev);
122
123 if (vlan > 4095) {
124 DP_NOTICE(edev, "Illegal vlan value %d\n", vlan);
125 return -EINVAL;
126 }
127
128 if (vlan_proto != htons(ETH_P_8021Q))
129 return -EPROTONOSUPPORT;
130
131 DP_VERBOSE(edev, QED_MSG_IOV, "Setting Vlan 0x%04x to VF [%d]\n",
132 vlan, vf);
133
134 return edev->ops->iov->set_vlan(edev->cdev, vlan, vf);
135}
136
137static int qede_set_vf_mac(struct net_device *ndev, int vfidx, u8 *mac)
138{
139 struct qede_dev *edev = netdev_priv(dev: ndev);
140
141 DP_VERBOSE(edev, QED_MSG_IOV, "Setting MAC %pM to VF [%d]\n", mac, vfidx);
142
143 if (!is_valid_ether_addr(addr: mac)) {
144 DP_VERBOSE(edev, QED_MSG_IOV, "MAC address isn't valid\n");
145 return -EINVAL;
146 }
147
148 return edev->ops->iov->set_mac(edev->cdev, mac, vfidx);
149}
150
151static int qede_sriov_configure(struct pci_dev *pdev, int num_vfs_param)
152{
153 struct qede_dev *edev = netdev_priv(dev: pci_get_drvdata(pdev));
154 struct qed_dev_info *qed_info = &edev->dev_info.common;
155 struct qed_update_vport_params *vport_params;
156 int rc;
157
158 vport_params = vzalloc(sizeof(*vport_params));
159 if (!vport_params)
160 return -ENOMEM;
161 DP_VERBOSE(edev, QED_MSG_IOV, "Requested %d VFs\n", num_vfs_param);
162
163 rc = edev->ops->iov->configure(edev->cdev, num_vfs_param);
164
165 /* Enable/Disable Tx switching for PF */
166 if ((rc == num_vfs_param) && netif_running(dev: edev->ndev) &&
167 !qed_info->b_inter_pf_switch && qed_info->tx_switching) {
168 vport_params->vport_id = 0;
169 vport_params->update_tx_switching_flg = 1;
170 vport_params->tx_switching_flg = num_vfs_param ? 1 : 0;
171 edev->ops->vport_update(edev->cdev, vport_params);
172 }
173
174 vfree(addr: vport_params);
175 return rc;
176}
177#endif
178
179static int __maybe_unused qede_suspend(struct device *dev)
180{
181 dev_info(dev, "Device does not support suspend operation\n");
182
183 return -EOPNOTSUPP;
184}
185
186static DEFINE_SIMPLE_DEV_PM_OPS(qede_pm_ops, qede_suspend, NULL);
187
188static const struct pci_error_handlers qede_err_handler = {
189 .error_detected = qede_io_error_detected,
190};
191
192static struct pci_driver qede_pci_driver = {
193 .name = "qede",
194 .id_table = qede_pci_tbl,
195 .probe = qede_probe,
196 .remove = qede_remove,
197 .shutdown = qede_shutdown,
198#ifdef CONFIG_QED_SRIOV
199 .sriov_configure = qede_sriov_configure,
200#endif
201 .err_handler = &qede_err_handler,
202 .driver.pm = &qede_pm_ops,
203};
204
205static struct qed_eth_cb_ops qede_ll_ops = {
206 .common = {
207#ifdef CONFIG_RFS_ACCEL
208 .arfs_filter_op = qede_arfs_filter_op,
209#endif
210 .link_update = qede_link_update,
211 .schedule_recovery_handler = qede_schedule_recovery_handler,
212 .schedule_hw_err_handler = qede_schedule_hw_err_handler,
213 .get_generic_tlv_data = qede_get_generic_tlv_data,
214 .get_protocol_tlv_data = qede_get_eth_tlv_data,
215 },
216 .force_mac = qede_force_mac,
217 .ports_update = qede_udp_ports_update,
218};
219
220static int qede_netdev_event(struct notifier_block *this, unsigned long event,
221 void *ptr)
222{
223 struct net_device *ndev = netdev_notifier_info_to_dev(info: ptr);
224 struct ethtool_drvinfo drvinfo;
225 struct qede_dev *edev;
226
227 if (event != NETDEV_CHANGENAME && event != NETDEV_CHANGEADDR)
228 goto done;
229
230 /* Check whether this is a qede device */
231 if (!ndev || !ndev->ethtool_ops || !ndev->ethtool_ops->get_drvinfo)
232 goto done;
233
234 memset(&drvinfo, 0, sizeof(drvinfo));
235 ndev->ethtool_ops->get_drvinfo(ndev, &drvinfo);
236 if (strcmp(drvinfo.driver, "qede"))
237 goto done;
238 edev = netdev_priv(dev: ndev);
239
240 switch (event) {
241 case NETDEV_CHANGENAME:
242 /* Notify qed of the name change */
243 if (!edev->ops || !edev->ops->common)
244 goto done;
245 edev->ops->common->set_name(edev->cdev, edev->ndev->name);
246 break;
247 case NETDEV_CHANGEADDR:
248 edev = netdev_priv(dev: ndev);
249 qede_rdma_event_changeaddr(edr: edev);
250 break;
251 }
252
253done:
254 return NOTIFY_DONE;
255}
256
257static struct notifier_block qede_netdev_notifier = {
258 .notifier_call = qede_netdev_event,
259};
260
261static
262int __init qede_init(void)
263{
264 int ret;
265
266 pr_info("qede init: QLogic FastLinQ 4xxxx Ethernet Driver qede\n");
267
268 qede_forced_speed_maps_init();
269
270 qed_ops = qed_get_eth_ops();
271 if (!qed_ops) {
272 pr_notice("Failed to get qed ethtool operations\n");
273 return -EINVAL;
274 }
275
276 /* Must register notifier before pci ops, since we might miss
277 * interface rename after pci probe and netdev registration.
278 */
279 ret = register_netdevice_notifier(nb: &qede_netdev_notifier);
280 if (ret) {
281 pr_notice("Failed to register netdevice_notifier\n");
282 qed_put_eth_ops();
283 return -EINVAL;
284 }
285
286 ret = pci_register_driver(&qede_pci_driver);
287 if (ret) {
288 pr_notice("Failed to register driver\n");
289 unregister_netdevice_notifier(nb: &qede_netdev_notifier);
290 qed_put_eth_ops();
291 return -EINVAL;
292 }
293
294 return 0;
295}
296
297static void __exit qede_cleanup(void)
298{
299 if (debug & QED_LOG_INFO_MASK)
300 pr_info("qede_cleanup called\n");
301
302 unregister_netdevice_notifier(nb: &qede_netdev_notifier);
303 pci_unregister_driver(dev: &qede_pci_driver);
304 qed_put_eth_ops();
305}
306
307module_init(qede_init);
308module_exit(qede_cleanup);
309
310static int qede_open(struct net_device *ndev);
311static int qede_close(struct net_device *ndev);
312
313void qede_fill_by_demand_stats(struct qede_dev *edev)
314{
315 struct qede_stats_common *p_common = &edev->stats.common;
316 struct qed_eth_stats stats;
317
318 edev->ops->get_vport_stats(edev->cdev, &stats);
319
320 spin_lock(lock: &edev->stats_lock);
321
322 p_common->no_buff_discards = stats.common.no_buff_discards;
323 p_common->packet_too_big_discard = stats.common.packet_too_big_discard;
324 p_common->ttl0_discard = stats.common.ttl0_discard;
325 p_common->rx_ucast_bytes = stats.common.rx_ucast_bytes;
326 p_common->rx_mcast_bytes = stats.common.rx_mcast_bytes;
327 p_common->rx_bcast_bytes = stats.common.rx_bcast_bytes;
328 p_common->rx_ucast_pkts = stats.common.rx_ucast_pkts;
329 p_common->rx_mcast_pkts = stats.common.rx_mcast_pkts;
330 p_common->rx_bcast_pkts = stats.common.rx_bcast_pkts;
331 p_common->mftag_filter_discards = stats.common.mftag_filter_discards;
332 p_common->mac_filter_discards = stats.common.mac_filter_discards;
333 p_common->gft_filter_drop = stats.common.gft_filter_drop;
334
335 p_common->tx_ucast_bytes = stats.common.tx_ucast_bytes;
336 p_common->tx_mcast_bytes = stats.common.tx_mcast_bytes;
337 p_common->tx_bcast_bytes = stats.common.tx_bcast_bytes;
338 p_common->tx_ucast_pkts = stats.common.tx_ucast_pkts;
339 p_common->tx_mcast_pkts = stats.common.tx_mcast_pkts;
340 p_common->tx_bcast_pkts = stats.common.tx_bcast_pkts;
341 p_common->tx_err_drop_pkts = stats.common.tx_err_drop_pkts;
342 p_common->coalesced_pkts = stats.common.tpa_coalesced_pkts;
343 p_common->coalesced_events = stats.common.tpa_coalesced_events;
344 p_common->coalesced_aborts_num = stats.common.tpa_aborts_num;
345 p_common->non_coalesced_pkts = stats.common.tpa_not_coalesced_pkts;
346 p_common->coalesced_bytes = stats.common.tpa_coalesced_bytes;
347
348 p_common->rx_64_byte_packets = stats.common.rx_64_byte_packets;
349 p_common->rx_65_to_127_byte_packets =
350 stats.common.rx_65_to_127_byte_packets;
351 p_common->rx_128_to_255_byte_packets =
352 stats.common.rx_128_to_255_byte_packets;
353 p_common->rx_256_to_511_byte_packets =
354 stats.common.rx_256_to_511_byte_packets;
355 p_common->rx_512_to_1023_byte_packets =
356 stats.common.rx_512_to_1023_byte_packets;
357 p_common->rx_1024_to_1518_byte_packets =
358 stats.common.rx_1024_to_1518_byte_packets;
359 p_common->rx_crc_errors = stats.common.rx_crc_errors;
360 p_common->rx_mac_crtl_frames = stats.common.rx_mac_crtl_frames;
361 p_common->rx_pause_frames = stats.common.rx_pause_frames;
362 p_common->rx_pfc_frames = stats.common.rx_pfc_frames;
363 p_common->rx_align_errors = stats.common.rx_align_errors;
364 p_common->rx_carrier_errors = stats.common.rx_carrier_errors;
365 p_common->rx_oversize_packets = stats.common.rx_oversize_packets;
366 p_common->rx_jabbers = stats.common.rx_jabbers;
367 p_common->rx_undersize_packets = stats.common.rx_undersize_packets;
368 p_common->rx_fragments = stats.common.rx_fragments;
369 p_common->tx_64_byte_packets = stats.common.tx_64_byte_packets;
370 p_common->tx_65_to_127_byte_packets =
371 stats.common.tx_65_to_127_byte_packets;
372 p_common->tx_128_to_255_byte_packets =
373 stats.common.tx_128_to_255_byte_packets;
374 p_common->tx_256_to_511_byte_packets =
375 stats.common.tx_256_to_511_byte_packets;
376 p_common->tx_512_to_1023_byte_packets =
377 stats.common.tx_512_to_1023_byte_packets;
378 p_common->tx_1024_to_1518_byte_packets =
379 stats.common.tx_1024_to_1518_byte_packets;
380 p_common->tx_pause_frames = stats.common.tx_pause_frames;
381 p_common->tx_pfc_frames = stats.common.tx_pfc_frames;
382 p_common->brb_truncates = stats.common.brb_truncates;
383 p_common->brb_discards = stats.common.brb_discards;
384 p_common->tx_mac_ctrl_frames = stats.common.tx_mac_ctrl_frames;
385 p_common->link_change_count = stats.common.link_change_count;
386 p_common->ptp_skip_txts = edev->ptp_skip_txts;
387
388 if (QEDE_IS_BB(edev)) {
389 struct qede_stats_bb *p_bb = &edev->stats.bb;
390
391 p_bb->rx_1519_to_1522_byte_packets =
392 stats.bb.rx_1519_to_1522_byte_packets;
393 p_bb->rx_1519_to_2047_byte_packets =
394 stats.bb.rx_1519_to_2047_byte_packets;
395 p_bb->rx_2048_to_4095_byte_packets =
396 stats.bb.rx_2048_to_4095_byte_packets;
397 p_bb->rx_4096_to_9216_byte_packets =
398 stats.bb.rx_4096_to_9216_byte_packets;
399 p_bb->rx_9217_to_16383_byte_packets =
400 stats.bb.rx_9217_to_16383_byte_packets;
401 p_bb->tx_1519_to_2047_byte_packets =
402 stats.bb.tx_1519_to_2047_byte_packets;
403 p_bb->tx_2048_to_4095_byte_packets =
404 stats.bb.tx_2048_to_4095_byte_packets;
405 p_bb->tx_4096_to_9216_byte_packets =
406 stats.bb.tx_4096_to_9216_byte_packets;
407 p_bb->tx_9217_to_16383_byte_packets =
408 stats.bb.tx_9217_to_16383_byte_packets;
409 p_bb->tx_lpi_entry_count = stats.bb.tx_lpi_entry_count;
410 p_bb->tx_total_collisions = stats.bb.tx_total_collisions;
411 } else {
412 struct qede_stats_ah *p_ah = &edev->stats.ah;
413
414 p_ah->rx_1519_to_max_byte_packets =
415 stats.ah.rx_1519_to_max_byte_packets;
416 p_ah->tx_1519_to_max_byte_packets =
417 stats.ah.tx_1519_to_max_byte_packets;
418 }
419
420 spin_unlock(lock: &edev->stats_lock);
421}
422
423static void qede_get_stats64(struct net_device *dev,
424 struct rtnl_link_stats64 *stats)
425{
426 struct qede_dev *edev = netdev_priv(dev);
427 struct qede_stats_common *p_common;
428
429 p_common = &edev->stats.common;
430
431 spin_lock(lock: &edev->stats_lock);
432
433 stats->rx_packets = p_common->rx_ucast_pkts + p_common->rx_mcast_pkts +
434 p_common->rx_bcast_pkts;
435 stats->tx_packets = p_common->tx_ucast_pkts + p_common->tx_mcast_pkts +
436 p_common->tx_bcast_pkts;
437
438 stats->rx_bytes = p_common->rx_ucast_bytes + p_common->rx_mcast_bytes +
439 p_common->rx_bcast_bytes;
440 stats->tx_bytes = p_common->tx_ucast_bytes + p_common->tx_mcast_bytes +
441 p_common->tx_bcast_bytes;
442
443 stats->tx_errors = p_common->tx_err_drop_pkts;
444 stats->multicast = p_common->rx_mcast_pkts + p_common->rx_bcast_pkts;
445
446 stats->rx_fifo_errors = p_common->no_buff_discards;
447
448 if (QEDE_IS_BB(edev))
449 stats->collisions = edev->stats.bb.tx_total_collisions;
450 stats->rx_crc_errors = p_common->rx_crc_errors;
451 stats->rx_frame_errors = p_common->rx_align_errors;
452
453 spin_unlock(lock: &edev->stats_lock);
454}
455
456#ifdef CONFIG_QED_SRIOV
457static int qede_get_vf_config(struct net_device *dev, int vfidx,
458 struct ifla_vf_info *ivi)
459{
460 struct qede_dev *edev = netdev_priv(dev);
461
462 if (!edev->ops)
463 return -EINVAL;
464
465 return edev->ops->iov->get_config(edev->cdev, vfidx, ivi);
466}
467
468static int qede_set_vf_rate(struct net_device *dev, int vfidx,
469 int min_tx_rate, int max_tx_rate)
470{
471 struct qede_dev *edev = netdev_priv(dev);
472
473 return edev->ops->iov->set_rate(edev->cdev, vfidx, min_tx_rate,
474 max_tx_rate);
475}
476
477static int qede_set_vf_spoofchk(struct net_device *dev, int vfidx, bool val)
478{
479 struct qede_dev *edev = netdev_priv(dev);
480
481 if (!edev->ops)
482 return -EINVAL;
483
484 return edev->ops->iov->set_spoof(edev->cdev, vfidx, val);
485}
486
487static int qede_set_vf_link_state(struct net_device *dev, int vfidx,
488 int link_state)
489{
490 struct qede_dev *edev = netdev_priv(dev);
491
492 if (!edev->ops)
493 return -EINVAL;
494
495 return edev->ops->iov->set_link_state(edev->cdev, vfidx, link_state);
496}
497
498static int qede_set_vf_trust(struct net_device *dev, int vfidx, bool setting)
499{
500 struct qede_dev *edev = netdev_priv(dev);
501
502 if (!edev->ops)
503 return -EINVAL;
504
505 return edev->ops->iov->set_trust(edev->cdev, vfidx, setting);
506}
507#endif
508
509static void qede_fp_sb_dump(struct qede_dev *edev, struct qede_fastpath *fp)
510{
511 char *p_sb = (char *)fp->sb_info->sb_virt;
512 u32 sb_size, i;
513
514 sb_size = sizeof(struct status_block);
515
516 for (i = 0; i < sb_size; i += 8)
517 DP_NOTICE(edev,
518 "%02hhX %02hhX %02hhX %02hhX %02hhX %02hhX %02hhX %02hhX\n",
519 p_sb[i], p_sb[i + 1], p_sb[i + 2], p_sb[i + 3],
520 p_sb[i + 4], p_sb[i + 5], p_sb[i + 6], p_sb[i + 7]);
521}
522
523static void
524qede_txq_fp_log_metadata(struct qede_dev *edev,
525 struct qede_fastpath *fp, struct qede_tx_queue *txq)
526{
527 struct qed_chain *p_chain = &txq->tx_pbl;
528
529 /* Dump txq/fp/sb ids etc. other metadata */
530 DP_NOTICE(edev,
531 "fpid 0x%x sbid 0x%x txqid [0x%x] ndev_qid [0x%x] cos [0x%x] p_chain %p cap %d size %d jiffies %lu HZ 0x%x\n",
532 fp->id, fp->sb_info->igu_sb_id, txq->index, txq->ndev_txq_id, txq->cos,
533 p_chain, p_chain->capacity, p_chain->size, jiffies, HZ);
534
535 /* Dump all the relevant prod/cons indexes */
536 DP_NOTICE(edev,
537 "hw cons %04x sw_tx_prod=0x%x, sw_tx_cons=0x%x, bd_prod 0x%x bd_cons 0x%x\n",
538 le16_to_cpu(*txq->hw_cons_ptr), txq->sw_tx_prod, txq->sw_tx_cons,
539 qed_chain_get_prod_idx(p_chain), qed_chain_get_cons_idx(p_chain));
540}
541
542static void
543qede_tx_log_print(struct qede_dev *edev, struct qede_fastpath *fp, struct qede_tx_queue *txq)
544{
545 struct qed_sb_info_dbg sb_dbg;
546 int rc;
547
548 /* sb info */
549 qede_fp_sb_dump(edev, fp);
550
551 memset(&sb_dbg, 0, sizeof(sb_dbg));
552 rc = edev->ops->common->get_sb_info(edev->cdev, fp->sb_info, (u16)fp->id, &sb_dbg);
553
554 DP_NOTICE(edev, "IGU: prod %08x cons %08x CAU Tx %04x\n",
555 sb_dbg.igu_prod, sb_dbg.igu_cons, sb_dbg.pi[TX_PI(txq->cos)]);
556
557 /* report to mfw */
558 edev->ops->common->mfw_report(edev->cdev,
559 "Txq[%d]: FW cons [host] %04x, SW cons %04x, SW prod %04x [Jiffies %lu]\n",
560 txq->index, le16_to_cpu(*txq->hw_cons_ptr),
561 qed_chain_get_cons_idx(chain: &txq->tx_pbl),
562 qed_chain_get_prod_idx(chain: &txq->tx_pbl), jiffies);
563 if (!rc)
564 edev->ops->common->mfw_report(edev->cdev,
565 "Txq[%d]: SB[0x%04x] - IGU: prod %08x cons %08x CAU Tx %04x\n",
566 txq->index, fp->sb_info->igu_sb_id,
567 sb_dbg.igu_prod, sb_dbg.igu_cons,
568 sb_dbg.pi[TX_PI(txq->cos)]);
569}
570
571static void qede_tx_timeout(struct net_device *dev, unsigned int txqueue)
572{
573 struct qede_dev *edev = netdev_priv(dev);
574 int i;
575
576 netif_carrier_off(dev);
577 DP_NOTICE(edev, "TX timeout on queue %u!\n", txqueue);
578
579 for_each_queue(i) {
580 struct qede_tx_queue *txq;
581 struct qede_fastpath *fp;
582 int cos;
583
584 fp = &edev->fp_array[i];
585 if (!(fp->type & QEDE_FASTPATH_TX))
586 continue;
587
588 for_each_cos_in_txq(edev, cos) {
589 txq = &fp->txq[cos];
590
591 /* Dump basic metadata for all queues */
592 qede_txq_fp_log_metadata(edev, fp, txq);
593
594 if (qed_chain_get_cons_idx(chain: &txq->tx_pbl) !=
595 qed_chain_get_prod_idx(chain: &txq->tx_pbl))
596 qede_tx_log_print(edev, fp, txq);
597 }
598 }
599
600 if (IS_VF(edev))
601 return;
602
603 if (test_and_set_bit(QEDE_ERR_IS_HANDLED, addr: &edev->err_flags) ||
604 edev->state == QEDE_STATE_RECOVERY) {
605 DP_INFO(edev,
606 "Avoid handling a Tx timeout while another HW error is being handled\n");
607 return;
608 }
609
610 set_bit(QEDE_ERR_GET_DBG_INFO, addr: &edev->err_flags);
611 set_bit(QEDE_SP_HW_ERR, addr: &edev->sp_flags);
612 schedule_delayed_work(dwork: &edev->sp_task, delay: 0);
613}
614
615static int qede_setup_tc(struct net_device *ndev, u8 num_tc)
616{
617 struct qede_dev *edev = netdev_priv(dev: ndev);
618 int cos, count, offset;
619
620 if (num_tc > edev->dev_info.num_tc)
621 return -EINVAL;
622
623 netdev_reset_tc(dev: ndev);
624 netdev_set_num_tc(dev: ndev, num_tc);
625
626 for_each_cos_in_txq(edev, cos) {
627 count = QEDE_TSS_COUNT(edev);
628 offset = cos * QEDE_TSS_COUNT(edev);
629 netdev_set_tc_queue(dev: ndev, tc: cos, count, offset);
630 }
631
632 return 0;
633}
634
635static int
636qede_set_flower(struct qede_dev *edev, struct flow_cls_offload *f,
637 __be16 proto)
638{
639 switch (f->command) {
640 case FLOW_CLS_REPLACE:
641 return qede_add_tc_flower_fltr(edev, proto, f);
642 case FLOW_CLS_DESTROY:
643 return qede_delete_flow_filter(edev, cookie: f->cookie);
644 default:
645 return -EOPNOTSUPP;
646 }
647}
648
649static int qede_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
650 void *cb_priv)
651{
652 struct flow_cls_offload *f;
653 struct qede_dev *edev = cb_priv;
654
655 if (!tc_cls_can_offload_and_chain0(dev: edev->ndev, common: type_data))
656 return -EOPNOTSUPP;
657
658 switch (type) {
659 case TC_SETUP_CLSFLOWER:
660 f = type_data;
661 return qede_set_flower(edev, f, proto: f->common.protocol);
662 default:
663 return -EOPNOTSUPP;
664 }
665}
666
667static LIST_HEAD(qede_block_cb_list);
668
669static int
670qede_setup_tc_offload(struct net_device *dev, enum tc_setup_type type,
671 void *type_data)
672{
673 struct qede_dev *edev = netdev_priv(dev);
674 struct tc_mqprio_qopt *mqprio;
675
676 switch (type) {
677 case TC_SETUP_BLOCK:
678 return flow_block_cb_setup_simple(f: type_data,
679 driver_list: &qede_block_cb_list,
680 cb: qede_setup_tc_block_cb,
681 cb_ident: edev, cb_priv: edev, ingress_only: true);
682 case TC_SETUP_QDISC_MQPRIO:
683 mqprio = type_data;
684
685 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
686 return qede_setup_tc(ndev: dev, num_tc: mqprio->num_tc);
687 default:
688 return -EOPNOTSUPP;
689 }
690}
691
692static const struct net_device_ops qede_netdev_ops = {
693 .ndo_open = qede_open,
694 .ndo_stop = qede_close,
695 .ndo_start_xmit = qede_start_xmit,
696 .ndo_select_queue = qede_select_queue,
697 .ndo_set_rx_mode = qede_set_rx_mode,
698 .ndo_set_mac_address = qede_set_mac_addr,
699 .ndo_validate_addr = eth_validate_addr,
700 .ndo_change_mtu = qede_change_mtu,
701 .ndo_tx_timeout = qede_tx_timeout,
702#ifdef CONFIG_QED_SRIOV
703 .ndo_set_vf_mac = qede_set_vf_mac,
704 .ndo_set_vf_vlan = qede_set_vf_vlan,
705 .ndo_set_vf_trust = qede_set_vf_trust,
706#endif
707 .ndo_vlan_rx_add_vid = qede_vlan_rx_add_vid,
708 .ndo_vlan_rx_kill_vid = qede_vlan_rx_kill_vid,
709 .ndo_fix_features = qede_fix_features,
710 .ndo_set_features = qede_set_features,
711 .ndo_get_stats64 = qede_get_stats64,
712#ifdef CONFIG_QED_SRIOV
713 .ndo_set_vf_link_state = qede_set_vf_link_state,
714 .ndo_set_vf_spoofchk = qede_set_vf_spoofchk,
715 .ndo_get_vf_config = qede_get_vf_config,
716 .ndo_set_vf_rate = qede_set_vf_rate,
717#endif
718 .ndo_features_check = qede_features_check,
719 .ndo_bpf = qede_xdp,
720#ifdef CONFIG_RFS_ACCEL
721 .ndo_rx_flow_steer = qede_rx_flow_steer,
722#endif
723 .ndo_xdp_xmit = qede_xdp_transmit,
724 .ndo_setup_tc = qede_setup_tc_offload,
725 .ndo_hwtstamp_get = qede_hwtstamp_get,
726 .ndo_hwtstamp_set = qede_hwtstamp_set,
727};
728
729static const struct net_device_ops qede_netdev_vf_ops = {
730 .ndo_open = qede_open,
731 .ndo_stop = qede_close,
732 .ndo_start_xmit = qede_start_xmit,
733 .ndo_select_queue = qede_select_queue,
734 .ndo_set_rx_mode = qede_set_rx_mode,
735 .ndo_set_mac_address = qede_set_mac_addr,
736 .ndo_validate_addr = eth_validate_addr,
737 .ndo_change_mtu = qede_change_mtu,
738 .ndo_vlan_rx_add_vid = qede_vlan_rx_add_vid,
739 .ndo_vlan_rx_kill_vid = qede_vlan_rx_kill_vid,
740 .ndo_fix_features = qede_fix_features,
741 .ndo_set_features = qede_set_features,
742 .ndo_get_stats64 = qede_get_stats64,
743 .ndo_features_check = qede_features_check,
744};
745
746static const struct net_device_ops qede_netdev_vf_xdp_ops = {
747 .ndo_open = qede_open,
748 .ndo_stop = qede_close,
749 .ndo_start_xmit = qede_start_xmit,
750 .ndo_select_queue = qede_select_queue,
751 .ndo_set_rx_mode = qede_set_rx_mode,
752 .ndo_set_mac_address = qede_set_mac_addr,
753 .ndo_validate_addr = eth_validate_addr,
754 .ndo_change_mtu = qede_change_mtu,
755 .ndo_vlan_rx_add_vid = qede_vlan_rx_add_vid,
756 .ndo_vlan_rx_kill_vid = qede_vlan_rx_kill_vid,
757 .ndo_fix_features = qede_fix_features,
758 .ndo_set_features = qede_set_features,
759 .ndo_get_stats64 = qede_get_stats64,
760 .ndo_features_check = qede_features_check,
761 .ndo_bpf = qede_xdp,
762 .ndo_xdp_xmit = qede_xdp_transmit,
763};
764
765/* -------------------------------------------------------------------------
766 * START OF PROBE / REMOVE
767 * -------------------------------------------------------------------------
768 */
769
770static struct qede_dev *qede_alloc_etherdev(struct qed_dev *cdev,
771 struct pci_dev *pdev,
772 struct qed_dev_eth_info *info,
773 u32 dp_module, u8 dp_level)
774{
775 struct net_device *ndev;
776 struct qede_dev *edev;
777
778 ndev = alloc_etherdev_mqs(sizeof_priv: sizeof(*edev),
779 txqs: info->num_queues * info->num_tc,
780 rxqs: info->num_queues);
781 if (!ndev) {
782 pr_err("etherdev allocation failed\n");
783 return NULL;
784 }
785
786 edev = netdev_priv(dev: ndev);
787 edev->ndev = ndev;
788 edev->cdev = cdev;
789 edev->pdev = pdev;
790 edev->dp_module = dp_module;
791 edev->dp_level = dp_level;
792 edev->ops = qed_ops;
793
794 if (is_kdump_kernel()) {
795 edev->q_num_rx_buffers = NUM_RX_BDS_KDUMP_MIN;
796 edev->q_num_tx_buffers = NUM_TX_BDS_KDUMP_MIN;
797 } else {
798 edev->q_num_rx_buffers = NUM_RX_BDS_DEF;
799 edev->q_num_tx_buffers = NUM_TX_BDS_DEF;
800 }
801
802 DP_INFO(edev, "Allocated netdev with %d tx queues and %d rx queues\n",
803 info->num_queues, info->num_queues);
804
805 SET_NETDEV_DEV(ndev, &pdev->dev);
806
807 memset(&edev->stats, 0, sizeof(edev->stats));
808 memcpy(&edev->dev_info, info, sizeof(*info));
809
810 /* As ethtool doesn't have the ability to show WoL behavior as
811 * 'default', if device supports it declare it's enabled.
812 */
813 if (edev->dev_info.common.wol_support)
814 edev->wol_enabled = true;
815
816 INIT_LIST_HEAD(list: &edev->vlan_list);
817
818 return edev;
819}
820
821static void qede_init_ndev(struct qede_dev *edev)
822{
823 struct net_device *ndev = edev->ndev;
824 struct pci_dev *pdev = edev->pdev;
825 bool udp_tunnel_enable = false;
826 netdev_features_t hw_features;
827
828 pci_set_drvdata(pdev, data: ndev);
829
830 ndev->mem_start = edev->dev_info.common.pci_mem_start;
831 ndev->base_addr = ndev->mem_start;
832 ndev->mem_end = edev->dev_info.common.pci_mem_end;
833 ndev->irq = edev->dev_info.common.pci_irq;
834
835 ndev->watchdog_timeo = TX_TIMEOUT;
836
837 if (IS_VF(edev)) {
838 if (edev->dev_info.xdp_supported)
839 ndev->netdev_ops = &qede_netdev_vf_xdp_ops;
840 else
841 ndev->netdev_ops = &qede_netdev_vf_ops;
842 } else {
843 ndev->netdev_ops = &qede_netdev_ops;
844 }
845
846 qede_set_ethtool_ops(netdev: ndev);
847
848 ndev->priv_flags |= IFF_UNICAST_FLT;
849
850 /* user-changeble features */
851 hw_features = NETIF_F_GRO | NETIF_F_GRO_HW | NETIF_F_SG |
852 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
853 NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_HW_TC;
854
855 if (edev->dev_info.common.b_arfs_capable)
856 hw_features |= NETIF_F_NTUPLE;
857
858 if (edev->dev_info.common.vxlan_enable ||
859 edev->dev_info.common.geneve_enable)
860 udp_tunnel_enable = true;
861
862 if (udp_tunnel_enable || edev->dev_info.common.gre_enable) {
863 hw_features |= NETIF_F_TSO_ECN;
864 ndev->hw_enc_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
865 NETIF_F_SG | NETIF_F_TSO |
866 NETIF_F_TSO_ECN | NETIF_F_TSO6 |
867 NETIF_F_RXCSUM;
868 }
869
870 if (udp_tunnel_enable) {
871 hw_features |= (NETIF_F_GSO_UDP_TUNNEL |
872 NETIF_F_GSO_UDP_TUNNEL_CSUM);
873 ndev->hw_enc_features |= (NETIF_F_GSO_UDP_TUNNEL |
874 NETIF_F_GSO_UDP_TUNNEL_CSUM);
875
876 qede_set_udp_tunnels(edev);
877 }
878
879 if (edev->dev_info.common.gre_enable) {
880 hw_features |= (NETIF_F_GSO_GRE | NETIF_F_GSO_GRE_CSUM);
881 ndev->hw_enc_features |= (NETIF_F_GSO_GRE |
882 NETIF_F_GSO_GRE_CSUM);
883 }
884
885 ndev->vlan_features = hw_features | NETIF_F_RXHASH | NETIF_F_RXCSUM |
886 NETIF_F_HIGHDMA;
887 ndev->features = hw_features | NETIF_F_RXHASH | NETIF_F_RXCSUM |
888 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HIGHDMA |
889 NETIF_F_HW_VLAN_CTAG_FILTER | NETIF_F_HW_VLAN_CTAG_TX;
890
891 ndev->hw_features = hw_features;
892
893 ndev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
894 NETDEV_XDP_ACT_NDO_XMIT;
895
896 /* MTU range: 46 - 9600 */
897 ndev->min_mtu = ETH_ZLEN - ETH_HLEN;
898 ndev->max_mtu = QEDE_MAX_JUMBO_PACKET_SIZE;
899
900 /* Set network device HW mac */
901 eth_hw_addr_set(dev: edev->ndev, addr: edev->dev_info.common.hw_mac);
902
903 ndev->mtu = edev->dev_info.common.mtu;
904}
905
906/* This function converts from 32b param to two params of level and module
907 * Input 32b decoding:
908 * b31 - enable all NOTICE prints. NOTICE prints are for deviation from the
909 * 'happy' flow, e.g. memory allocation failed.
910 * b30 - enable all INFO prints. INFO prints are for major steps in the flow
911 * and provide important parameters.
912 * b29-b0 - per-module bitmap, where each bit enables VERBOSE prints of that
913 * module. VERBOSE prints are for tracking the specific flow in low level.
914 *
915 * Notice that the level should be that of the lowest required logs.
916 */
917void qede_config_debug(uint debug, u32 *p_dp_module, u8 *p_dp_level)
918{
919 *p_dp_level = QED_LEVEL_NOTICE;
920 *p_dp_module = 0;
921
922 if (debug & QED_LOG_VERBOSE_MASK) {
923 *p_dp_level = QED_LEVEL_VERBOSE;
924 *p_dp_module = (debug & 0x3FFFFFFF);
925 } else if (debug & QED_LOG_INFO_MASK) {
926 *p_dp_level = QED_LEVEL_INFO;
927 } else if (debug & QED_LOG_NOTICE_MASK) {
928 *p_dp_level = QED_LEVEL_NOTICE;
929 }
930}
931
932static void qede_free_fp_array(struct qede_dev *edev)
933{
934 if (edev->fp_array) {
935 struct qede_fastpath *fp;
936 int i;
937
938 for_each_queue(i) {
939 fp = &edev->fp_array[i];
940
941 kfree(objp: fp->sb_info);
942 /* Handle mem alloc failure case where qede_init_fp
943 * didn't register xdp_rxq_info yet.
944 * Implicit only (fp->type & QEDE_FASTPATH_RX)
945 */
946 if (fp->rxq && xdp_rxq_info_is_reg(xdp_rxq: &fp->rxq->xdp_rxq))
947 xdp_rxq_info_unreg(xdp_rxq: &fp->rxq->xdp_rxq);
948 kfree(objp: fp->rxq);
949 kfree(objp: fp->xdp_tx);
950 kfree(objp: fp->txq);
951 }
952 kfree(objp: edev->fp_array);
953 }
954
955 edev->num_queues = 0;
956 edev->fp_num_tx = 0;
957 edev->fp_num_rx = 0;
958}
959
960static int qede_alloc_fp_array(struct qede_dev *edev)
961{
962 u8 fp_combined, fp_rx = edev->fp_num_rx;
963 struct qede_fastpath *fp;
964 int i;
965
966 edev->fp_array = kcalloc(QEDE_QUEUE_CNT(edev),
967 sizeof(*edev->fp_array), GFP_KERNEL);
968 if (!edev->fp_array) {
969 DP_NOTICE(edev, "fp array allocation failed\n");
970 goto err;
971 }
972
973 if (!edev->coal_entry) {
974 edev->coal_entry = kcalloc(QEDE_MAX_RSS_CNT(edev),
975 sizeof(*edev->coal_entry),
976 GFP_KERNEL);
977 if (!edev->coal_entry) {
978 DP_ERR(edev, "coalesce entry allocation failed\n");
979 goto err;
980 }
981 }
982
983 fp_combined = QEDE_QUEUE_CNT(edev) - fp_rx - edev->fp_num_tx;
984
985 /* Allocate the FP elements for Rx queues followed by combined and then
986 * the Tx. This ordering should be maintained so that the respective
987 * queues (Rx or Tx) will be together in the fastpath array and the
988 * associated ids will be sequential.
989 */
990 for_each_queue(i) {
991 fp = &edev->fp_array[i];
992
993 fp->sb_info = kzalloc(sizeof(*fp->sb_info), GFP_KERNEL);
994 if (!fp->sb_info) {
995 DP_NOTICE(edev, "sb info struct allocation failed\n");
996 goto err;
997 }
998
999 if (fp_rx) {
1000 fp->type = QEDE_FASTPATH_RX;
1001 fp_rx--;
1002 } else if (fp_combined) {
1003 fp->type = QEDE_FASTPATH_COMBINED;
1004 fp_combined--;
1005 } else {
1006 fp->type = QEDE_FASTPATH_TX;
1007 }
1008
1009 if (fp->type & QEDE_FASTPATH_TX) {
1010 fp->txq = kcalloc(edev->dev_info.num_tc,
1011 sizeof(*fp->txq), GFP_KERNEL);
1012 if (!fp->txq)
1013 goto err;
1014 }
1015
1016 if (fp->type & QEDE_FASTPATH_RX) {
1017 fp->rxq = kzalloc(sizeof(*fp->rxq), GFP_KERNEL);
1018 if (!fp->rxq)
1019 goto err;
1020
1021 if (edev->xdp_prog) {
1022 fp->xdp_tx = kzalloc(sizeof(*fp->xdp_tx),
1023 GFP_KERNEL);
1024 if (!fp->xdp_tx)
1025 goto err;
1026 fp->type |= QEDE_FASTPATH_XDP;
1027 }
1028 }
1029 }
1030
1031 return 0;
1032err:
1033 qede_free_fp_array(edev);
1034 return -ENOMEM;
1035}
1036
1037/* The qede lock is used to protect driver state change and driver flows that
1038 * are not reentrant.
1039 */
1040void __qede_lock(struct qede_dev *edev)
1041{
1042 mutex_lock(&edev->qede_lock);
1043}
1044
1045void __qede_unlock(struct qede_dev *edev)
1046{
1047 mutex_unlock(lock: &edev->qede_lock);
1048}
1049
1050/* This version of the lock should be used when acquiring the RTNL lock is also
1051 * needed in addition to the internal qede lock.
1052 */
1053static void qede_lock(struct qede_dev *edev)
1054{
1055 rtnl_lock();
1056 __qede_lock(edev);
1057}
1058
1059static void qede_unlock(struct qede_dev *edev)
1060{
1061 __qede_unlock(edev);
1062 rtnl_unlock();
1063}
1064
1065static void qede_periodic_task(struct work_struct *work)
1066{
1067 struct qede_dev *edev = container_of(work, struct qede_dev,
1068 periodic_task.work);
1069
1070 qede_fill_by_demand_stats(edev);
1071 schedule_delayed_work(dwork: &edev->periodic_task, delay: edev->stats_coal_ticks);
1072}
1073
1074static void qede_init_periodic_task(struct qede_dev *edev)
1075{
1076 INIT_DELAYED_WORK(&edev->periodic_task, qede_periodic_task);
1077 spin_lock_init(&edev->stats_lock);
1078 edev->stats_coal_usecs = USEC_PER_SEC;
1079 edev->stats_coal_ticks = usecs_to_jiffies(USEC_PER_SEC);
1080}
1081
1082static void qede_sp_task(struct work_struct *work)
1083{
1084 struct qede_dev *edev = container_of(work, struct qede_dev,
1085 sp_task.work);
1086
1087 /* Disable execution of this deferred work once
1088 * qede removal is in progress, this stop any future
1089 * scheduling of sp_task.
1090 */
1091 if (test_bit(QEDE_SP_DISABLE, &edev->sp_flags))
1092 return;
1093
1094 /* The locking scheme depends on the specific flag:
1095 * In case of QEDE_SP_RECOVERY, acquiring the RTNL lock is required to
1096 * ensure that ongoing flows are ended and new ones are not started.
1097 * In other cases - only the internal qede lock should be acquired.
1098 */
1099
1100 if (test_and_clear_bit(QEDE_SP_RECOVERY, addr: &edev->sp_flags)) {
1101 cancel_delayed_work_sync(dwork: &edev->periodic_task);
1102#ifdef CONFIG_QED_SRIOV
1103 /* SRIOV must be disabled outside the lock to avoid a deadlock.
1104 * The recovery of the active VFs is currently not supported.
1105 */
1106 if (pci_num_vf(dev: edev->pdev))
1107 qede_sriov_configure(pdev: edev->pdev, num_vfs_param: 0);
1108#endif
1109 qede_lock(edev);
1110 qede_recovery_handler(edev);
1111 qede_unlock(edev);
1112 }
1113
1114 __qede_lock(edev);
1115
1116 if (test_and_clear_bit(QEDE_SP_RX_MODE, addr: &edev->sp_flags))
1117 if (edev->state == QEDE_STATE_OPEN)
1118 qede_config_rx_mode(ndev: edev->ndev);
1119
1120#ifdef CONFIG_RFS_ACCEL
1121 if (test_and_clear_bit(QEDE_SP_ARFS_CONFIG, addr: &edev->sp_flags)) {
1122 if (edev->state == QEDE_STATE_OPEN)
1123 qede_process_arfs_filters(edev, free_fltr: false);
1124 }
1125#endif
1126 if (test_and_clear_bit(QEDE_SP_HW_ERR, addr: &edev->sp_flags))
1127 qede_generic_hw_err_handler(edev);
1128 __qede_unlock(edev);
1129
1130 if (test_and_clear_bit(QEDE_SP_AER, addr: &edev->sp_flags)) {
1131#ifdef CONFIG_QED_SRIOV
1132 /* SRIOV must be disabled outside the lock to avoid a deadlock.
1133 * The recovery of the active VFs is currently not supported.
1134 */
1135 if (pci_num_vf(dev: edev->pdev))
1136 qede_sriov_configure(pdev: edev->pdev, num_vfs_param: 0);
1137#endif
1138 edev->ops->common->recovery_process(edev->cdev);
1139 }
1140}
1141
1142static void qede_update_pf_params(struct qed_dev *cdev)
1143{
1144 struct qed_pf_params pf_params;
1145 u16 num_cons;
1146
1147 /* 64 rx + 64 tx + 64 XDP */
1148 memset(&pf_params, 0, sizeof(struct qed_pf_params));
1149
1150 /* 1 rx + 1 xdp + max tx cos */
1151 num_cons = QED_MIN_L2_CONS;
1152
1153 pf_params.eth_pf_params.num_cons = (MAX_SB_PER_PF_MIMD - 1) * num_cons;
1154
1155 /* Same for VFs - make sure they'll have sufficient connections
1156 * to support XDP Tx queues.
1157 */
1158 pf_params.eth_pf_params.num_vf_cons = 48;
1159
1160 pf_params.eth_pf_params.num_arfs_filters = QEDE_RFS_MAX_FLTR;
1161 qed_ops->common->update_pf_params(cdev, &pf_params);
1162}
1163
1164#define QEDE_FW_VER_STR_SIZE 80
1165
1166static void qede_log_probe(struct qede_dev *edev)
1167{
1168 struct qed_dev_info *p_dev_info = &edev->dev_info.common;
1169 u8 buf[QEDE_FW_VER_STR_SIZE];
1170 size_t left_size;
1171
1172 snprintf(buf, QEDE_FW_VER_STR_SIZE,
1173 fmt: "Storm FW %d.%d.%d.%d, Management FW %d.%d.%d.%d",
1174 p_dev_info->fw_major, p_dev_info->fw_minor, p_dev_info->fw_rev,
1175 p_dev_info->fw_eng,
1176 (p_dev_info->mfw_rev & QED_MFW_VERSION_3_MASK) >>
1177 QED_MFW_VERSION_3_OFFSET,
1178 (p_dev_info->mfw_rev & QED_MFW_VERSION_2_MASK) >>
1179 QED_MFW_VERSION_2_OFFSET,
1180 (p_dev_info->mfw_rev & QED_MFW_VERSION_1_MASK) >>
1181 QED_MFW_VERSION_1_OFFSET,
1182 (p_dev_info->mfw_rev & QED_MFW_VERSION_0_MASK) >>
1183 QED_MFW_VERSION_0_OFFSET);
1184
1185 left_size = QEDE_FW_VER_STR_SIZE - strlen(buf);
1186 if (p_dev_info->mbi_version && left_size)
1187 snprintf(buf: buf + strlen(buf), size: left_size,
1188 fmt: " [MBI %d.%d.%d]",
1189 (p_dev_info->mbi_version & QED_MBI_VERSION_2_MASK) >>
1190 QED_MBI_VERSION_2_OFFSET,
1191 (p_dev_info->mbi_version & QED_MBI_VERSION_1_MASK) >>
1192 QED_MBI_VERSION_1_OFFSET,
1193 (p_dev_info->mbi_version & QED_MBI_VERSION_0_MASK) >>
1194 QED_MBI_VERSION_0_OFFSET);
1195
1196 pr_info("qede %02x:%02x.%02x: %s [%s]\n", edev->pdev->bus->number,
1197 PCI_SLOT(edev->pdev->devfn), PCI_FUNC(edev->pdev->devfn),
1198 buf, edev->ndev->name);
1199}
1200
1201enum qede_probe_mode {
1202 QEDE_PROBE_NORMAL,
1203 QEDE_PROBE_RECOVERY,
1204};
1205
1206static int __qede_probe(struct pci_dev *pdev, u32 dp_module, u8 dp_level,
1207 bool is_vf, enum qede_probe_mode mode)
1208{
1209 struct qed_probe_params probe_params;
1210 struct qed_slowpath_params sp_params;
1211 struct qed_dev_eth_info dev_info;
1212 struct qede_dev *edev;
1213 struct qed_dev *cdev;
1214 int rc;
1215
1216 if (unlikely(dp_level & QED_LEVEL_INFO))
1217 pr_notice("Starting qede probe\n");
1218
1219 memset(&probe_params, 0, sizeof(probe_params));
1220 probe_params.protocol = QED_PROTOCOL_ETH;
1221 probe_params.dp_module = dp_module;
1222 probe_params.dp_level = dp_level;
1223 probe_params.is_vf = is_vf;
1224 probe_params.recov_in_prog = (mode == QEDE_PROBE_RECOVERY);
1225 cdev = qed_ops->common->probe(pdev, &probe_params);
1226 if (!cdev) {
1227 rc = -ENODEV;
1228 goto err0;
1229 }
1230
1231 qede_update_pf_params(cdev);
1232
1233 /* Start the Slowpath-process */
1234 memset(&sp_params, 0, sizeof(sp_params));
1235 sp_params.int_mode = QED_INT_MODE_MSIX;
1236 strscpy(sp_params.name, "qede LAN", QED_DRV_VER_STR_SIZE);
1237 rc = qed_ops->common->slowpath_start(cdev, &sp_params);
1238 if (rc) {
1239 pr_notice("Cannot start slowpath\n");
1240 goto err1;
1241 }
1242
1243 /* Learn information crucial for qede to progress */
1244 rc = qed_ops->fill_dev_info(cdev, &dev_info);
1245 if (rc)
1246 goto err2;
1247
1248 if (mode != QEDE_PROBE_RECOVERY) {
1249 edev = qede_alloc_etherdev(cdev, pdev, info: &dev_info, dp_module,
1250 dp_level);
1251 if (!edev) {
1252 rc = -ENOMEM;
1253 goto err2;
1254 }
1255
1256 edev->devlink = qed_ops->common->devlink_register(cdev);
1257 if (IS_ERR(ptr: edev->devlink)) {
1258 DP_NOTICE(edev, "Cannot register devlink\n");
1259 rc = PTR_ERR(ptr: edev->devlink);
1260 edev->devlink = NULL;
1261 goto err3;
1262 }
1263 } else {
1264 struct net_device *ndev = pci_get_drvdata(pdev);
1265 struct qed_devlink *qdl;
1266
1267 edev = netdev_priv(dev: ndev);
1268 qdl = devlink_priv(devlink: edev->devlink);
1269 qdl->cdev = cdev;
1270 edev->cdev = cdev;
1271 memset(&edev->stats, 0, sizeof(edev->stats));
1272 memcpy(&edev->dev_info, &dev_info, sizeof(dev_info));
1273 }
1274
1275 if (is_vf)
1276 set_bit(nr: QEDE_FLAGS_IS_VF, addr: &edev->flags);
1277
1278 qede_init_ndev(edev);
1279
1280 rc = qede_rdma_dev_add(dev: edev, recovery: (mode == QEDE_PROBE_RECOVERY));
1281 if (rc)
1282 goto err3;
1283
1284 if (mode != QEDE_PROBE_RECOVERY) {
1285 /* Prepare the lock prior to the registration of the netdev,
1286 * as once it's registered we might reach flows requiring it
1287 * [it's even possible to reach a flow needing it directly
1288 * from there, although it's unlikely].
1289 */
1290 INIT_DELAYED_WORK(&edev->sp_task, qede_sp_task);
1291 mutex_init(&edev->qede_lock);
1292 qede_init_periodic_task(edev);
1293
1294 rc = register_netdev(dev: edev->ndev);
1295 if (rc) {
1296 DP_NOTICE(edev, "Cannot register net-device\n");
1297 goto err4;
1298 }
1299 }
1300
1301 edev->ops->common->set_name(cdev, edev->ndev->name);
1302
1303 /* PTP not supported on VFs */
1304 if (!is_vf)
1305 qede_ptp_enable(edev);
1306
1307 edev->ops->register_ops(cdev, &qede_ll_ops, edev);
1308
1309#ifdef CONFIG_DCB
1310 if (!IS_VF(edev))
1311 qede_set_dcbnl_ops(ndev: edev->ndev);
1312#endif
1313
1314 edev->rx_copybreak = QEDE_RX_HDR_SIZE;
1315
1316 qede_log_probe(edev);
1317
1318 /* retain user config (for example - after recovery) */
1319 if (edev->stats_coal_usecs)
1320 schedule_delayed_work(dwork: &edev->periodic_task, delay: 0);
1321
1322 return 0;
1323
1324err4:
1325 qede_rdma_dev_remove(dev: edev, recovery: (mode == QEDE_PROBE_RECOVERY));
1326err3:
1327 if (mode != QEDE_PROBE_RECOVERY)
1328 free_netdev(dev: edev->ndev);
1329 else
1330 edev->cdev = NULL;
1331err2:
1332 qed_ops->common->slowpath_stop(cdev);
1333err1:
1334 qed_ops->common->remove(cdev);
1335err0:
1336 return rc;
1337}
1338
1339static int qede_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1340{
1341 bool is_vf = false;
1342 u32 dp_module = 0;
1343 u8 dp_level = 0;
1344
1345 switch ((enum qede_pci_private)id->driver_data) {
1346 case QEDE_PRIVATE_VF:
1347 if (debug & QED_LOG_VERBOSE_MASK)
1348 dev_err(&pdev->dev, "Probing a VF\n");
1349 is_vf = true;
1350 break;
1351 default:
1352 if (debug & QED_LOG_VERBOSE_MASK)
1353 dev_err(&pdev->dev, "Probing a PF\n");
1354 }
1355
1356 qede_config_debug(debug, p_dp_module: &dp_module, p_dp_level: &dp_level);
1357
1358 return __qede_probe(pdev, dp_module, dp_level, is_vf,
1359 mode: QEDE_PROBE_NORMAL);
1360}
1361
1362enum qede_remove_mode {
1363 QEDE_REMOVE_NORMAL,
1364 QEDE_REMOVE_RECOVERY,
1365};
1366
1367static void __qede_remove(struct pci_dev *pdev, enum qede_remove_mode mode)
1368{
1369 struct net_device *ndev = pci_get_drvdata(pdev);
1370 struct qede_dev *edev;
1371 struct qed_dev *cdev;
1372
1373 if (!ndev) {
1374 dev_info(&pdev->dev, "Device has already been removed\n");
1375 return;
1376 }
1377
1378 edev = netdev_priv(dev: ndev);
1379 cdev = edev->cdev;
1380
1381 DP_INFO(edev, "Starting qede_remove\n");
1382
1383 qede_rdma_dev_remove(dev: edev, recovery: (mode == QEDE_REMOVE_RECOVERY));
1384
1385 if (mode != QEDE_REMOVE_RECOVERY) {
1386 set_bit(QEDE_SP_DISABLE, addr: &edev->sp_flags);
1387 unregister_netdev(dev: ndev);
1388
1389 cancel_delayed_work_sync(dwork: &edev->sp_task);
1390 cancel_delayed_work_sync(dwork: &edev->periodic_task);
1391
1392 edev->ops->common->set_power_state(cdev, PCI_D0);
1393
1394 pci_set_drvdata(pdev, NULL);
1395 }
1396
1397 qede_ptp_disable(edev);
1398
1399 /* Use global ops since we've freed edev */
1400 qed_ops->common->slowpath_stop(cdev);
1401 if (system_state == SYSTEM_POWER_OFF)
1402 return;
1403
1404 if (mode != QEDE_REMOVE_RECOVERY && edev->devlink) {
1405 qed_ops->common->devlink_unregister(edev->devlink);
1406 edev->devlink = NULL;
1407 }
1408 qed_ops->common->remove(cdev);
1409 edev->cdev = NULL;
1410
1411 /* Since this can happen out-of-sync with other flows,
1412 * don't release the netdevice until after slowpath stop
1413 * has been called to guarantee various other contexts
1414 * [e.g., QED register callbacks] won't break anything when
1415 * accessing the netdevice.
1416 */
1417 if (mode != QEDE_REMOVE_RECOVERY) {
1418 kfree(objp: edev->coal_entry);
1419 free_netdev(dev: ndev);
1420 }
1421
1422 dev_info(&pdev->dev, "Ending qede_remove successfully\n");
1423}
1424
1425static void qede_remove(struct pci_dev *pdev)
1426{
1427 __qede_remove(pdev, mode: QEDE_REMOVE_NORMAL);
1428}
1429
1430static void qede_shutdown(struct pci_dev *pdev)
1431{
1432 __qede_remove(pdev, mode: QEDE_REMOVE_NORMAL);
1433}
1434
1435/* -------------------------------------------------------------------------
1436 * START OF LOAD / UNLOAD
1437 * -------------------------------------------------------------------------
1438 */
1439
1440static int qede_set_num_queues(struct qede_dev *edev)
1441{
1442 int rc;
1443 u16 rss_num;
1444
1445 /* Setup queues according to possible resources*/
1446 if (edev->req_queues)
1447 rss_num = edev->req_queues;
1448 else
1449 rss_num = netif_get_num_default_rss_queues() *
1450 edev->dev_info.common.num_hwfns;
1451
1452 rss_num = min_t(u16, QEDE_MAX_RSS_CNT(edev), rss_num);
1453
1454 rc = edev->ops->common->set_fp_int(edev->cdev, rss_num);
1455 if (rc > 0) {
1456 /* Managed to request interrupts for our queues */
1457 edev->num_queues = rc;
1458 DP_INFO(edev, "Managed %d [of %d] RSS queues\n",
1459 QEDE_QUEUE_CNT(edev), rss_num);
1460 rc = 0;
1461 }
1462
1463 edev->fp_num_tx = edev->req_num_tx;
1464 edev->fp_num_rx = edev->req_num_rx;
1465
1466 return rc;
1467}
1468
1469static void qede_free_mem_sb(struct qede_dev *edev, struct qed_sb_info *sb_info,
1470 u16 sb_id)
1471{
1472 if (sb_info->sb_virt) {
1473 edev->ops->common->sb_release(edev->cdev, sb_info, sb_id,
1474 QED_SB_TYPE_L2_QUEUE);
1475 dma_free_coherent(dev: &edev->pdev->dev, size: sizeof(*sb_info->sb_virt),
1476 cpu_addr: (void *)sb_info->sb_virt, dma_handle: sb_info->sb_phys);
1477 memset(sb_info, 0, sizeof(*sb_info));
1478 }
1479}
1480
1481/* This function allocates fast-path status block memory */
1482static int qede_alloc_mem_sb(struct qede_dev *edev,
1483 struct qed_sb_info *sb_info, u16 sb_id)
1484{
1485 struct status_block *sb_virt;
1486 dma_addr_t sb_phys;
1487 int rc;
1488
1489 sb_virt = dma_alloc_coherent(dev: &edev->pdev->dev,
1490 size: sizeof(*sb_virt), dma_handle: &sb_phys, GFP_KERNEL);
1491 if (!sb_virt) {
1492 DP_ERR(edev, "Status block allocation failed\n");
1493 return -ENOMEM;
1494 }
1495
1496 rc = edev->ops->common->sb_init(edev->cdev, sb_info,
1497 sb_virt, sb_phys, sb_id,
1498 QED_SB_TYPE_L2_QUEUE);
1499 if (rc) {
1500 DP_ERR(edev, "Status block initialization failed\n");
1501 dma_free_coherent(dev: &edev->pdev->dev, size: sizeof(*sb_virt),
1502 cpu_addr: sb_virt, dma_handle: sb_phys);
1503 return rc;
1504 }
1505
1506 return 0;
1507}
1508
1509static void qede_free_rx_buffers(struct qede_dev *edev,
1510 struct qede_rx_queue *rxq)
1511{
1512 u16 i;
1513
1514 for (i = rxq->sw_rx_cons; i != rxq->sw_rx_prod; i++) {
1515 struct sw_rx_data *rx_buf;
1516 struct page *data;
1517
1518 rx_buf = &rxq->sw_rx_ring[i & NUM_RX_BDS_MAX];
1519 data = rx_buf->data;
1520
1521 dma_unmap_page(&edev->pdev->dev,
1522 rx_buf->mapping, PAGE_SIZE, rxq->data_direction);
1523
1524 rx_buf->data = NULL;
1525 __free_page(data);
1526 }
1527}
1528
1529static void qede_free_mem_rxq(struct qede_dev *edev, struct qede_rx_queue *rxq)
1530{
1531 /* Free rx buffers */
1532 qede_free_rx_buffers(edev, rxq);
1533
1534 /* Free the parallel SW ring */
1535 kfree(objp: rxq->sw_rx_ring);
1536
1537 /* Free the real RQ ring used by FW */
1538 edev->ops->common->chain_free(edev->cdev, &rxq->rx_bd_ring);
1539 edev->ops->common->chain_free(edev->cdev, &rxq->rx_comp_ring);
1540}
1541
1542static void qede_set_tpa_param(struct qede_rx_queue *rxq)
1543{
1544 int i;
1545
1546 for (i = 0; i < ETH_TPA_MAX_AGGS_NUM; i++) {
1547 struct qede_agg_info *tpa_info = &rxq->tpa_info[i];
1548
1549 tpa_info->state = QEDE_AGG_STATE_NONE;
1550 }
1551}
1552
1553/* This function allocates all memory needed per Rx queue */
1554static int qede_alloc_mem_rxq(struct qede_dev *edev, struct qede_rx_queue *rxq)
1555{
1556 struct qed_chain_init_params params = {
1557 .cnt_type = QED_CHAIN_CNT_TYPE_U16,
1558 .num_elems = RX_RING_SIZE,
1559 };
1560 struct qed_dev *cdev = edev->cdev;
1561 int i, rc, size;
1562
1563 rxq->num_rx_buffers = edev->q_num_rx_buffers;
1564
1565 rxq->rx_buf_size = NET_IP_ALIGN + ETH_OVERHEAD + edev->ndev->mtu;
1566
1567 rxq->rx_headroom = edev->xdp_prog ? XDP_PACKET_HEADROOM : NET_SKB_PAD;
1568 size = rxq->rx_headroom +
1569 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1570
1571 /* Make sure that the headroom and payload fit in a single page */
1572 if (rxq->rx_buf_size + size > PAGE_SIZE)
1573 rxq->rx_buf_size = PAGE_SIZE - size;
1574
1575 /* Segment size to split a page in multiple equal parts,
1576 * unless XDP is used in which case we'd use the entire page.
1577 */
1578 if (!edev->xdp_prog) {
1579 size = size + rxq->rx_buf_size;
1580 rxq->rx_buf_seg_size = roundup_pow_of_two(size);
1581 } else {
1582 rxq->rx_buf_seg_size = PAGE_SIZE;
1583 edev->ndev->features &= ~NETIF_F_GRO_HW;
1584 }
1585
1586 /* Allocate the parallel driver ring for Rx buffers */
1587 size = sizeof(*rxq->sw_rx_ring) * RX_RING_SIZE;
1588 rxq->sw_rx_ring = kzalloc(size, GFP_KERNEL);
1589 if (!rxq->sw_rx_ring) {
1590 DP_ERR(edev, "Rx buffers ring allocation failed\n");
1591 rc = -ENOMEM;
1592 goto err;
1593 }
1594
1595 /* Allocate FW Rx ring */
1596 params.mode = QED_CHAIN_MODE_NEXT_PTR;
1597 params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE;
1598 params.elem_size = sizeof(struct eth_rx_bd);
1599
1600 rc = edev->ops->common->chain_alloc(cdev, &rxq->rx_bd_ring, &params);
1601 if (rc)
1602 goto err;
1603
1604 /* Allocate FW completion ring */
1605 params.mode = QED_CHAIN_MODE_PBL;
1606 params.intended_use = QED_CHAIN_USE_TO_CONSUME;
1607 params.elem_size = sizeof(union eth_rx_cqe);
1608
1609 rc = edev->ops->common->chain_alloc(cdev, &rxq->rx_comp_ring, &params);
1610 if (rc)
1611 goto err;
1612
1613 /* Allocate buffers for the Rx ring */
1614 rxq->filled_buffers = 0;
1615 for (i = 0; i < rxq->num_rx_buffers; i++) {
1616 rc = qede_alloc_rx_buffer(rxq, allow_lazy: false);
1617 if (rc) {
1618 DP_ERR(edev,
1619 "Rx buffers allocation failed at index %d\n", i);
1620 goto err;
1621 }
1622 }
1623
1624 edev->gro_disable = !(edev->ndev->features & NETIF_F_GRO_HW);
1625 if (!edev->gro_disable)
1626 qede_set_tpa_param(rxq);
1627err:
1628 return rc;
1629}
1630
1631static void qede_free_mem_txq(struct qede_dev *edev, struct qede_tx_queue *txq)
1632{
1633 /* Free the parallel SW ring */
1634 if (txq->is_xdp)
1635 kfree(objp: txq->sw_tx_ring.xdp);
1636 else
1637 kfree(objp: txq->sw_tx_ring.skbs);
1638
1639 /* Free the real RQ ring used by FW */
1640 edev->ops->common->chain_free(edev->cdev, &txq->tx_pbl);
1641}
1642
1643/* This function allocates all memory needed per Tx queue */
1644static int qede_alloc_mem_txq(struct qede_dev *edev, struct qede_tx_queue *txq)
1645{
1646 struct qed_chain_init_params params = {
1647 .mode = QED_CHAIN_MODE_PBL,
1648 .intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1649 .cnt_type = QED_CHAIN_CNT_TYPE_U16,
1650 .num_elems = edev->q_num_tx_buffers,
1651 .elem_size = sizeof(union eth_tx_bd_types),
1652 };
1653 int size, rc;
1654
1655 txq->num_tx_buffers = edev->q_num_tx_buffers;
1656
1657 /* Allocate the parallel driver ring for Tx buffers */
1658 if (txq->is_xdp) {
1659 size = sizeof(*txq->sw_tx_ring.xdp) * txq->num_tx_buffers;
1660 txq->sw_tx_ring.xdp = kzalloc(size, GFP_KERNEL);
1661 if (!txq->sw_tx_ring.xdp)
1662 goto err;
1663 } else {
1664 size = sizeof(*txq->sw_tx_ring.skbs) * txq->num_tx_buffers;
1665 txq->sw_tx_ring.skbs = kzalloc(size, GFP_KERNEL);
1666 if (!txq->sw_tx_ring.skbs)
1667 goto err;
1668 }
1669
1670 rc = edev->ops->common->chain_alloc(edev->cdev, &txq->tx_pbl, &params);
1671 if (rc)
1672 goto err;
1673
1674 return 0;
1675
1676err:
1677 qede_free_mem_txq(edev, txq);
1678 return -ENOMEM;
1679}
1680
1681/* This function frees all memory of a single fp */
1682static void qede_free_mem_fp(struct qede_dev *edev, struct qede_fastpath *fp)
1683{
1684 qede_free_mem_sb(edev, sb_info: fp->sb_info, sb_id: fp->id);
1685
1686 if (fp->type & QEDE_FASTPATH_RX)
1687 qede_free_mem_rxq(edev, rxq: fp->rxq);
1688
1689 if (fp->type & QEDE_FASTPATH_XDP)
1690 qede_free_mem_txq(edev, txq: fp->xdp_tx);
1691
1692 if (fp->type & QEDE_FASTPATH_TX) {
1693 int cos;
1694
1695 for_each_cos_in_txq(edev, cos)
1696 qede_free_mem_txq(edev, txq: &fp->txq[cos]);
1697 }
1698}
1699
1700/* This function allocates all memory needed for a single fp (i.e. an entity
1701 * which contains status block, one rx queue and/or multiple per-TC tx queues.
1702 */
1703static int qede_alloc_mem_fp(struct qede_dev *edev, struct qede_fastpath *fp)
1704{
1705 int rc = 0;
1706
1707 rc = qede_alloc_mem_sb(edev, sb_info: fp->sb_info, sb_id: fp->id);
1708 if (rc)
1709 goto out;
1710
1711 if (fp->type & QEDE_FASTPATH_RX) {
1712 rc = qede_alloc_mem_rxq(edev, rxq: fp->rxq);
1713 if (rc)
1714 goto out;
1715 }
1716
1717 if (fp->type & QEDE_FASTPATH_XDP) {
1718 rc = qede_alloc_mem_txq(edev, txq: fp->xdp_tx);
1719 if (rc)
1720 goto out;
1721 }
1722
1723 if (fp->type & QEDE_FASTPATH_TX) {
1724 int cos;
1725
1726 for_each_cos_in_txq(edev, cos) {
1727 rc = qede_alloc_mem_txq(edev, txq: &fp->txq[cos]);
1728 if (rc)
1729 goto out;
1730 }
1731 }
1732
1733out:
1734 return rc;
1735}
1736
1737static void qede_free_mem_load(struct qede_dev *edev)
1738{
1739 int i;
1740
1741 for_each_queue(i) {
1742 struct qede_fastpath *fp = &edev->fp_array[i];
1743
1744 qede_free_mem_fp(edev, fp);
1745 }
1746}
1747
1748/* This function allocates all qede memory at NIC load. */
1749static int qede_alloc_mem_load(struct qede_dev *edev)
1750{
1751 int rc = 0, queue_id;
1752
1753 for (queue_id = 0; queue_id < QEDE_QUEUE_CNT(edev); queue_id++) {
1754 struct qede_fastpath *fp = &edev->fp_array[queue_id];
1755
1756 rc = qede_alloc_mem_fp(edev, fp);
1757 if (rc) {
1758 DP_ERR(edev,
1759 "Failed to allocate memory for fastpath - rss id = %d\n",
1760 queue_id);
1761 qede_free_mem_load(edev);
1762 return rc;
1763 }
1764 }
1765
1766 return 0;
1767}
1768
1769static void qede_empty_tx_queue(struct qede_dev *edev,
1770 struct qede_tx_queue *txq)
1771{
1772 unsigned int pkts_compl = 0, bytes_compl = 0;
1773 struct netdev_queue *netdev_txq;
1774 int rc, len = 0;
1775
1776 netdev_txq = netdev_get_tx_queue(dev: edev->ndev, index: txq->ndev_txq_id);
1777
1778 while (qed_chain_get_cons_idx(chain: &txq->tx_pbl) !=
1779 qed_chain_get_prod_idx(chain: &txq->tx_pbl)) {
1780 DP_VERBOSE(edev, NETIF_MSG_IFDOWN,
1781 "Freeing a packet on tx queue[%d]: chain_cons 0x%x, chain_prod 0x%x\n",
1782 txq->index, qed_chain_get_cons_idx(&txq->tx_pbl),
1783 qed_chain_get_prod_idx(&txq->tx_pbl));
1784
1785 rc = qede_free_tx_pkt(edev, txq, len: &len);
1786 if (rc) {
1787 DP_NOTICE(edev,
1788 "Failed to free a packet on tx queue[%d]: chain_cons 0x%x, chain_prod 0x%x\n",
1789 txq->index,
1790 qed_chain_get_cons_idx(&txq->tx_pbl),
1791 qed_chain_get_prod_idx(&txq->tx_pbl));
1792 break;
1793 }
1794
1795 bytes_compl += len;
1796 pkts_compl++;
1797 txq->sw_tx_cons++;
1798 }
1799
1800 netdev_tx_completed_queue(dev_queue: netdev_txq, pkts: pkts_compl, bytes: bytes_compl);
1801}
1802
1803static void qede_empty_tx_queues(struct qede_dev *edev)
1804{
1805 int i;
1806
1807 for_each_queue(i)
1808 if (edev->fp_array[i].type & QEDE_FASTPATH_TX) {
1809 int cos;
1810
1811 for_each_cos_in_txq(edev, cos) {
1812 struct qede_fastpath *fp;
1813
1814 fp = &edev->fp_array[i];
1815 qede_empty_tx_queue(edev,
1816 txq: &fp->txq[cos]);
1817 }
1818 }
1819}
1820
1821/* This function inits fp content and resets the SB, RXQ and TXQ structures */
1822static void qede_init_fp(struct qede_dev *edev)
1823{
1824 int queue_id, rxq_index = 0, txq_index = 0;
1825 struct qede_fastpath *fp;
1826 bool init_xdp = false;
1827
1828 for_each_queue(queue_id) {
1829 fp = &edev->fp_array[queue_id];
1830
1831 fp->edev = edev;
1832 fp->id = queue_id;
1833
1834 if (fp->type & QEDE_FASTPATH_XDP) {
1835 fp->xdp_tx->index = QEDE_TXQ_IDX_TO_XDP(edev,
1836 rxq_index);
1837 fp->xdp_tx->is_xdp = 1;
1838
1839 spin_lock_init(&fp->xdp_tx->xdp_tx_lock);
1840 init_xdp = true;
1841 }
1842
1843 if (fp->type & QEDE_FASTPATH_RX) {
1844 fp->rxq->rxq_id = rxq_index++;
1845
1846 /* Determine how to map buffers for this queue */
1847 if (fp->type & QEDE_FASTPATH_XDP)
1848 fp->rxq->data_direction = DMA_BIDIRECTIONAL;
1849 else
1850 fp->rxq->data_direction = DMA_FROM_DEVICE;
1851 fp->rxq->dev = &edev->pdev->dev;
1852
1853 /* Driver have no error path from here */
1854 WARN_ON(xdp_rxq_info_reg(&fp->rxq->xdp_rxq, edev->ndev,
1855 fp->rxq->rxq_id, 0) < 0);
1856
1857 if (xdp_rxq_info_reg_mem_model(xdp_rxq: &fp->rxq->xdp_rxq,
1858 type: MEM_TYPE_PAGE_ORDER0,
1859 NULL)) {
1860 DP_NOTICE(edev,
1861 "Failed to register XDP memory model\n");
1862 }
1863 }
1864
1865 if (fp->type & QEDE_FASTPATH_TX) {
1866 int cos;
1867
1868 for_each_cos_in_txq(edev, cos) {
1869 struct qede_tx_queue *txq = &fp->txq[cos];
1870 u16 ndev_tx_id;
1871
1872 txq->cos = cos;
1873 txq->index = txq_index;
1874 ndev_tx_id = QEDE_TXQ_TO_NDEV_TXQ_ID(edev, txq);
1875 txq->ndev_txq_id = ndev_tx_id;
1876
1877 if (edev->dev_info.is_legacy)
1878 txq->is_legacy = true;
1879 txq->dev = &edev->pdev->dev;
1880 }
1881
1882 txq_index++;
1883 }
1884
1885 snprintf(buf: fp->name, size: sizeof(fp->name), fmt: "%s-fp-%d",
1886 edev->ndev->name, queue_id);
1887 }
1888
1889 if (init_xdp) {
1890 edev->total_xdp_queues = QEDE_RSS_COUNT(edev);
1891 DP_INFO(edev, "Total XDP queues: %u\n", edev->total_xdp_queues);
1892 }
1893}
1894
1895static int qede_set_real_num_queues(struct qede_dev *edev)
1896{
1897 int rc = 0;
1898
1899 rc = netif_set_real_num_tx_queues(dev: edev->ndev,
1900 QEDE_TSS_COUNT(edev) *
1901 edev->dev_info.num_tc);
1902 if (rc) {
1903 DP_NOTICE(edev, "Failed to set real number of Tx queues\n");
1904 return rc;
1905 }
1906
1907 rc = netif_set_real_num_rx_queues(dev: edev->ndev, QEDE_RSS_COUNT(edev));
1908 if (rc) {
1909 DP_NOTICE(edev, "Failed to set real number of Rx queues\n");
1910 return rc;
1911 }
1912
1913 return 0;
1914}
1915
1916static void qede_napi_disable_remove(struct qede_dev *edev)
1917{
1918 int i;
1919
1920 for_each_queue(i) {
1921 napi_disable(n: &edev->fp_array[i].napi);
1922
1923 netif_napi_del(napi: &edev->fp_array[i].napi);
1924 }
1925}
1926
1927static void qede_napi_add_enable(struct qede_dev *edev)
1928{
1929 int i;
1930
1931 /* Add NAPI objects */
1932 for_each_queue(i) {
1933 netif_napi_add(dev: edev->ndev, napi: &edev->fp_array[i].napi, poll: qede_poll);
1934 napi_enable(n: &edev->fp_array[i].napi);
1935 }
1936}
1937
1938static void qede_sync_free_irqs(struct qede_dev *edev)
1939{
1940 int i;
1941
1942 for (i = 0; i < edev->int_info.used_cnt; i++) {
1943 if (edev->int_info.msix_cnt) {
1944 free_irq(edev->int_info.msix[i].vector,
1945 &edev->fp_array[i]);
1946 } else {
1947 edev->ops->common->simd_handler_clean(edev->cdev, i);
1948 }
1949 }
1950
1951 edev->int_info.used_cnt = 0;
1952 edev->int_info.msix_cnt = 0;
1953}
1954
1955static int qede_req_msix_irqs(struct qede_dev *edev)
1956{
1957 int i, rc;
1958
1959 /* Sanitize number of interrupts == number of prepared RSS queues */
1960 if (QEDE_QUEUE_CNT(edev) > edev->int_info.msix_cnt) {
1961 DP_ERR(edev,
1962 "Interrupt mismatch: %d RSS queues > %d MSI-x vectors\n",
1963 QEDE_QUEUE_CNT(edev), edev->int_info.msix_cnt);
1964 return -EINVAL;
1965 }
1966
1967 for (i = 0; i < QEDE_QUEUE_CNT(edev); i++) {
1968#ifdef CONFIG_RFS_ACCEL
1969 struct qede_fastpath *fp = &edev->fp_array[i];
1970
1971 if (edev->ndev->rx_cpu_rmap && (fp->type & QEDE_FASTPATH_RX)) {
1972 rc = irq_cpu_rmap_add(rmap: edev->ndev->rx_cpu_rmap,
1973 irq: edev->int_info.msix[i].vector);
1974 if (rc) {
1975 DP_ERR(edev, "Failed to add CPU rmap\n");
1976 qede_free_arfs(edev);
1977 }
1978 }
1979#endif
1980 rc = request_irq(irq: edev->int_info.msix[i].vector,
1981 handler: qede_msix_fp_int, flags: 0, name: edev->fp_array[i].name,
1982 dev: &edev->fp_array[i]);
1983 if (rc) {
1984 DP_ERR(edev, "Request fp %d irq failed\n", i);
1985#ifdef CONFIG_RFS_ACCEL
1986 if (edev->ndev->rx_cpu_rmap)
1987 free_irq_cpu_rmap(rmap: edev->ndev->rx_cpu_rmap);
1988
1989 edev->ndev->rx_cpu_rmap = NULL;
1990#endif
1991 qede_sync_free_irqs(edev);
1992 return rc;
1993 }
1994 DP_VERBOSE(edev, NETIF_MSG_INTR,
1995 "Requested fp irq for %s [entry %d]. Cookie is at %p\n",
1996 edev->fp_array[i].name, i,
1997 &edev->fp_array[i]);
1998 edev->int_info.used_cnt++;
1999 }
2000
2001 return 0;
2002}
2003
2004static void qede_simd_fp_handler(void *cookie)
2005{
2006 struct qede_fastpath *fp = (struct qede_fastpath *)cookie;
2007
2008 napi_schedule_irqoff(n: &fp->napi);
2009}
2010
2011static int qede_setup_irqs(struct qede_dev *edev)
2012{
2013 int i, rc = 0;
2014
2015 /* Learn Interrupt configuration */
2016 rc = edev->ops->common->get_fp_int(edev->cdev, &edev->int_info);
2017 if (rc)
2018 return rc;
2019
2020 if (edev->int_info.msix_cnt) {
2021 rc = qede_req_msix_irqs(edev);
2022 if (rc)
2023 return rc;
2024 edev->ndev->irq = edev->int_info.msix[0].vector;
2025 } else {
2026 const struct qed_common_ops *ops;
2027
2028 /* qed should learn receive the RSS ids and callbacks */
2029 ops = edev->ops->common;
2030 for (i = 0; i < QEDE_QUEUE_CNT(edev); i++)
2031 ops->simd_handler_config(edev->cdev,
2032 &edev->fp_array[i], i,
2033 qede_simd_fp_handler);
2034 edev->int_info.used_cnt = QEDE_QUEUE_CNT(edev);
2035 }
2036 return 0;
2037}
2038
2039static int qede_drain_txq(struct qede_dev *edev,
2040 struct qede_tx_queue *txq, bool allow_drain)
2041{
2042 int rc, cnt = 1000;
2043
2044 while (txq->sw_tx_cons != txq->sw_tx_prod) {
2045 if (!cnt) {
2046 if (allow_drain) {
2047 DP_NOTICE(edev,
2048 "Tx queue[%d] is stuck, requesting MCP to drain\n",
2049 txq->index);
2050 rc = edev->ops->common->drain(edev->cdev);
2051 if (rc)
2052 return rc;
2053 return qede_drain_txq(edev, txq, allow_drain: false);
2054 }
2055 DP_NOTICE(edev,
2056 "Timeout waiting for tx queue[%d]: PROD=%d, CONS=%d\n",
2057 txq->index, txq->sw_tx_prod,
2058 txq->sw_tx_cons);
2059 return -ENODEV;
2060 }
2061 cnt--;
2062 usleep_range(min: 1000, max: 2000);
2063 barrier();
2064 }
2065
2066 /* FW finished processing, wait for HW to transmit all tx packets */
2067 usleep_range(min: 1000, max: 2000);
2068
2069 return 0;
2070}
2071
2072static int qede_stop_txq(struct qede_dev *edev,
2073 struct qede_tx_queue *txq, int rss_id)
2074{
2075 /* delete doorbell from doorbell recovery mechanism */
2076 edev->ops->common->db_recovery_del(edev->cdev, txq->doorbell_addr,
2077 &txq->tx_db);
2078
2079 return edev->ops->q_tx_stop(edev->cdev, rss_id, txq->handle);
2080}
2081
2082static int qede_stop_queues(struct qede_dev *edev)
2083{
2084 struct qed_update_vport_params *vport_update_params;
2085 struct qed_dev *cdev = edev->cdev;
2086 struct qede_fastpath *fp;
2087 int rc, i;
2088
2089 /* Disable the vport */
2090 vport_update_params = vzalloc(sizeof(*vport_update_params));
2091 if (!vport_update_params)
2092 return -ENOMEM;
2093
2094 vport_update_params->vport_id = 0;
2095 vport_update_params->update_vport_active_flg = 1;
2096 vport_update_params->vport_active_flg = 0;
2097 vport_update_params->update_rss_flg = 0;
2098
2099 rc = edev->ops->vport_update(cdev, vport_update_params);
2100 vfree(addr: vport_update_params);
2101
2102 if (rc) {
2103 DP_ERR(edev, "Failed to update vport\n");
2104 return rc;
2105 }
2106
2107 /* Flush Tx queues. If needed, request drain from MCP */
2108 for_each_queue(i) {
2109 fp = &edev->fp_array[i];
2110
2111 if (fp->type & QEDE_FASTPATH_TX) {
2112 int cos;
2113
2114 for_each_cos_in_txq(edev, cos) {
2115 rc = qede_drain_txq(edev, txq: &fp->txq[cos], allow_drain: true);
2116 if (rc)
2117 return rc;
2118 }
2119 }
2120
2121 if (fp->type & QEDE_FASTPATH_XDP) {
2122 rc = qede_drain_txq(edev, txq: fp->xdp_tx, allow_drain: true);
2123 if (rc)
2124 return rc;
2125 }
2126 }
2127
2128 /* Stop all Queues in reverse order */
2129 for (i = QEDE_QUEUE_CNT(edev) - 1; i >= 0; i--) {
2130 fp = &edev->fp_array[i];
2131
2132 /* Stop the Tx Queue(s) */
2133 if (fp->type & QEDE_FASTPATH_TX) {
2134 int cos;
2135
2136 for_each_cos_in_txq(edev, cos) {
2137 rc = qede_stop_txq(edev, txq: &fp->txq[cos], rss_id: i);
2138 if (rc)
2139 return rc;
2140 }
2141 }
2142
2143 /* Stop the Rx Queue */
2144 if (fp->type & QEDE_FASTPATH_RX) {
2145 rc = edev->ops->q_rx_stop(cdev, i, fp->rxq->handle);
2146 if (rc) {
2147 DP_ERR(edev, "Failed to stop RXQ #%d\n", i);
2148 return rc;
2149 }
2150 }
2151
2152 /* Stop the XDP forwarding queue */
2153 if (fp->type & QEDE_FASTPATH_XDP) {
2154 rc = qede_stop_txq(edev, txq: fp->xdp_tx, rss_id: i);
2155 if (rc)
2156 return rc;
2157
2158 bpf_prog_put(prog: fp->rxq->xdp_prog);
2159 }
2160 }
2161
2162 /* Stop the vport */
2163 rc = edev->ops->vport_stop(cdev, 0);
2164 if (rc)
2165 DP_ERR(edev, "Failed to stop VPORT\n");
2166
2167 return rc;
2168}
2169
2170static int qede_start_txq(struct qede_dev *edev,
2171 struct qede_fastpath *fp,
2172 struct qede_tx_queue *txq, u8 rss_id, u16 sb_idx)
2173{
2174 dma_addr_t phys_table = qed_chain_get_pbl_phys(chain: &txq->tx_pbl);
2175 u32 page_cnt = qed_chain_get_page_cnt(chain: &txq->tx_pbl);
2176 struct qed_queue_start_common_params params;
2177 struct qed_txq_start_ret_params ret_params;
2178 int rc;
2179
2180 memset(&params, 0, sizeof(params));
2181 memset(&ret_params, 0, sizeof(ret_params));
2182
2183 /* Let the XDP queue share the queue-zone with one of the regular txq.
2184 * We don't really care about its coalescing.
2185 */
2186 if (txq->is_xdp)
2187 params.queue_id = QEDE_TXQ_XDP_TO_IDX(edev, txq);
2188 else
2189 params.queue_id = txq->index;
2190
2191 params.p_sb = fp->sb_info;
2192 params.sb_idx = sb_idx;
2193 params.tc = txq->cos;
2194
2195 rc = edev->ops->q_tx_start(edev->cdev, rss_id, &params, phys_table,
2196 page_cnt, &ret_params);
2197 if (rc) {
2198 DP_ERR(edev, "Start TXQ #%d failed %d\n", txq->index, rc);
2199 return rc;
2200 }
2201
2202 txq->doorbell_addr = ret_params.p_doorbell;
2203 txq->handle = ret_params.p_handle;
2204
2205 /* Determine the FW consumer address associated */
2206 txq->hw_cons_ptr = &fp->sb_info->sb_virt->pi_array[sb_idx];
2207
2208 /* Prepare the doorbell parameters */
2209 SET_FIELD(txq->tx_db.data.params, ETH_DB_DATA_DEST, DB_DEST_XCM);
2210 SET_FIELD(txq->tx_db.data.params, ETH_DB_DATA_AGG_CMD, DB_AGG_CMD_SET);
2211 SET_FIELD(txq->tx_db.data.params, ETH_DB_DATA_AGG_VAL_SEL,
2212 DQ_XCM_ETH_TX_BD_PROD_CMD);
2213 txq->tx_db.data.agg_flags = DQ_XCM_ETH_DQ_CF_CMD;
2214
2215 /* register doorbell with doorbell recovery mechanism */
2216 rc = edev->ops->common->db_recovery_add(edev->cdev, txq->doorbell_addr,
2217 &txq->tx_db, DB_REC_WIDTH_32B,
2218 DB_REC_KERNEL);
2219
2220 return rc;
2221}
2222
2223static int qede_start_queues(struct qede_dev *edev, bool clear_stats)
2224{
2225 int vlan_removal_en = 1;
2226 struct qed_dev *cdev = edev->cdev;
2227 struct qed_dev_info *qed_info = &edev->dev_info.common;
2228 struct qed_update_vport_params *vport_update_params;
2229 struct qed_queue_start_common_params q_params;
2230 struct qed_start_vport_params start = {0};
2231 int rc, i;
2232
2233 if (!edev->num_queues) {
2234 DP_ERR(edev,
2235 "Cannot update V-VPORT as active as there are no Rx queues\n");
2236 return -EINVAL;
2237 }
2238
2239 vport_update_params = vzalloc(sizeof(*vport_update_params));
2240 if (!vport_update_params)
2241 return -ENOMEM;
2242
2243 start.handle_ptp_pkts = !!(edev->ptp);
2244 start.gro_enable = !edev->gro_disable;
2245 start.mtu = edev->ndev->mtu;
2246 start.vport_id = 0;
2247 start.drop_ttl0 = true;
2248 start.remove_inner_vlan = vlan_removal_en;
2249 start.clear_stats = clear_stats;
2250
2251 rc = edev->ops->vport_start(cdev, &start);
2252
2253 if (rc) {
2254 DP_ERR(edev, "Start V-PORT failed %d\n", rc);
2255 goto out;
2256 }
2257
2258 DP_VERBOSE(edev, NETIF_MSG_IFUP,
2259 "Start vport ramrod passed, vport_id = %d, MTU = %d, vlan_removal_en = %d\n",
2260 start.vport_id, edev->ndev->mtu + 0xe, vlan_removal_en);
2261
2262 for_each_queue(i) {
2263 struct qede_fastpath *fp = &edev->fp_array[i];
2264 dma_addr_t p_phys_table;
2265 u32 page_cnt;
2266
2267 if (fp->type & QEDE_FASTPATH_RX) {
2268 struct qed_rxq_start_ret_params ret_params;
2269 struct qede_rx_queue *rxq = fp->rxq;
2270 __le16 *val;
2271
2272 memset(&ret_params, 0, sizeof(ret_params));
2273 memset(&q_params, 0, sizeof(q_params));
2274 q_params.queue_id = rxq->rxq_id;
2275 q_params.vport_id = 0;
2276 q_params.p_sb = fp->sb_info;
2277 q_params.sb_idx = RX_PI;
2278
2279 p_phys_table =
2280 qed_chain_get_pbl_phys(chain: &rxq->rx_comp_ring);
2281 page_cnt = qed_chain_get_page_cnt(chain: &rxq->rx_comp_ring);
2282
2283 rc = edev->ops->q_rx_start(cdev, i, &q_params,
2284 rxq->rx_buf_size,
2285 rxq->rx_bd_ring.p_phys_addr,
2286 p_phys_table,
2287 page_cnt, &ret_params);
2288 if (rc) {
2289 DP_ERR(edev, "Start RXQ #%d failed %d\n", i,
2290 rc);
2291 goto out;
2292 }
2293
2294 /* Use the return parameters */
2295 rxq->hw_rxq_prod_addr = ret_params.p_prod;
2296 rxq->handle = ret_params.p_handle;
2297
2298 val = &fp->sb_info->sb_virt->pi_array[RX_PI];
2299 rxq->hw_cons_ptr = val;
2300
2301 qede_update_rx_prod(edev, rxq);
2302 }
2303
2304 if (fp->type & QEDE_FASTPATH_XDP) {
2305 rc = qede_start_txq(edev, fp, txq: fp->xdp_tx, rss_id: i, XDP_PI);
2306 if (rc)
2307 goto out;
2308
2309 bpf_prog_add(prog: edev->xdp_prog, i: 1);
2310 fp->rxq->xdp_prog = edev->xdp_prog;
2311 }
2312
2313 if (fp->type & QEDE_FASTPATH_TX) {
2314 int cos;
2315
2316 for_each_cos_in_txq(edev, cos) {
2317 rc = qede_start_txq(edev, fp, txq: &fp->txq[cos], rss_id: i,
2318 TX_PI(cos));
2319 if (rc)
2320 goto out;
2321 }
2322 }
2323 }
2324
2325 /* Prepare and send the vport enable */
2326 vport_update_params->vport_id = start.vport_id;
2327 vport_update_params->update_vport_active_flg = 1;
2328 vport_update_params->vport_active_flg = 1;
2329
2330 if ((qed_info->b_inter_pf_switch || pci_num_vf(dev: edev->pdev)) &&
2331 qed_info->tx_switching) {
2332 vport_update_params->update_tx_switching_flg = 1;
2333 vport_update_params->tx_switching_flg = 1;
2334 }
2335
2336 qede_fill_rss_params(edev, rss: &vport_update_params->rss_params,
2337 update: &vport_update_params->update_rss_flg);
2338
2339 rc = edev->ops->vport_update(cdev, vport_update_params);
2340 if (rc)
2341 DP_ERR(edev, "Update V-PORT failed %d\n", rc);
2342
2343out:
2344 vfree(addr: vport_update_params);
2345 return rc;
2346}
2347
2348enum qede_unload_mode {
2349 QEDE_UNLOAD_NORMAL,
2350 QEDE_UNLOAD_RECOVERY,
2351};
2352
2353static void qede_unload(struct qede_dev *edev, enum qede_unload_mode mode,
2354 bool is_locked)
2355{
2356 struct qed_link_params link_params;
2357 int rc;
2358
2359 DP_INFO(edev, "Starting qede unload\n");
2360
2361 if (!is_locked)
2362 __qede_lock(edev);
2363
2364 clear_bit(nr: QEDE_FLAGS_LINK_REQUESTED, addr: &edev->flags);
2365
2366 if (mode != QEDE_UNLOAD_RECOVERY)
2367 edev->state = QEDE_STATE_CLOSED;
2368
2369 qede_rdma_dev_event_close(dev: edev);
2370
2371 /* Close OS Tx */
2372 netif_tx_disable(dev: edev->ndev);
2373 netif_carrier_off(dev: edev->ndev);
2374
2375 if (mode != QEDE_UNLOAD_RECOVERY) {
2376 /* Reset the link */
2377 memset(&link_params, 0, sizeof(link_params));
2378 link_params.link_up = false;
2379 edev->ops->common->set_link(edev->cdev, &link_params);
2380
2381 rc = qede_stop_queues(edev);
2382 if (rc) {
2383#ifdef CONFIG_RFS_ACCEL
2384 if (edev->dev_info.common.b_arfs_capable) {
2385 qede_poll_for_freeing_arfs_filters(edev);
2386 if (edev->ndev->rx_cpu_rmap)
2387 free_irq_cpu_rmap(rmap: edev->ndev->rx_cpu_rmap);
2388
2389 edev->ndev->rx_cpu_rmap = NULL;
2390 }
2391#endif
2392 qede_sync_free_irqs(edev);
2393 goto out;
2394 }
2395
2396 DP_INFO(edev, "Stopped Queues\n");
2397 }
2398
2399 qede_vlan_mark_nonconfigured(edev);
2400 edev->ops->fastpath_stop(edev->cdev);
2401
2402 if (edev->dev_info.common.b_arfs_capable) {
2403 qede_poll_for_freeing_arfs_filters(edev);
2404 qede_free_arfs(edev);
2405 }
2406
2407 /* Release the interrupts */
2408 qede_sync_free_irqs(edev);
2409 edev->ops->common->set_fp_int(edev->cdev, 0);
2410
2411 qede_napi_disable_remove(edev);
2412
2413 if (mode == QEDE_UNLOAD_RECOVERY)
2414 qede_empty_tx_queues(edev);
2415
2416 qede_free_mem_load(edev);
2417 qede_free_fp_array(edev);
2418
2419out:
2420 if (!is_locked)
2421 __qede_unlock(edev);
2422
2423 if (mode != QEDE_UNLOAD_RECOVERY)
2424 DP_NOTICE(edev, "Link is down\n");
2425
2426 edev->ptp_skip_txts = 0;
2427
2428 DP_INFO(edev, "Ending qede unload\n");
2429}
2430
2431enum qede_load_mode {
2432 QEDE_LOAD_NORMAL,
2433 QEDE_LOAD_RELOAD,
2434 QEDE_LOAD_RECOVERY,
2435};
2436
2437static int qede_load(struct qede_dev *edev, enum qede_load_mode mode,
2438 bool is_locked)
2439{
2440 struct qed_link_params link_params;
2441 struct ethtool_coalesce coal = {};
2442 u8 num_tc;
2443 int rc, i;
2444
2445 DP_INFO(edev, "Starting qede load\n");
2446
2447 if (!is_locked)
2448 __qede_lock(edev);
2449
2450 rc = qede_set_num_queues(edev);
2451 if (rc)
2452 goto out;
2453
2454 rc = qede_alloc_fp_array(edev);
2455 if (rc)
2456 goto out;
2457
2458 qede_init_fp(edev);
2459
2460 rc = qede_alloc_mem_load(edev);
2461 if (rc)
2462 goto err1;
2463 DP_INFO(edev, "Allocated %d Rx, %d Tx queues\n",
2464 QEDE_RSS_COUNT(edev), QEDE_TSS_COUNT(edev));
2465
2466 rc = qede_set_real_num_queues(edev);
2467 if (rc)
2468 goto err2;
2469
2470 if (qede_alloc_arfs(edev)) {
2471 edev->ndev->features &= ~NETIF_F_NTUPLE;
2472 edev->dev_info.common.b_arfs_capable = false;
2473 }
2474
2475 qede_napi_add_enable(edev);
2476 DP_INFO(edev, "Napi added and enabled\n");
2477
2478 rc = qede_setup_irqs(edev);
2479 if (rc)
2480 goto err3;
2481 DP_INFO(edev, "Setup IRQs succeeded\n");
2482
2483 rc = qede_start_queues(edev, clear_stats: mode != QEDE_LOAD_RELOAD);
2484 if (rc)
2485 goto err4;
2486 DP_INFO(edev, "Start VPORT, RXQ and TXQ succeeded\n");
2487
2488 num_tc = netdev_get_num_tc(dev: edev->ndev);
2489 num_tc = num_tc ? num_tc : edev->dev_info.num_tc;
2490 qede_setup_tc(ndev: edev->ndev, num_tc);
2491
2492 /* Program un-configured VLANs */
2493 qede_configure_vlan_filters(edev);
2494
2495 set_bit(nr: QEDE_FLAGS_LINK_REQUESTED, addr: &edev->flags);
2496
2497 /* Ask for link-up using current configuration */
2498 memset(&link_params, 0, sizeof(link_params));
2499 link_params.link_up = true;
2500 edev->ops->common->set_link(edev->cdev, &link_params);
2501
2502 edev->state = QEDE_STATE_OPEN;
2503
2504 coal.rx_coalesce_usecs = QED_DEFAULT_RX_USECS;
2505 coal.tx_coalesce_usecs = QED_DEFAULT_TX_USECS;
2506
2507 for_each_queue(i) {
2508 if (edev->coal_entry[i].isvalid) {
2509 coal.rx_coalesce_usecs = edev->coal_entry[i].rxc;
2510 coal.tx_coalesce_usecs = edev->coal_entry[i].txc;
2511 }
2512 __qede_unlock(edev);
2513 qede_set_per_coalesce(dev: edev->ndev, queue: i, coal: &coal);
2514 __qede_lock(edev);
2515 }
2516 DP_INFO(edev, "Ending successfully qede load\n");
2517
2518 goto out;
2519err4:
2520 qede_sync_free_irqs(edev);
2521err3:
2522 qede_napi_disable_remove(edev);
2523err2:
2524 qede_free_mem_load(edev);
2525err1:
2526 edev->ops->common->set_fp_int(edev->cdev, 0);
2527 qede_free_fp_array(edev);
2528 edev->num_queues = 0;
2529 edev->fp_num_tx = 0;
2530 edev->fp_num_rx = 0;
2531out:
2532 if (!is_locked)
2533 __qede_unlock(edev);
2534
2535 return rc;
2536}
2537
2538/* 'func' should be able to run between unload and reload assuming interface
2539 * is actually running, or afterwards in case it's currently DOWN.
2540 */
2541void qede_reload(struct qede_dev *edev,
2542 struct qede_reload_args *args, bool is_locked)
2543{
2544 if (!is_locked)
2545 __qede_lock(edev);
2546
2547 /* Since qede_lock is held, internal state wouldn't change even
2548 * if netdev state would start transitioning. Check whether current
2549 * internal configuration indicates device is up, then reload.
2550 */
2551 if (edev->state == QEDE_STATE_OPEN) {
2552 qede_unload(edev, mode: QEDE_UNLOAD_NORMAL, is_locked: true);
2553 if (args)
2554 args->func(edev, args);
2555 qede_load(edev, mode: QEDE_LOAD_RELOAD, is_locked: true);
2556
2557 /* Since no one is going to do it for us, re-configure */
2558 qede_config_rx_mode(ndev: edev->ndev);
2559 } else if (args) {
2560 args->func(edev, args);
2561 }
2562
2563 if (!is_locked)
2564 __qede_unlock(edev);
2565}
2566
2567/* called with rtnl_lock */
2568static int qede_open(struct net_device *ndev)
2569{
2570 struct qede_dev *edev = netdev_priv(dev: ndev);
2571 int rc;
2572
2573 netif_carrier_off(dev: ndev);
2574
2575 edev->ops->common->set_power_state(edev->cdev, PCI_D0);
2576
2577 rc = qede_load(edev, mode: QEDE_LOAD_NORMAL, is_locked: false);
2578 if (rc)
2579 return rc;
2580
2581 udp_tunnel_nic_reset_ntf(dev: ndev);
2582
2583 edev->ops->common->update_drv_state(edev->cdev, true);
2584
2585 return 0;
2586}
2587
2588static int qede_close(struct net_device *ndev)
2589{
2590 struct qede_dev *edev = netdev_priv(dev: ndev);
2591
2592 qede_unload(edev, mode: QEDE_UNLOAD_NORMAL, is_locked: false);
2593
2594 if (edev->cdev)
2595 edev->ops->common->update_drv_state(edev->cdev, false);
2596
2597 return 0;
2598}
2599
2600static void qede_link_update(void *dev, struct qed_link_output *link)
2601{
2602 struct qede_dev *edev = dev;
2603
2604 if (!test_bit(QEDE_FLAGS_LINK_REQUESTED, &edev->flags)) {
2605 DP_VERBOSE(edev, NETIF_MSG_LINK, "Interface is not ready\n");
2606 return;
2607 }
2608
2609 if (link->link_up) {
2610 if (!netif_carrier_ok(dev: edev->ndev)) {
2611 DP_NOTICE(edev, "Link is up\n");
2612 netif_tx_start_all_queues(dev: edev->ndev);
2613 netif_carrier_on(dev: edev->ndev);
2614 qede_rdma_dev_event_open(dev: edev);
2615 }
2616 } else {
2617 if (netif_carrier_ok(dev: edev->ndev)) {
2618 DP_NOTICE(edev, "Link is down\n");
2619 netif_tx_disable(dev: edev->ndev);
2620 netif_carrier_off(dev: edev->ndev);
2621 qede_rdma_dev_event_close(dev: edev);
2622 }
2623 }
2624}
2625
2626static void qede_schedule_recovery_handler(void *dev)
2627{
2628 struct qede_dev *edev = dev;
2629
2630 if (edev->state == QEDE_STATE_RECOVERY) {
2631 DP_NOTICE(edev,
2632 "Avoid scheduling a recovery handling since already in recovery state\n");
2633 return;
2634 }
2635
2636 set_bit(QEDE_SP_RECOVERY, addr: &edev->sp_flags);
2637 schedule_delayed_work(dwork: &edev->sp_task, delay: 0);
2638
2639 DP_INFO(edev, "Scheduled a recovery handler\n");
2640}
2641
2642static void qede_recovery_failed(struct qede_dev *edev)
2643{
2644 netdev_err(dev: edev->ndev, format: "Recovery handling has failed. Power cycle is needed.\n");
2645
2646 netif_device_detach(dev: edev->ndev);
2647
2648 if (edev->cdev)
2649 edev->ops->common->set_power_state(edev->cdev, PCI_D3hot);
2650}
2651
2652static void qede_recovery_handler(struct qede_dev *edev)
2653{
2654 u32 curr_state = edev->state;
2655 int rc;
2656
2657 DP_NOTICE(edev, "Starting a recovery process\n");
2658
2659 /* No need to acquire first the qede_lock since is done by qede_sp_task
2660 * before calling this function.
2661 */
2662 edev->state = QEDE_STATE_RECOVERY;
2663
2664 edev->ops->common->recovery_prolog(edev->cdev);
2665
2666 if (curr_state == QEDE_STATE_OPEN)
2667 qede_unload(edev, mode: QEDE_UNLOAD_RECOVERY, is_locked: true);
2668
2669 __qede_remove(pdev: edev->pdev, mode: QEDE_REMOVE_RECOVERY);
2670
2671 rc = __qede_probe(pdev: edev->pdev, dp_module: edev->dp_module, dp_level: edev->dp_level,
2672 IS_VF(edev), mode: QEDE_PROBE_RECOVERY);
2673 if (rc) {
2674 edev->cdev = NULL;
2675 goto err;
2676 }
2677
2678 if (curr_state == QEDE_STATE_OPEN) {
2679 rc = qede_load(edev, mode: QEDE_LOAD_RECOVERY, is_locked: true);
2680 if (rc)
2681 goto err;
2682
2683 qede_config_rx_mode(ndev: edev->ndev);
2684 udp_tunnel_nic_reset_ntf(dev: edev->ndev);
2685 }
2686
2687 edev->state = curr_state;
2688
2689 DP_NOTICE(edev, "Recovery handling is done\n");
2690
2691 return;
2692
2693err:
2694 qede_recovery_failed(edev);
2695}
2696
2697static void qede_atomic_hw_err_handler(struct qede_dev *edev)
2698{
2699 struct qed_dev *cdev = edev->cdev;
2700
2701 DP_NOTICE(edev,
2702 "Generic non-sleepable HW error handling started - err_flags 0x%lx\n",
2703 edev->err_flags);
2704
2705 /* Get a call trace of the flow that led to the error */
2706 WARN_ON(test_bit(QEDE_ERR_WARN, &edev->err_flags));
2707
2708 /* Prevent HW attentions from being reasserted */
2709 if (test_bit(QEDE_ERR_ATTN_CLR_EN, &edev->err_flags))
2710 edev->ops->common->attn_clr_enable(cdev, true);
2711
2712 DP_NOTICE(edev, "Generic non-sleepable HW error handling is done\n");
2713}
2714
2715static void qede_generic_hw_err_handler(struct qede_dev *edev)
2716{
2717 DP_NOTICE(edev,
2718 "Generic sleepable HW error handling started - err_flags 0x%lx\n",
2719 edev->err_flags);
2720
2721 if (edev->devlink) {
2722 DP_NOTICE(edev, "Reporting fatal error to devlink\n");
2723 edev->ops->common->report_fatal_error(edev->devlink, edev->last_err_type);
2724 }
2725
2726 clear_bit(QEDE_ERR_IS_HANDLED, addr: &edev->err_flags);
2727
2728 DP_NOTICE(edev, "Generic sleepable HW error handling is done\n");
2729}
2730
2731static void qede_set_hw_err_flags(struct qede_dev *edev,
2732 enum qed_hw_err_type err_type)
2733{
2734 unsigned long err_flags = 0;
2735
2736 switch (err_type) {
2737 case QED_HW_ERR_DMAE_FAIL:
2738 set_bit(QEDE_ERR_WARN, addr: &err_flags);
2739 fallthrough;
2740 case QED_HW_ERR_MFW_RESP_FAIL:
2741 case QED_HW_ERR_HW_ATTN:
2742 case QED_HW_ERR_RAMROD_FAIL:
2743 case QED_HW_ERR_FW_ASSERT:
2744 set_bit(QEDE_ERR_ATTN_CLR_EN, addr: &err_flags);
2745 set_bit(QEDE_ERR_GET_DBG_INFO, addr: &err_flags);
2746 /* make this error as recoverable and start recovery*/
2747 set_bit(QEDE_ERR_IS_RECOVERABLE, addr: &err_flags);
2748 break;
2749
2750 default:
2751 DP_NOTICE(edev, "Unexpected HW error [%d]\n", err_type);
2752 break;
2753 }
2754
2755 edev->err_flags |= err_flags;
2756}
2757
2758static void qede_schedule_hw_err_handler(void *dev,
2759 enum qed_hw_err_type err_type)
2760{
2761 struct qede_dev *edev = dev;
2762
2763 /* Fan failure cannot be masked by handling of another HW error or by a
2764 * concurrent recovery process.
2765 */
2766 if ((test_and_set_bit(QEDE_ERR_IS_HANDLED, addr: &edev->err_flags) ||
2767 edev->state == QEDE_STATE_RECOVERY) &&
2768 err_type != QED_HW_ERR_FAN_FAIL) {
2769 DP_INFO(edev,
2770 "Avoid scheduling an error handling while another HW error is being handled\n");
2771 return;
2772 }
2773
2774 if (err_type >= QED_HW_ERR_LAST) {
2775 DP_NOTICE(edev, "Unknown HW error [%d]\n", err_type);
2776 clear_bit(QEDE_ERR_IS_HANDLED, addr: &edev->err_flags);
2777 return;
2778 }
2779
2780 edev->last_err_type = err_type;
2781 qede_set_hw_err_flags(edev, err_type);
2782 qede_atomic_hw_err_handler(edev);
2783 set_bit(QEDE_SP_HW_ERR, addr: &edev->sp_flags);
2784 schedule_delayed_work(dwork: &edev->sp_task, delay: 0);
2785
2786 DP_INFO(edev, "Scheduled a error handler [err_type %d]\n", err_type);
2787}
2788
2789static bool qede_is_txq_full(struct qede_dev *edev, struct qede_tx_queue *txq)
2790{
2791 struct netdev_queue *netdev_txq;
2792
2793 netdev_txq = netdev_get_tx_queue(dev: edev->ndev, index: txq->ndev_txq_id);
2794 if (netif_xmit_stopped(dev_queue: netdev_txq))
2795 return true;
2796
2797 return false;
2798}
2799
2800static void qede_get_generic_tlv_data(void *dev, struct qed_generic_tlvs *data)
2801{
2802 struct qede_dev *edev = dev;
2803 struct netdev_hw_addr *ha;
2804 int i;
2805
2806 if (edev->ndev->features & NETIF_F_IP_CSUM)
2807 data->feat_flags |= QED_TLV_IP_CSUM;
2808 if (edev->ndev->features & NETIF_F_TSO)
2809 data->feat_flags |= QED_TLV_LSO;
2810
2811 ether_addr_copy(dst: data->mac[0], src: edev->ndev->dev_addr);
2812 eth_zero_addr(addr: data->mac[1]);
2813 eth_zero_addr(addr: data->mac[2]);
2814 /* Copy the first two UC macs */
2815 netif_addr_lock_bh(dev: edev->ndev);
2816 i = 1;
2817 netdev_for_each_uc_addr(ha, edev->ndev) {
2818 ether_addr_copy(dst: data->mac[i++], src: ha->addr);
2819 if (i == QED_TLV_MAC_COUNT)
2820 break;
2821 }
2822
2823 netif_addr_unlock_bh(dev: edev->ndev);
2824}
2825
2826static void qede_get_eth_tlv_data(void *dev, void *data)
2827{
2828 struct qed_mfw_tlv_eth *etlv = data;
2829 struct qede_dev *edev = dev;
2830 struct qede_fastpath *fp;
2831 int i;
2832
2833 etlv->lso_maxoff_size = 0XFFFF;
2834 etlv->lso_maxoff_size_set = true;
2835 etlv->lso_minseg_size = (u16)ETH_TX_LSO_WINDOW_MIN_LEN;
2836 etlv->lso_minseg_size_set = true;
2837 etlv->prom_mode = !!(edev->ndev->flags & IFF_PROMISC);
2838 etlv->prom_mode_set = true;
2839 etlv->tx_descr_size = QEDE_TSS_COUNT(edev);
2840 etlv->tx_descr_size_set = true;
2841 etlv->rx_descr_size = QEDE_RSS_COUNT(edev);
2842 etlv->rx_descr_size_set = true;
2843 etlv->iov_offload = QED_MFW_TLV_IOV_OFFLOAD_VEB;
2844 etlv->iov_offload_set = true;
2845
2846 /* Fill information regarding queues; Should be done under the qede
2847 * lock to guarantee those don't change beneath our feet.
2848 */
2849 etlv->txqs_empty = true;
2850 etlv->rxqs_empty = true;
2851 etlv->num_txqs_full = 0;
2852 etlv->num_rxqs_full = 0;
2853
2854 __qede_lock(edev);
2855 for_each_queue(i) {
2856 fp = &edev->fp_array[i];
2857 if (fp->type & QEDE_FASTPATH_TX) {
2858 struct qede_tx_queue *txq = QEDE_FP_TC0_TXQ(fp);
2859
2860 if (txq->sw_tx_cons != txq->sw_tx_prod)
2861 etlv->txqs_empty = false;
2862 if (qede_is_txq_full(edev, txq))
2863 etlv->num_txqs_full++;
2864 }
2865 if (fp->type & QEDE_FASTPATH_RX) {
2866 if (qede_has_rx_work(rxq: fp->rxq))
2867 etlv->rxqs_empty = false;
2868
2869 /* This one is a bit tricky; Firmware might stop
2870 * placing packets if ring is not yet full.
2871 * Give an approximation.
2872 */
2873 if (le16_to_cpu(*fp->rxq->hw_cons_ptr) -
2874 qed_chain_get_cons_idx(chain: &fp->rxq->rx_comp_ring) >
2875 RX_RING_SIZE - 100)
2876 etlv->num_rxqs_full++;
2877 }
2878 }
2879 __qede_unlock(edev);
2880
2881 etlv->txqs_empty_set = true;
2882 etlv->rxqs_empty_set = true;
2883 etlv->num_txqs_full_set = true;
2884 etlv->num_rxqs_full_set = true;
2885}
2886
2887/**
2888 * qede_io_error_detected(): Called when PCI error is detected
2889 *
2890 * @pdev: Pointer to PCI device
2891 * @state: The current pci connection state
2892 *
2893 *Return: pci_ers_result_t.
2894 *
2895 * This function is called after a PCI bus error affecting
2896 * this device has been detected.
2897 */
2898static pci_ers_result_t
2899qede_io_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
2900{
2901 struct net_device *dev = pci_get_drvdata(pdev);
2902 struct qede_dev *edev = netdev_priv(dev);
2903
2904 if (!edev)
2905 return PCI_ERS_RESULT_NONE;
2906
2907 DP_NOTICE(edev, "IO error detected [%d]\n", state);
2908
2909 __qede_lock(edev);
2910 if (edev->state == QEDE_STATE_RECOVERY) {
2911 DP_NOTICE(edev, "Device already in the recovery state\n");
2912 __qede_unlock(edev);
2913 return PCI_ERS_RESULT_NONE;
2914 }
2915
2916 /* PF handles the recovery of its VFs */
2917 if (IS_VF(edev)) {
2918 DP_VERBOSE(edev, QED_MSG_IOV,
2919 "VF recovery is handled by its PF\n");
2920 __qede_unlock(edev);
2921 return PCI_ERS_RESULT_RECOVERED;
2922 }
2923
2924 /* Close OS Tx */
2925 netif_tx_disable(dev: edev->ndev);
2926 netif_carrier_off(dev: edev->ndev);
2927
2928 set_bit(QEDE_SP_AER, addr: &edev->sp_flags);
2929 schedule_delayed_work(dwork: &edev->sp_task, delay: 0);
2930
2931 __qede_unlock(edev);
2932
2933 return PCI_ERS_RESULT_CAN_RECOVER;
2934}
2935

source code of linux/drivers/net/ethernet/qlogic/qede/qede_main.c