1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2/* Copyright (c) 2015 - 2021 Intel Corporation */
3#include "main.h"
4
5MODULE_ALIAS("i40iw");
6MODULE_DESCRIPTION("Intel(R) Ethernet Protocol Driver for RDMA");
7MODULE_LICENSE("Dual BSD/GPL");
8
9static struct notifier_block irdma_inetaddr_notifier = {
10 .notifier_call = irdma_inetaddr_event
11};
12
13static struct notifier_block irdma_inetaddr6_notifier = {
14 .notifier_call = irdma_inet6addr_event
15};
16
17static struct notifier_block irdma_net_notifier = {
18 .notifier_call = irdma_net_event
19};
20
21static struct notifier_block irdma_netdevice_notifier = {
22 .notifier_call = irdma_netdevice_event
23};
24
25static void irdma_register_notifiers(void)
26{
27 register_inetaddr_notifier(nb: &irdma_inetaddr_notifier);
28 register_inet6addr_notifier(nb: &irdma_inetaddr6_notifier);
29 register_netevent_notifier(nb: &irdma_net_notifier);
30 register_netdevice_notifier(nb: &irdma_netdevice_notifier);
31}
32
33static void irdma_unregister_notifiers(void)
34{
35 unregister_netevent_notifier(nb: &irdma_net_notifier);
36 unregister_inetaddr_notifier(nb: &irdma_inetaddr_notifier);
37 unregister_inet6addr_notifier(nb: &irdma_inetaddr6_notifier);
38 unregister_netdevice_notifier(nb: &irdma_netdevice_notifier);
39}
40
41static void irdma_prep_tc_change(struct irdma_device *iwdev)
42{
43 iwdev->vsi.tc_change_pending = true;
44 irdma_sc_suspend_resume_qps(vsi: &iwdev->vsi, suspend: IRDMA_OP_SUSPEND);
45
46 /* Wait for all qp's to suspend */
47 wait_event_timeout(iwdev->suspend_wq,
48 !atomic_read(&iwdev->vsi.qp_suspend_reqs),
49 msecs_to_jiffies(IRDMA_EVENT_TIMEOUT_MS));
50 irdma_ws_reset(vsi: &iwdev->vsi);
51}
52
53static void irdma_log_invalid_mtu(u16 mtu, struct irdma_sc_dev *dev)
54{
55 if (mtu < IRDMA_MIN_MTU_IPV4)
56 ibdev_warn(ibdev: to_ibdev(dev), format: "MTU setting [%d] too low for RDMA traffic. Minimum MTU is 576 for IPv4\n", mtu);
57 else if (mtu < IRDMA_MIN_MTU_IPV6)
58 ibdev_warn(ibdev: to_ibdev(dev), format: "MTU setting [%d] too low for RDMA traffic. Minimum MTU is 1280 for IPv6\\n", mtu);
59}
60
61static void irdma_fill_qos_info(struct irdma_l2params *l2params,
62 struct iidc_rdma_qos_params *qos_info)
63{
64 int i;
65
66 l2params->num_tc = qos_info->num_tc;
67 l2params->vsi_prio_type = qos_info->vport_priority_type;
68 l2params->vsi_rel_bw = qos_info->vport_relative_bw;
69 for (i = 0; i < l2params->num_tc; i++) {
70 l2params->tc_info[i].egress_virt_up =
71 qos_info->tc_info[i].egress_virt_up;
72 l2params->tc_info[i].ingress_virt_up =
73 qos_info->tc_info[i].ingress_virt_up;
74 l2params->tc_info[i].prio_type = qos_info->tc_info[i].prio_type;
75 l2params->tc_info[i].rel_bw = qos_info->tc_info[i].rel_bw;
76 l2params->tc_info[i].tc_ctx = qos_info->tc_info[i].tc_ctx;
77 }
78 for (i = 0; i < IIDC_MAX_USER_PRIORITY; i++)
79 l2params->up2tc[i] = qos_info->up2tc[i];
80 if (qos_info->pfc_mode == IIDC_DSCP_PFC_MODE) {
81 l2params->dscp_mode = true;
82 memcpy(l2params->dscp_map, qos_info->dscp_map, sizeof(l2params->dscp_map));
83 }
84}
85
86static void irdma_iidc_event_handler(struct iidc_rdma_core_dev_info *cdev_info,
87 struct iidc_rdma_event *event)
88{
89 struct irdma_device *iwdev = dev_get_drvdata(dev: &cdev_info->adev->dev);
90 struct irdma_l2params l2params = {};
91
92 if (*event->type & BIT(IIDC_RDMA_EVENT_AFTER_MTU_CHANGE)) {
93 ibdev_dbg(&iwdev->ibdev, "CLNT: new MTU = %d\n", iwdev->netdev->mtu);
94 if (iwdev->vsi.mtu != iwdev->netdev->mtu) {
95 l2params.mtu = iwdev->netdev->mtu;
96 l2params.mtu_changed = true;
97 irdma_log_invalid_mtu(mtu: l2params.mtu, dev: &iwdev->rf->sc_dev);
98 irdma_change_l2params(vsi: &iwdev->vsi, l2params: &l2params);
99 }
100 } else if (*event->type & BIT(IIDC_RDMA_EVENT_BEFORE_TC_CHANGE)) {
101 if (iwdev->vsi.tc_change_pending)
102 return;
103
104 irdma_prep_tc_change(iwdev);
105 } else if (*event->type & BIT(IIDC_RDMA_EVENT_AFTER_TC_CHANGE)) {
106 struct iidc_rdma_priv_dev_info *iidc_priv = cdev_info->iidc_priv;
107
108 if (!iwdev->vsi.tc_change_pending)
109 return;
110
111 l2params.tc_changed = true;
112 ibdev_dbg(&iwdev->ibdev, "CLNT: TC Change\n");
113
114 irdma_fill_qos_info(l2params: &l2params, qos_info: &iidc_priv->qos_info);
115 if (iwdev->rf->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY)
116 iwdev->dcb_vlan_mode =
117 l2params.num_tc > 1 && !l2params.dscp_mode;
118 irdma_change_l2params(vsi: &iwdev->vsi, l2params: &l2params);
119 } else if (*event->type & BIT(IIDC_RDMA_EVENT_CRIT_ERR)) {
120 ibdev_warn(ibdev: &iwdev->ibdev, format: "ICE OICR event notification: oicr = 0x%08x\n",
121 event->reg);
122 if (event->reg & IRDMAPFINT_OICR_PE_CRITERR_M) {
123 u32 pe_criterr;
124
125 pe_criterr = readl(addr: iwdev->rf->sc_dev.hw_regs[IRDMA_GLPE_CRITERR]);
126#define IRDMA_Q1_RESOURCE_ERR 0x0001024d
127 if (pe_criterr != IRDMA_Q1_RESOURCE_ERR) {
128 ibdev_err(ibdev: &iwdev->ibdev, format: "critical PE Error, GLPE_CRITERR=0x%08x\n",
129 pe_criterr);
130 iwdev->rf->reset = true;
131 } else {
132 ibdev_warn(ibdev: &iwdev->ibdev, format: "Q1 Resource Check\n");
133 }
134 }
135 if (event->reg & IRDMAPFINT_OICR_HMC_ERR_M) {
136 ibdev_err(ibdev: &iwdev->ibdev, format: "HMC Error\n");
137 iwdev->rf->reset = true;
138 }
139 if (event->reg & IRDMAPFINT_OICR_PE_PUSH_M) {
140 ibdev_err(ibdev: &iwdev->ibdev, format: "PE Push Error\n");
141 iwdev->rf->reset = true;
142 }
143 if (iwdev->rf->reset)
144 iwdev->rf->gen_ops.request_reset(iwdev->rf);
145 }
146}
147
148/**
149 * irdma_request_reset - Request a reset
150 * @rf: RDMA PCI function
151 */
152static void irdma_request_reset(struct irdma_pci_f *rf)
153{
154 ibdev_warn(ibdev: &rf->iwdev->ibdev, format: "Requesting a reset\n");
155 ice_rdma_request_reset(cdev: rf->cdev, reset_type: IIDC_FUNC_RESET);
156}
157
158/**
159 * irdma_lan_register_qset - Register qset with LAN driver
160 * @vsi: vsi structure
161 * @tc_node: Traffic class node
162 */
163static int irdma_lan_register_qset(struct irdma_sc_vsi *vsi,
164 struct irdma_ws_node *tc_node)
165{
166 struct irdma_device *iwdev = vsi->back_vsi;
167 struct iidc_rdma_core_dev_info *cdev_info;
168 struct iidc_rdma_qset_params qset = {};
169 int ret;
170
171 cdev_info = iwdev->rf->cdev;
172 qset.qs_handle = tc_node->qs_handle;
173 qset.tc = tc_node->traffic_class;
174 qset.vport_id = vsi->vsi_idx;
175 ret = ice_add_rdma_qset(cdev: cdev_info, qset: &qset);
176 if (ret) {
177 ibdev_dbg(&iwdev->ibdev, "WS: LAN alloc_res for rdma qset failed.\n");
178 return ret;
179 }
180
181 tc_node->l2_sched_node_id = qset.teid;
182 vsi->qos[tc_node->user_pri].l2_sched_node_id = qset.teid;
183
184 return 0;
185}
186
187/**
188 * irdma_lan_unregister_qset - Unregister qset with LAN driver
189 * @vsi: vsi structure
190 * @tc_node: Traffic class node
191 */
192static void irdma_lan_unregister_qset(struct irdma_sc_vsi *vsi,
193 struct irdma_ws_node *tc_node)
194{
195 struct irdma_device *iwdev = vsi->back_vsi;
196 struct iidc_rdma_core_dev_info *cdev_info;
197 struct iidc_rdma_qset_params qset = {};
198
199 cdev_info = iwdev->rf->cdev;
200 qset.qs_handle = tc_node->qs_handle;
201 qset.tc = tc_node->traffic_class;
202 qset.vport_id = vsi->vsi_idx;
203 qset.teid = tc_node->l2_sched_node_id;
204
205 if (ice_del_rdma_qset(cdev: cdev_info, qset: &qset))
206 ibdev_dbg(&iwdev->ibdev, "WS: LAN free_res for rdma qset failed.\n");
207}
208
209static int irdma_init_interrupts(struct irdma_pci_f *rf, struct iidc_rdma_core_dev_info *cdev)
210{
211 int i;
212
213 rf->msix_count = num_online_cpus() + IRDMA_NUM_AEQ_MSIX;
214 rf->msix_entries = kcalloc(rf->msix_count, sizeof(*rf->msix_entries),
215 GFP_KERNEL);
216 if (!rf->msix_entries)
217 return -ENOMEM;
218
219 for (i = 0; i < rf->msix_count; i++)
220 if (ice_alloc_rdma_qvector(cdev, entry: &rf->msix_entries[i]))
221 break;
222
223 if (i < IRDMA_MIN_MSIX) {
224 while (--i >= 0)
225 ice_free_rdma_qvector(cdev, entry: &rf->msix_entries[i]);
226
227 kfree(objp: rf->msix_entries);
228 return -ENOMEM;
229 }
230
231 rf->msix_count = i;
232
233 return 0;
234}
235
236static void irdma_deinit_interrupts(struct irdma_pci_f *rf, struct iidc_rdma_core_dev_info *cdev)
237{
238 int i;
239
240 for (i = 0; i < rf->msix_count; i++)
241 ice_free_rdma_qvector(cdev, entry: &rf->msix_entries[i]);
242
243 kfree(objp: rf->msix_entries);
244}
245
246static void irdma_remove(struct auxiliary_device *aux_dev)
247{
248 struct irdma_device *iwdev = auxiliary_get_drvdata(auxdev: aux_dev);
249 struct iidc_rdma_core_auxiliary_dev *iidc_adev;
250 struct iidc_rdma_core_dev_info *cdev_info;
251
252 iidc_adev = container_of(aux_dev, struct iidc_rdma_core_auxiliary_dev, adev);
253 cdev_info = iidc_adev->cdev_info;
254
255 ice_rdma_update_vsi_filter(cdev: cdev_info, vsi_id: iwdev->vsi_num, enable: false);
256 irdma_ib_unregister_device(iwdev);
257 irdma_deinit_interrupts(rf: iwdev->rf, cdev: cdev_info);
258
259 kfree(objp: iwdev->rf);
260
261 pr_debug("INIT: Gen2 PF[%d] device remove success\n", PCI_FUNC(cdev_info->pdev->devfn));
262}
263
264static void irdma_fill_device_info(struct irdma_device *iwdev,
265 struct iidc_rdma_core_dev_info *cdev_info)
266{
267 struct iidc_rdma_priv_dev_info *iidc_priv = cdev_info->iidc_priv;
268 struct irdma_pci_f *rf = iwdev->rf;
269
270 rf->sc_dev.hw = &rf->hw;
271 rf->iwdev = iwdev;
272 rf->cdev = cdev_info;
273 rf->hw.hw_addr = iidc_priv->hw_addr;
274 rf->pcidev = cdev_info->pdev;
275 rf->hw.device = &rf->pcidev->dev;
276 rf->pf_id = iidc_priv->pf_id;
277 rf->gen_ops.register_qset = irdma_lan_register_qset;
278 rf->gen_ops.unregister_qset = irdma_lan_unregister_qset;
279
280 rf->default_vsi.vsi_idx = iidc_priv->vport_id;
281 rf->protocol_used =
282 cdev_info->rdma_protocol == IIDC_RDMA_PROTOCOL_ROCEV2 ?
283 IRDMA_ROCE_PROTOCOL_ONLY : IRDMA_IWARP_PROTOCOL_ONLY;
284 rf->rdma_ver = IRDMA_GEN_2;
285 rf->rsrc_profile = IRDMA_HMC_PROFILE_DEFAULT;
286 rf->rst_to = IRDMA_RST_TIMEOUT_HZ;
287 rf->gen_ops.request_reset = irdma_request_reset;
288 rf->limits_sel = 7;
289 rf->iwdev = iwdev;
290
291 mutex_init(&iwdev->ah_tbl_lock);
292
293 iwdev->netdev = iidc_priv->netdev;
294 iwdev->vsi_num = iidc_priv->vport_id;
295 iwdev->init_state = INITIAL_STATE;
296 iwdev->roce_cwnd = IRDMA_ROCE_CWND_DEFAULT;
297 iwdev->roce_ackcreds = IRDMA_ROCE_ACKCREDS_DEFAULT;
298 iwdev->rcv_wnd = IRDMA_CM_DEFAULT_RCV_WND_SCALED;
299 iwdev->rcv_wscale = IRDMA_CM_DEFAULT_RCV_WND_SCALE;
300 if (rf->protocol_used == IRDMA_ROCE_PROTOCOL_ONLY)
301 iwdev->roce_mode = true;
302}
303
304static int irdma_probe(struct auxiliary_device *aux_dev, const struct auxiliary_device_id *id)
305{
306 struct iidc_rdma_core_auxiliary_dev *iidc_adev;
307 struct iidc_rdma_core_dev_info *cdev_info;
308 struct iidc_rdma_priv_dev_info *iidc_priv;
309 struct irdma_l2params l2params = {};
310 struct irdma_device *iwdev;
311 struct irdma_pci_f *rf;
312 int err;
313
314 iidc_adev = container_of(aux_dev, struct iidc_rdma_core_auxiliary_dev, adev);
315 cdev_info = iidc_adev->cdev_info;
316 iidc_priv = cdev_info->iidc_priv;
317
318 iwdev = ib_alloc_device(irdma_device, ibdev);
319 if (!iwdev)
320 return -ENOMEM;
321 iwdev->rf = kzalloc(sizeof(*rf), GFP_KERNEL);
322 if (!iwdev->rf) {
323 ib_dealloc_device(device: &iwdev->ibdev);
324 return -ENOMEM;
325 }
326
327 irdma_fill_device_info(iwdev, cdev_info);
328 rf = iwdev->rf;
329
330 err = irdma_init_interrupts(rf, cdev: cdev_info);
331 if (err)
332 goto err_init_interrupts;
333
334 err = irdma_ctrl_init_hw(rf);
335 if (err)
336 goto err_ctrl_init;
337
338 l2params.mtu = iwdev->netdev->mtu;
339 irdma_fill_qos_info(l2params: &l2params, qos_info: &iidc_priv->qos_info);
340 if (iwdev->rf->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY)
341 iwdev->dcb_vlan_mode = l2params.num_tc > 1 && !l2params.dscp_mode;
342
343 err = irdma_rt_init_hw(iwdev, l2params: &l2params);
344 if (err)
345 goto err_rt_init;
346
347 err = irdma_ib_register_device(iwdev);
348 if (err)
349 goto err_ibreg;
350
351 ice_rdma_update_vsi_filter(cdev: cdev_info, vsi_id: iwdev->vsi_num, enable: true);
352
353 ibdev_dbg(&iwdev->ibdev, "INIT: Gen2 PF[%d] device probe success\n", PCI_FUNC(rf->pcidev->devfn));
354 auxiliary_set_drvdata(auxdev: aux_dev, data: iwdev);
355
356 return 0;
357
358err_ibreg:
359 irdma_rt_deinit_hw(iwdev);
360err_rt_init:
361 irdma_ctrl_deinit_hw(rf);
362err_ctrl_init:
363 irdma_deinit_interrupts(rf, cdev: cdev_info);
364err_init_interrupts:
365 kfree(objp: iwdev->rf);
366 ib_dealloc_device(device: &iwdev->ibdev);
367
368 return err;
369}
370
371static const struct auxiliary_device_id irdma_auxiliary_id_table[] = {
372 {.name = "ice.iwarp", },
373 {.name = "ice.roce", },
374 {},
375};
376
377MODULE_DEVICE_TABLE(auxiliary, irdma_auxiliary_id_table);
378
379static struct iidc_rdma_core_auxiliary_drv irdma_auxiliary_drv = {
380 .adrv = {
381 .id_table = irdma_auxiliary_id_table,
382 .probe = irdma_probe,
383 .remove = irdma_remove,
384 },
385 .event_handler = irdma_iidc_event_handler,
386};
387
388static int __init irdma_init_module(void)
389{
390 int ret;
391
392 ret = auxiliary_driver_register(&i40iw_auxiliary_drv);
393 if (ret) {
394 pr_err("Failed i40iw(gen_1) auxiliary_driver_register() ret=%d\n",
395 ret);
396 return ret;
397 }
398
399 ret = auxiliary_driver_register(&irdma_auxiliary_drv.adrv);
400 if (ret) {
401 auxiliary_driver_unregister(auxdrv: &i40iw_auxiliary_drv);
402 pr_err("Failed irdma auxiliary_driver_register() ret=%d\n",
403 ret);
404 return ret;
405 }
406
407 irdma_register_notifiers();
408
409 return 0;
410}
411
412static void __exit irdma_exit_module(void)
413{
414 irdma_unregister_notifiers();
415 auxiliary_driver_unregister(auxdrv: &irdma_auxiliary_drv.adrv);
416 auxiliary_driver_unregister(auxdrv: &i40iw_auxiliary_drv);
417}
418
419module_init(irdma_init_module);
420module_exit(irdma_exit_module);
421

source code of linux/drivers/infiniband/hw/irdma/main.c