1 | // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB |
2 | /* Copyright (c) 2015 - 2021 Intel Corporation */ |
3 | #include "main.h" |
4 | #include "../../../net/ethernet/intel/ice/ice.h" |
5 | |
6 | MODULE_ALIAS("i40iw" ); |
7 | MODULE_AUTHOR("Intel Corporation, <e1000-rdma@lists.sourceforge.net>" ); |
8 | MODULE_DESCRIPTION("Intel(R) Ethernet Protocol Driver for RDMA" ); |
9 | MODULE_LICENSE("Dual BSD/GPL" ); |
10 | |
11 | static struct notifier_block irdma_inetaddr_notifier = { |
12 | .notifier_call = irdma_inetaddr_event |
13 | }; |
14 | |
15 | static struct notifier_block irdma_inetaddr6_notifier = { |
16 | .notifier_call = irdma_inet6addr_event |
17 | }; |
18 | |
19 | static struct notifier_block irdma_net_notifier = { |
20 | .notifier_call = irdma_net_event |
21 | }; |
22 | |
23 | static struct notifier_block irdma_netdevice_notifier = { |
24 | .notifier_call = irdma_netdevice_event |
25 | }; |
26 | |
27 | static void irdma_register_notifiers(void) |
28 | { |
29 | register_inetaddr_notifier(nb: &irdma_inetaddr_notifier); |
30 | register_inet6addr_notifier(nb: &irdma_inetaddr6_notifier); |
31 | register_netevent_notifier(nb: &irdma_net_notifier); |
32 | register_netdevice_notifier(nb: &irdma_netdevice_notifier); |
33 | } |
34 | |
35 | static void irdma_unregister_notifiers(void) |
36 | { |
37 | unregister_netevent_notifier(nb: &irdma_net_notifier); |
38 | unregister_inetaddr_notifier(nb: &irdma_inetaddr_notifier); |
39 | unregister_inet6addr_notifier(nb: &irdma_inetaddr6_notifier); |
40 | unregister_netdevice_notifier(nb: &irdma_netdevice_notifier); |
41 | } |
42 | |
43 | static void irdma_prep_tc_change(struct irdma_device *iwdev) |
44 | { |
45 | iwdev->vsi.tc_change_pending = true; |
46 | irdma_sc_suspend_resume_qps(vsi: &iwdev->vsi, suspend: IRDMA_OP_SUSPEND); |
47 | |
48 | /* Wait for all qp's to suspend */ |
49 | wait_event_timeout(iwdev->suspend_wq, |
50 | !atomic_read(&iwdev->vsi.qp_suspend_reqs), |
51 | IRDMA_EVENT_TIMEOUT); |
52 | irdma_ws_reset(vsi: &iwdev->vsi); |
53 | } |
54 | |
55 | static void irdma_log_invalid_mtu(u16 mtu, struct irdma_sc_dev *dev) |
56 | { |
57 | if (mtu < IRDMA_MIN_MTU_IPV4) |
58 | ibdev_warn(ibdev: to_ibdev(dev), format: "MTU setting [%d] too low for RDMA traffic. Minimum MTU is 576 for IPv4\n" , mtu); |
59 | else if (mtu < IRDMA_MIN_MTU_IPV6) |
60 | ibdev_warn(ibdev: to_ibdev(dev), format: "MTU setting [%d] too low for RDMA traffic. Minimum MTU is 1280 for IPv6\\n" , mtu); |
61 | } |
62 | |
63 | static void irdma_fill_qos_info(struct irdma_l2params *l2params, |
64 | struct iidc_qos_params *qos_info) |
65 | { |
66 | int i; |
67 | |
68 | l2params->num_tc = qos_info->num_tc; |
69 | l2params->vsi_prio_type = qos_info->vport_priority_type; |
70 | l2params->vsi_rel_bw = qos_info->vport_relative_bw; |
71 | for (i = 0; i < l2params->num_tc; i++) { |
72 | l2params->tc_info[i].egress_virt_up = |
73 | qos_info->tc_info[i].egress_virt_up; |
74 | l2params->tc_info[i].ingress_virt_up = |
75 | qos_info->tc_info[i].ingress_virt_up; |
76 | l2params->tc_info[i].prio_type = qos_info->tc_info[i].prio_type; |
77 | l2params->tc_info[i].rel_bw = qos_info->tc_info[i].rel_bw; |
78 | l2params->tc_info[i].tc_ctx = qos_info->tc_info[i].tc_ctx; |
79 | } |
80 | for (i = 0; i < IIDC_MAX_USER_PRIORITY; i++) |
81 | l2params->up2tc[i] = qos_info->up2tc[i]; |
82 | if (qos_info->pfc_mode == IIDC_DSCP_PFC_MODE) { |
83 | l2params->dscp_mode = true; |
84 | memcpy(l2params->dscp_map, qos_info->dscp_map, sizeof(l2params->dscp_map)); |
85 | } |
86 | } |
87 | |
88 | static void irdma_iidc_event_handler(struct ice_pf *pf, struct iidc_event *event) |
89 | { |
90 | struct irdma_device *iwdev = dev_get_drvdata(dev: &pf->adev->dev); |
91 | struct irdma_l2params l2params = {}; |
92 | |
93 | if (*event->type & BIT(IIDC_EVENT_AFTER_MTU_CHANGE)) { |
94 | ibdev_dbg(&iwdev->ibdev, "CLNT: new MTU = %d\n" , iwdev->netdev->mtu); |
95 | if (iwdev->vsi.mtu != iwdev->netdev->mtu) { |
96 | l2params.mtu = iwdev->netdev->mtu; |
97 | l2params.mtu_changed = true; |
98 | irdma_log_invalid_mtu(mtu: l2params.mtu, dev: &iwdev->rf->sc_dev); |
99 | irdma_change_l2params(vsi: &iwdev->vsi, l2params: &l2params); |
100 | } |
101 | } else if (*event->type & BIT(IIDC_EVENT_BEFORE_TC_CHANGE)) { |
102 | if (iwdev->vsi.tc_change_pending) |
103 | return; |
104 | |
105 | irdma_prep_tc_change(iwdev); |
106 | } else if (*event->type & BIT(IIDC_EVENT_AFTER_TC_CHANGE)) { |
107 | struct iidc_qos_params qos_info = {}; |
108 | |
109 | if (!iwdev->vsi.tc_change_pending) |
110 | return; |
111 | |
112 | l2params.tc_changed = true; |
113 | ibdev_dbg(&iwdev->ibdev, "CLNT: TC Change\n" ); |
114 | ice_get_qos_params(pf, qos: &qos_info); |
115 | irdma_fill_qos_info(l2params: &l2params, qos_info: &qos_info); |
116 | if (iwdev->rf->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY) |
117 | iwdev->dcb_vlan_mode = qos_info.num_tc > 1 && !l2params.dscp_mode; |
118 | irdma_change_l2params(vsi: &iwdev->vsi, l2params: &l2params); |
119 | } else if (*event->type & BIT(IIDC_EVENT_CRIT_ERR)) { |
120 | ibdev_warn(ibdev: &iwdev->ibdev, format: "ICE OICR event notification: oicr = 0x%08x\n" , |
121 | event->reg); |
122 | if (event->reg & IRDMAPFINT_OICR_PE_CRITERR_M) { |
123 | u32 pe_criterr; |
124 | |
125 | pe_criterr = readl(addr: iwdev->rf->sc_dev.hw_regs[IRDMA_GLPE_CRITERR]); |
126 | #define IRDMA_Q1_RESOURCE_ERR 0x0001024d |
127 | if (pe_criterr != IRDMA_Q1_RESOURCE_ERR) { |
128 | ibdev_err(ibdev: &iwdev->ibdev, format: "critical PE Error, GLPE_CRITERR=0x%08x\n" , |
129 | pe_criterr); |
130 | iwdev->rf->reset = true; |
131 | } else { |
132 | ibdev_warn(ibdev: &iwdev->ibdev, format: "Q1 Resource Check\n" ); |
133 | } |
134 | } |
135 | if (event->reg & IRDMAPFINT_OICR_HMC_ERR_M) { |
136 | ibdev_err(ibdev: &iwdev->ibdev, format: "HMC Error\n" ); |
137 | iwdev->rf->reset = true; |
138 | } |
139 | if (event->reg & IRDMAPFINT_OICR_PE_PUSH_M) { |
140 | ibdev_err(ibdev: &iwdev->ibdev, format: "PE Push Error\n" ); |
141 | iwdev->rf->reset = true; |
142 | } |
143 | if (iwdev->rf->reset) |
144 | iwdev->rf->gen_ops.request_reset(iwdev->rf); |
145 | } |
146 | } |
147 | |
148 | /** |
149 | * irdma_request_reset - Request a reset |
150 | * @rf: RDMA PCI function |
151 | */ |
152 | static void irdma_request_reset(struct irdma_pci_f *rf) |
153 | { |
154 | struct ice_pf *pf = rf->cdev; |
155 | |
156 | ibdev_warn(ibdev: &rf->iwdev->ibdev, format: "Requesting a reset\n" ); |
157 | ice_rdma_request_reset(pf, reset_type: IIDC_PFR); |
158 | } |
159 | |
160 | /** |
161 | * irdma_lan_register_qset - Register qset with LAN driver |
162 | * @vsi: vsi structure |
163 | * @tc_node: Traffic class node |
164 | */ |
165 | static int irdma_lan_register_qset(struct irdma_sc_vsi *vsi, |
166 | struct irdma_ws_node *tc_node) |
167 | { |
168 | struct irdma_device *iwdev = vsi->back_vsi; |
169 | struct ice_pf *pf = iwdev->rf->cdev; |
170 | struct iidc_rdma_qset_params qset = {}; |
171 | int ret; |
172 | |
173 | qset.qs_handle = tc_node->qs_handle; |
174 | qset.tc = tc_node->traffic_class; |
175 | qset.vport_id = vsi->vsi_idx; |
176 | ret = ice_add_rdma_qset(pf, qset: &qset); |
177 | if (ret) { |
178 | ibdev_dbg(&iwdev->ibdev, "WS: LAN alloc_res for rdma qset failed.\n" ); |
179 | return ret; |
180 | } |
181 | |
182 | tc_node->l2_sched_node_id = qset.teid; |
183 | vsi->qos[tc_node->user_pri].l2_sched_node_id = qset.teid; |
184 | |
185 | return 0; |
186 | } |
187 | |
188 | /** |
189 | * irdma_lan_unregister_qset - Unregister qset with LAN driver |
190 | * @vsi: vsi structure |
191 | * @tc_node: Traffic class node |
192 | */ |
193 | static void irdma_lan_unregister_qset(struct irdma_sc_vsi *vsi, |
194 | struct irdma_ws_node *tc_node) |
195 | { |
196 | struct irdma_device *iwdev = vsi->back_vsi; |
197 | struct ice_pf *pf = iwdev->rf->cdev; |
198 | struct iidc_rdma_qset_params qset = {}; |
199 | |
200 | qset.qs_handle = tc_node->qs_handle; |
201 | qset.tc = tc_node->traffic_class; |
202 | qset.vport_id = vsi->vsi_idx; |
203 | qset.teid = tc_node->l2_sched_node_id; |
204 | |
205 | if (ice_del_rdma_qset(pf, qset: &qset)) |
206 | ibdev_dbg(&iwdev->ibdev, "WS: LAN free_res for rdma qset failed.\n" ); |
207 | } |
208 | |
209 | static void irdma_remove(struct auxiliary_device *aux_dev) |
210 | { |
211 | struct iidc_auxiliary_dev *iidc_adev = container_of(aux_dev, |
212 | struct iidc_auxiliary_dev, |
213 | adev); |
214 | struct ice_pf *pf = iidc_adev->pf; |
215 | struct irdma_device *iwdev = auxiliary_get_drvdata(auxdev: aux_dev); |
216 | |
217 | irdma_ib_unregister_device(iwdev); |
218 | ice_rdma_update_vsi_filter(pf, vsi_id: iwdev->vsi_num, enable: false); |
219 | |
220 | pr_debug("INIT: Gen2 PF[%d] device remove success\n" , PCI_FUNC(pf->pdev->devfn)); |
221 | } |
222 | |
223 | static void irdma_fill_device_info(struct irdma_device *iwdev, struct ice_pf *pf, |
224 | struct ice_vsi *vsi) |
225 | { |
226 | struct irdma_pci_f *rf = iwdev->rf; |
227 | |
228 | rf->cdev = pf; |
229 | rf->gen_ops.register_qset = irdma_lan_register_qset; |
230 | rf->gen_ops.unregister_qset = irdma_lan_unregister_qset; |
231 | rf->hw.hw_addr = pf->hw.hw_addr; |
232 | rf->pcidev = pf->pdev; |
233 | rf->msix_count = pf->num_rdma_msix; |
234 | rf->pf_id = pf->hw.pf_id; |
235 | rf->msix_entries = &pf->msix_entries[pf->rdma_base_vector]; |
236 | rf->default_vsi.vsi_idx = vsi->vsi_num; |
237 | rf->protocol_used = pf->rdma_mode & IIDC_RDMA_PROTOCOL_ROCEV2 ? |
238 | IRDMA_ROCE_PROTOCOL_ONLY : IRDMA_IWARP_PROTOCOL_ONLY; |
239 | rf->rdma_ver = IRDMA_GEN_2; |
240 | rf->rsrc_profile = IRDMA_HMC_PROFILE_DEFAULT; |
241 | rf->rst_to = IRDMA_RST_TIMEOUT_HZ; |
242 | rf->gen_ops.request_reset = irdma_request_reset; |
243 | rf->limits_sel = 7; |
244 | rf->iwdev = iwdev; |
245 | mutex_init(&iwdev->ah_tbl_lock); |
246 | iwdev->netdev = vsi->netdev; |
247 | iwdev->vsi_num = vsi->vsi_num; |
248 | iwdev->init_state = INITIAL_STATE; |
249 | iwdev->roce_cwnd = IRDMA_ROCE_CWND_DEFAULT; |
250 | iwdev->roce_ackcreds = IRDMA_ROCE_ACKCREDS_DEFAULT; |
251 | iwdev->rcv_wnd = IRDMA_CM_DEFAULT_RCV_WND_SCALED; |
252 | iwdev->rcv_wscale = IRDMA_CM_DEFAULT_RCV_WND_SCALE; |
253 | if (rf->protocol_used == IRDMA_ROCE_PROTOCOL_ONLY) |
254 | iwdev->roce_mode = true; |
255 | } |
256 | |
257 | static int irdma_probe(struct auxiliary_device *aux_dev, const struct auxiliary_device_id *id) |
258 | { |
259 | struct iidc_auxiliary_dev *iidc_adev = container_of(aux_dev, |
260 | struct iidc_auxiliary_dev, |
261 | adev); |
262 | struct ice_pf *pf = iidc_adev->pf; |
263 | struct ice_vsi *vsi = ice_get_main_vsi(pf); |
264 | struct iidc_qos_params qos_info = {}; |
265 | struct irdma_device *iwdev; |
266 | struct irdma_pci_f *rf; |
267 | struct irdma_l2params l2params = {}; |
268 | int err; |
269 | |
270 | if (!vsi) |
271 | return -EIO; |
272 | iwdev = ib_alloc_device(irdma_device, ibdev); |
273 | if (!iwdev) |
274 | return -ENOMEM; |
275 | iwdev->rf = kzalloc(size: sizeof(*rf), GFP_KERNEL); |
276 | if (!iwdev->rf) { |
277 | ib_dealloc_device(device: &iwdev->ibdev); |
278 | return -ENOMEM; |
279 | } |
280 | |
281 | irdma_fill_device_info(iwdev, pf, vsi); |
282 | rf = iwdev->rf; |
283 | |
284 | err = irdma_ctrl_init_hw(rf); |
285 | if (err) |
286 | goto err_ctrl_init; |
287 | |
288 | l2params.mtu = iwdev->netdev->mtu; |
289 | ice_get_qos_params(pf, qos: &qos_info); |
290 | irdma_fill_qos_info(l2params: &l2params, qos_info: &qos_info); |
291 | if (iwdev->rf->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY) |
292 | iwdev->dcb_vlan_mode = l2params.num_tc > 1 && !l2params.dscp_mode; |
293 | |
294 | err = irdma_rt_init_hw(iwdev, l2params: &l2params); |
295 | if (err) |
296 | goto err_rt_init; |
297 | |
298 | err = irdma_ib_register_device(iwdev); |
299 | if (err) |
300 | goto err_ibreg; |
301 | |
302 | ice_rdma_update_vsi_filter(pf, vsi_id: iwdev->vsi_num, enable: true); |
303 | |
304 | ibdev_dbg(&iwdev->ibdev, "INIT: Gen2 PF[%d] device probe success\n" , PCI_FUNC(rf->pcidev->devfn)); |
305 | auxiliary_set_drvdata(auxdev: aux_dev, data: iwdev); |
306 | |
307 | return 0; |
308 | |
309 | err_ibreg: |
310 | irdma_rt_deinit_hw(iwdev); |
311 | err_rt_init: |
312 | irdma_ctrl_deinit_hw(rf); |
313 | err_ctrl_init: |
314 | kfree(objp: iwdev->rf); |
315 | ib_dealloc_device(device: &iwdev->ibdev); |
316 | |
317 | return err; |
318 | } |
319 | |
320 | static const struct auxiliary_device_id irdma_auxiliary_id_table[] = { |
321 | {.name = "ice.iwarp" , }, |
322 | {.name = "ice.roce" , }, |
323 | {}, |
324 | }; |
325 | |
326 | MODULE_DEVICE_TABLE(auxiliary, irdma_auxiliary_id_table); |
327 | |
328 | static struct iidc_auxiliary_drv irdma_auxiliary_drv = { |
329 | .adrv = { |
330 | .id_table = irdma_auxiliary_id_table, |
331 | .probe = irdma_probe, |
332 | .remove = irdma_remove, |
333 | }, |
334 | .event_handler = irdma_iidc_event_handler, |
335 | }; |
336 | |
337 | static int __init irdma_init_module(void) |
338 | { |
339 | int ret; |
340 | |
341 | ret = auxiliary_driver_register(&i40iw_auxiliary_drv); |
342 | if (ret) { |
343 | pr_err("Failed i40iw(gen_1) auxiliary_driver_register() ret=%d\n" , |
344 | ret); |
345 | return ret; |
346 | } |
347 | |
348 | ret = auxiliary_driver_register(&irdma_auxiliary_drv.adrv); |
349 | if (ret) { |
350 | auxiliary_driver_unregister(auxdrv: &i40iw_auxiliary_drv); |
351 | pr_err("Failed irdma auxiliary_driver_register() ret=%d\n" , |
352 | ret); |
353 | return ret; |
354 | } |
355 | |
356 | irdma_register_notifiers(); |
357 | |
358 | return 0; |
359 | } |
360 | |
361 | static void __exit irdma_exit_module(void) |
362 | { |
363 | irdma_unregister_notifiers(); |
364 | auxiliary_driver_unregister(auxdrv: &irdma_auxiliary_drv.adrv); |
365 | auxiliary_driver_unregister(auxdrv: &i40iw_auxiliary_drv); |
366 | } |
367 | |
368 | module_init(irdma_init_module); |
369 | module_exit(irdma_exit_module); |
370 | |