1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2
3/* Authors: Cheng Xu <chengyou@linux.alibaba.com> */
4/* Kai Shen <kaishen@linux.alibaba.com> */
5/* Copyright (c) 2020-2022, Alibaba Group. */
6
7#include <linux/module.h>
8#include <net/addrconf.h>
9#include <rdma/erdma-abi.h>
10
11#include "erdma.h"
12#include "erdma_cm.h"
13#include "erdma_verbs.h"
14
15MODULE_AUTHOR("Cheng Xu <chengyou@linux.alibaba.com>");
16MODULE_DESCRIPTION("Alibaba elasticRDMA adapter driver");
17MODULE_LICENSE("Dual BSD/GPL");
18
19static int erdma_netdev_event(struct notifier_block *nb, unsigned long event,
20 void *arg)
21{
22 struct net_device *netdev = netdev_notifier_info_to_dev(info: arg);
23 struct erdma_dev *dev = container_of(nb, struct erdma_dev, netdev_nb);
24
25 if (dev->netdev == NULL || dev->netdev != netdev)
26 goto done;
27
28 switch (event) {
29 case NETDEV_CHANGEMTU:
30 if (dev->mtu != netdev->mtu) {
31 erdma_set_mtu(dev, mtu: netdev->mtu);
32 dev->mtu = netdev->mtu;
33 }
34 break;
35 case NETDEV_REGISTER:
36 case NETDEV_UNREGISTER:
37 case NETDEV_CHANGEADDR:
38 case NETDEV_GOING_DOWN:
39 case NETDEV_CHANGE:
40 default:
41 break;
42 }
43
44done:
45 return NOTIFY_OK;
46}
47
48static int erdma_enum_and_get_netdev(struct erdma_dev *dev)
49{
50 struct net_device *netdev;
51 int ret = -EPROBE_DEFER;
52
53 /* Already binded to a net_device, so we skip. */
54 if (dev->netdev)
55 return 0;
56
57 rtnl_lock();
58 for_each_netdev(&init_net, netdev) {
59 /*
60 * In erdma, the paired netdev and ibdev should have the same
61 * MAC address. erdma can get the value from its PCIe bar
62 * registers. Since erdma can not get the paired netdev
63 * reference directly, we do a traverse here to get the paired
64 * netdev.
65 */
66 if (ether_addr_equal_unaligned(addr1: netdev->perm_addr,
67 addr2: dev->attrs.peer_addr)) {
68 ret = ib_device_set_netdev(ib_dev: &dev->ibdev, ndev: netdev, port: 1);
69 if (ret) {
70 rtnl_unlock();
71 ibdev_warn(ibdev: &dev->ibdev,
72 format: "failed (%d) to link netdev", ret);
73 return ret;
74 }
75
76 dev->netdev = netdev;
77 break;
78 }
79 }
80
81 rtnl_unlock();
82
83 return ret;
84}
85
86static int erdma_device_register(struct erdma_dev *dev)
87{
88 struct ib_device *ibdev = &dev->ibdev;
89 int ret;
90
91 ret = erdma_enum_and_get_netdev(dev);
92 if (ret)
93 return ret;
94
95 dev->mtu = dev->netdev->mtu;
96 addrconf_addr_eui48(eui: (u8 *)&ibdev->node_guid, addr: dev->netdev->dev_addr);
97
98 ret = ib_register_device(device: ibdev, name: "erdma_%d", dma_device: &dev->pdev->dev);
99 if (ret) {
100 dev_err(&dev->pdev->dev,
101 "ib_register_device failed: ret = %d\n", ret);
102 return ret;
103 }
104
105 dev->netdev_nb.notifier_call = erdma_netdev_event;
106 ret = register_netdevice_notifier(nb: &dev->netdev_nb);
107 if (ret) {
108 ibdev_err(ibdev: &dev->ibdev, format: "failed to register notifier.\n");
109 ib_unregister_device(device: ibdev);
110 }
111
112 return ret;
113}
114
115static irqreturn_t erdma_comm_irq_handler(int irq, void *data)
116{
117 struct erdma_dev *dev = data;
118
119 erdma_cmdq_completion_handler(cmdq: &dev->cmdq);
120 erdma_aeq_event_handler(dev);
121
122 return IRQ_HANDLED;
123}
124
125static int erdma_request_vectors(struct erdma_dev *dev)
126{
127 int expect_irq_num = min(num_possible_cpus() + 1, ERDMA_NUM_MSIX_VEC);
128 int ret;
129
130 ret = pci_alloc_irq_vectors(dev: dev->pdev, min_vecs: 1, max_vecs: expect_irq_num, PCI_IRQ_MSIX);
131 if (ret < 0) {
132 dev_err(&dev->pdev->dev, "request irq vectors failed(%d)\n",
133 ret);
134 return ret;
135 }
136 dev->attrs.irq_num = ret;
137
138 return 0;
139}
140
141static int erdma_comm_irq_init(struct erdma_dev *dev)
142{
143 snprintf(buf: dev->comm_irq.name, ERDMA_IRQNAME_SIZE, fmt: "erdma-common@pci:%s",
144 pci_name(pdev: dev->pdev));
145 dev->comm_irq.msix_vector =
146 pci_irq_vector(dev: dev->pdev, ERDMA_MSIX_VECTOR_CMDQ);
147
148 cpumask_set_cpu(cpu: cpumask_first(srcp: cpumask_of_pcibus(bus: dev->pdev->bus)),
149 dstp: &dev->comm_irq.affinity_hint_mask);
150 irq_set_affinity_hint(irq: dev->comm_irq.msix_vector,
151 m: &dev->comm_irq.affinity_hint_mask);
152
153 return request_irq(irq: dev->comm_irq.msix_vector, handler: erdma_comm_irq_handler, flags: 0,
154 name: dev->comm_irq.name, dev);
155}
156
157static void erdma_comm_irq_uninit(struct erdma_dev *dev)
158{
159 irq_set_affinity_hint(irq: dev->comm_irq.msix_vector, NULL);
160 free_irq(dev->comm_irq.msix_vector, dev);
161}
162
163static int erdma_device_init(struct erdma_dev *dev, struct pci_dev *pdev)
164{
165 int ret;
166
167 dev->proto = erdma_reg_read32(dev, ERDMA_REGS_DEV_PROTO_REG);
168
169 dev->resp_pool = dma_pool_create(name: "erdma_resp_pool", dev: &pdev->dev,
170 ERDMA_HW_RESP_SIZE, ERDMA_HW_RESP_SIZE,
171 boundary: 0);
172 if (!dev->resp_pool)
173 return -ENOMEM;
174
175 dev->db_pool = dma_pool_create(name: "erdma_db_pool", dev: &pdev->dev,
176 ERDMA_DB_SIZE, ERDMA_DB_SIZE, boundary: 0);
177 if (!dev->db_pool) {
178 ret = -ENOMEM;
179 goto destroy_resp_pool;
180 }
181
182 ret = dma_set_mask_and_coherent(dev: &pdev->dev,
183 DMA_BIT_MASK(ERDMA_PCI_WIDTH));
184 if (ret)
185 goto destroy_db_pool;
186
187 dma_set_max_seg_size(dev: &pdev->dev, UINT_MAX);
188
189 return 0;
190
191destroy_db_pool:
192 dma_pool_destroy(pool: dev->db_pool);
193
194destroy_resp_pool:
195 dma_pool_destroy(pool: dev->resp_pool);
196
197 return ret;
198}
199
200static void erdma_device_uninit(struct erdma_dev *dev)
201{
202 dma_pool_destroy(pool: dev->db_pool);
203 dma_pool_destroy(pool: dev->resp_pool);
204}
205
206static void erdma_hw_reset(struct erdma_dev *dev)
207{
208 u32 ctrl = FIELD_PREP(ERDMA_REG_DEV_CTRL_RESET_MASK, 1);
209
210 erdma_reg_write32(dev, ERDMA_REGS_DEV_CTRL_REG, value: ctrl);
211}
212
213static int erdma_wait_hw_init_done(struct erdma_dev *dev)
214{
215 int i;
216
217 erdma_reg_write32(dev, ERDMA_REGS_DEV_CTRL_REG,
218 FIELD_PREP(ERDMA_REG_DEV_CTRL_INIT_MASK, 1));
219
220 for (i = 0; i < ERDMA_WAIT_DEV_DONE_CNT; i++) {
221 if (erdma_reg_read32_filed(dev, ERDMA_REGS_DEV_ST_REG,
222 ERDMA_REG_DEV_ST_INIT_DONE_MASK))
223 break;
224
225 msleep(ERDMA_REG_ACCESS_WAIT_MS);
226 }
227
228 if (i == ERDMA_WAIT_DEV_DONE_CNT) {
229 dev_err(&dev->pdev->dev, "wait init done failed.\n");
230 return -ETIMEDOUT;
231 }
232
233 return 0;
234}
235
236static const struct pci_device_id erdma_pci_tbl[] = {
237 { PCI_DEVICE(PCI_VENDOR_ID_ALIBABA, 0x107f) },
238 {}
239};
240
241static int erdma_probe_dev(struct pci_dev *pdev)
242{
243 struct erdma_dev *dev;
244 int bars, err;
245 u32 version;
246
247 err = pci_enable_device(dev: pdev);
248 if (err) {
249 dev_err(&pdev->dev, "pci_enable_device failed(%d)\n", err);
250 return err;
251 }
252
253 pci_set_master(dev: pdev);
254
255 dev = ib_alloc_device(erdma_dev, ibdev);
256 if (!dev) {
257 dev_err(&pdev->dev, "ib_alloc_device failed\n");
258 err = -ENOMEM;
259 goto err_disable_device;
260 }
261
262 pci_set_drvdata(pdev, data: dev);
263 dev->pdev = pdev;
264 dev->attrs.numa_node = dev_to_node(dev: &pdev->dev);
265
266 bars = pci_select_bars(dev: pdev, IORESOURCE_MEM);
267 err = pci_request_selected_regions(pdev, bars, DRV_MODULE_NAME);
268 if (bars != ERDMA_BAR_MASK || err) {
269 err = err ? err : -EINVAL;
270 goto err_ib_device_release;
271 }
272
273 dev->func_bar_addr = pci_resource_start(pdev, ERDMA_FUNC_BAR);
274 dev->func_bar_len = pci_resource_len(pdev, ERDMA_FUNC_BAR);
275
276 dev->func_bar =
277 devm_ioremap(dev: &pdev->dev, offset: dev->func_bar_addr, size: dev->func_bar_len);
278 if (!dev->func_bar) {
279 dev_err(&pdev->dev, "devm_ioremap failed.\n");
280 err = -EFAULT;
281 goto err_release_bars;
282 }
283
284 version = erdma_reg_read32(dev, ERDMA_REGS_VERSION_REG);
285 if (version == 0) {
286 /* we knows that it is a non-functional function. */
287 err = -ENODEV;
288 goto err_iounmap_func_bar;
289 }
290
291 err = erdma_device_init(dev, pdev);
292 if (err)
293 goto err_iounmap_func_bar;
294
295 err = erdma_request_vectors(dev);
296 if (err)
297 goto err_uninit_device;
298
299 err = erdma_comm_irq_init(dev);
300 if (err)
301 goto err_free_vectors;
302
303 err = erdma_aeq_init(dev);
304 if (err)
305 goto err_uninit_comm_irq;
306
307 err = erdma_cmdq_init(dev);
308 if (err)
309 goto err_uninit_aeq;
310
311 err = erdma_wait_hw_init_done(dev);
312 if (err)
313 goto err_uninit_cmdq;
314
315 err = erdma_ceqs_init(dev);
316 if (err)
317 goto err_reset_hw;
318
319 erdma_finish_cmdq_init(dev);
320
321 return 0;
322
323err_reset_hw:
324 erdma_hw_reset(dev);
325
326err_uninit_cmdq:
327 erdma_cmdq_destroy(dev);
328
329err_uninit_aeq:
330 erdma_eq_destroy(dev, eq: &dev->aeq);
331
332err_uninit_comm_irq:
333 erdma_comm_irq_uninit(dev);
334
335err_free_vectors:
336 pci_free_irq_vectors(dev: dev->pdev);
337
338err_uninit_device:
339 erdma_device_uninit(dev);
340
341err_iounmap_func_bar:
342 devm_iounmap(dev: &pdev->dev, addr: dev->func_bar);
343
344err_release_bars:
345 pci_release_selected_regions(pdev, bars);
346
347err_ib_device_release:
348 ib_dealloc_device(device: &dev->ibdev);
349
350err_disable_device:
351 pci_disable_device(dev: pdev);
352
353 return err;
354}
355
356static void erdma_remove_dev(struct pci_dev *pdev)
357{
358 struct erdma_dev *dev = pci_get_drvdata(pdev);
359
360 erdma_ceqs_uninit(dev);
361 erdma_hw_reset(dev);
362 erdma_cmdq_destroy(dev);
363 erdma_eq_destroy(dev, eq: &dev->aeq);
364 erdma_comm_irq_uninit(dev);
365 pci_free_irq_vectors(dev: dev->pdev);
366 erdma_device_uninit(dev);
367
368 devm_iounmap(dev: &pdev->dev, addr: dev->func_bar);
369 pci_release_selected_regions(pdev, ERDMA_BAR_MASK);
370
371 ib_dealloc_device(device: &dev->ibdev);
372
373 pci_disable_device(dev: pdev);
374}
375
376#define ERDMA_GET_CAP(name, cap) FIELD_GET(ERDMA_CMD_DEV_CAP_##name##_MASK, cap)
377
378static int erdma_dev_attrs_init(struct erdma_dev *dev)
379{
380 int err;
381 u64 req_hdr, cap0, cap1;
382
383 erdma_cmdq_build_reqhdr(hdr: &req_hdr, mod: CMDQ_SUBMOD_RDMA,
384 op: CMDQ_OPCODE_QUERY_DEVICE);
385
386 err = erdma_post_cmd_wait(cmdq: &dev->cmdq, req: &req_hdr, req_size: sizeof(req_hdr), resp0: &cap0,
387 resp1: &cap1, sleepable: true);
388 if (err)
389 return err;
390
391 dev->attrs.max_cqe = 1 << ERDMA_GET_CAP(MAX_CQE, cap0);
392 dev->attrs.max_mr_size = 1ULL << ERDMA_GET_CAP(MAX_MR_SIZE, cap0);
393 dev->attrs.max_mw = 1 << ERDMA_GET_CAP(MAX_MW, cap1);
394 dev->attrs.max_recv_wr = 1 << ERDMA_GET_CAP(MAX_RECV_WR, cap0);
395 dev->attrs.max_gid = 1 << ERDMA_GET_CAP(MAX_GID, cap0);
396 dev->attrs.max_ah = 1 << ERDMA_GET_CAP(MAX_AH, cap0);
397 dev->attrs.local_dma_key = ERDMA_GET_CAP(DMA_LOCAL_KEY, cap1);
398 dev->attrs.cc = ERDMA_GET_CAP(DEFAULT_CC, cap1);
399 dev->attrs.max_qp = ERDMA_NQP_PER_QBLOCK * ERDMA_GET_CAP(QBLOCK, cap1);
400 dev->attrs.max_mr = dev->attrs.max_qp << 1;
401 dev->attrs.max_cq = dev->attrs.max_qp << 1;
402 dev->attrs.cap_flags = ERDMA_GET_CAP(FLAGS, cap0);
403
404 dev->attrs.max_send_wr = ERDMA_MAX_SEND_WR;
405 dev->attrs.max_ord = ERDMA_MAX_ORD;
406 dev->attrs.max_ird = ERDMA_MAX_IRD;
407 dev->attrs.max_send_sge = ERDMA_MAX_SEND_SGE;
408 dev->attrs.max_recv_sge = ERDMA_MAX_RECV_SGE;
409 dev->attrs.max_sge_rd = ERDMA_MAX_SGE_RD;
410 dev->attrs.max_pd = ERDMA_MAX_PD;
411
412 dev->res_cb[ERDMA_RES_TYPE_PD].max_cap = ERDMA_MAX_PD;
413 dev->res_cb[ERDMA_RES_TYPE_STAG_IDX].max_cap = dev->attrs.max_mr;
414 dev->res_cb[ERDMA_RES_TYPE_AH].max_cap = dev->attrs.max_ah;
415
416 erdma_cmdq_build_reqhdr(hdr: &req_hdr, mod: CMDQ_SUBMOD_COMMON,
417 op: CMDQ_OPCODE_QUERY_FW_INFO);
418
419 err = erdma_post_cmd_wait(cmdq: &dev->cmdq, req: &req_hdr, req_size: sizeof(req_hdr), resp0: &cap0,
420 resp1: &cap1, sleepable: true);
421 if (!err)
422 dev->attrs.fw_version =
423 FIELD_GET(ERDMA_CMD_INFO0_FW_VER_MASK, cap0);
424
425 return err;
426}
427
428static int erdma_device_config(struct erdma_dev *dev)
429{
430 struct erdma_cmdq_config_device_req req = {};
431
432 if (!(dev->attrs.cap_flags & ERDMA_DEV_CAP_FLAGS_EXTEND_DB))
433 return 0;
434
435 erdma_cmdq_build_reqhdr(hdr: &req.hdr, mod: CMDQ_SUBMOD_COMMON,
436 op: CMDQ_OPCODE_CONF_DEVICE);
437
438 req.cfg = FIELD_PREP(ERDMA_CMD_CONFIG_DEVICE_PGSHIFT_MASK, PAGE_SHIFT) |
439 FIELD_PREP(ERDMA_CMD_CONFIG_DEVICE_PS_EN_MASK, 1);
440
441 return erdma_post_cmd_wait(cmdq: &dev->cmdq, req: &req, req_size: sizeof(req), NULL, NULL,
442 sleepable: true);
443}
444
445static int erdma_res_cb_init(struct erdma_dev *dev)
446{
447 int i, j;
448
449 for (i = 0; i < ERDMA_RES_CNT; i++) {
450 dev->res_cb[i].next_alloc_idx = 1;
451 spin_lock_init(&dev->res_cb[i].lock);
452 dev->res_cb[i].bitmap =
453 bitmap_zalloc(nbits: dev->res_cb[i].max_cap, GFP_KERNEL);
454 if (!dev->res_cb[i].bitmap)
455 goto err;
456 }
457
458 return 0;
459
460err:
461 for (j = 0; j < i; j++)
462 bitmap_free(bitmap: dev->res_cb[j].bitmap);
463
464 return -ENOMEM;
465}
466
467static void erdma_res_cb_free(struct erdma_dev *dev)
468{
469 int i;
470
471 for (i = 0; i < ERDMA_RES_CNT; i++)
472 bitmap_free(bitmap: dev->res_cb[i].bitmap);
473}
474
475static const struct ib_device_ops erdma_device_ops_rocev2 = {
476 .get_link_layer = erdma_get_link_layer,
477 .add_gid = erdma_add_gid,
478 .del_gid = erdma_del_gid,
479 .query_pkey = erdma_query_pkey,
480 .create_ah = erdma_create_ah,
481 .destroy_ah = erdma_destroy_ah,
482 .query_ah = erdma_query_ah,
483
484 INIT_RDMA_OBJ_SIZE(ib_ah, erdma_ah, ibah),
485};
486
487static const struct ib_device_ops erdma_device_ops_iwarp = {
488 .iw_accept = erdma_accept,
489 .iw_add_ref = erdma_qp_get_ref,
490 .iw_connect = erdma_connect,
491 .iw_create_listen = erdma_create_listen,
492 .iw_destroy_listen = erdma_destroy_listen,
493 .iw_get_qp = erdma_get_ibqp,
494 .iw_reject = erdma_reject,
495 .iw_rem_ref = erdma_qp_put_ref,
496};
497
498static const struct ib_device_ops erdma_device_ops = {
499 .owner = THIS_MODULE,
500 .driver_id = RDMA_DRIVER_ERDMA,
501 .uverbs_abi_ver = ERDMA_ABI_VERSION,
502
503 .alloc_hw_port_stats = erdma_alloc_hw_port_stats,
504 .alloc_mr = erdma_ib_alloc_mr,
505 .alloc_pd = erdma_alloc_pd,
506 .alloc_ucontext = erdma_alloc_ucontext,
507 .create_cq = erdma_create_cq,
508 .create_qp = erdma_create_qp,
509 .dealloc_pd = erdma_dealloc_pd,
510 .dealloc_ucontext = erdma_dealloc_ucontext,
511 .dereg_mr = erdma_dereg_mr,
512 .destroy_cq = erdma_destroy_cq,
513 .destroy_qp = erdma_destroy_qp,
514 .disassociate_ucontext = erdma_disassociate_ucontext,
515 .get_dma_mr = erdma_get_dma_mr,
516 .get_hw_stats = erdma_get_hw_stats,
517 .get_port_immutable = erdma_get_port_immutable,
518 .map_mr_sg = erdma_map_mr_sg,
519 .mmap = erdma_mmap,
520 .mmap_free = erdma_mmap_free,
521 .post_recv = erdma_post_recv,
522 .post_send = erdma_post_send,
523 .poll_cq = erdma_poll_cq,
524 .query_device = erdma_query_device,
525 .query_gid = erdma_query_gid,
526 .query_port = erdma_query_port,
527 .query_qp = erdma_query_qp,
528 .req_notify_cq = erdma_req_notify_cq,
529 .reg_user_mr = erdma_reg_user_mr,
530 .modify_qp = erdma_modify_qp,
531
532 INIT_RDMA_OBJ_SIZE(ib_cq, erdma_cq, ibcq),
533 INIT_RDMA_OBJ_SIZE(ib_pd, erdma_pd, ibpd),
534 INIT_RDMA_OBJ_SIZE(ib_ucontext, erdma_ucontext, ibucontext),
535 INIT_RDMA_OBJ_SIZE(ib_qp, erdma_qp, ibqp),
536};
537
538static int erdma_ib_device_add(struct pci_dev *pdev)
539{
540 struct erdma_dev *dev = pci_get_drvdata(pdev);
541 struct ib_device *ibdev = &dev->ibdev;
542 u64 mac;
543 int ret;
544
545 ret = erdma_dev_attrs_init(dev);
546 if (ret)
547 return ret;
548
549 ret = erdma_device_config(dev);
550 if (ret)
551 return ret;
552
553 if (erdma_device_iwarp(dev)) {
554 ibdev->node_type = RDMA_NODE_RNIC;
555 ib_set_device_ops(device: ibdev, ops: &erdma_device_ops_iwarp);
556 } else {
557 ibdev->node_type = RDMA_NODE_IB_CA;
558 ib_set_device_ops(device: ibdev, ops: &erdma_device_ops_rocev2);
559 }
560
561 memcpy(ibdev->node_desc, ERDMA_NODE_DESC, sizeof(ERDMA_NODE_DESC));
562
563 /*
564 * Current model (one-to-one device association):
565 * One ERDMA device per net_device or, equivalently,
566 * per physical port.
567 */
568 ibdev->phys_port_cnt = 1;
569 ibdev->num_comp_vectors = dev->attrs.irq_num - 1;
570
571 ib_set_device_ops(device: ibdev, ops: &erdma_device_ops);
572
573 INIT_LIST_HEAD(list: &dev->cep_list);
574
575 spin_lock_init(&dev->lock);
576 xa_init_flags(xa: &dev->qp_xa, XA_FLAGS_ALLOC1);
577 xa_init_flags(xa: &dev->cq_xa, XA_FLAGS_ALLOC1);
578 dev->next_alloc_cqn = 1;
579 dev->next_alloc_qpn = 1;
580
581 ret = erdma_res_cb_init(dev);
582 if (ret)
583 return ret;
584
585 atomic_set(v: &dev->num_ctx, i: 0);
586
587 mac = erdma_reg_read32(dev, ERDMA_REGS_NETDEV_MAC_L_REG);
588 mac |= (u64)erdma_reg_read32(dev, ERDMA_REGS_NETDEV_MAC_H_REG) << 32;
589
590 u64_to_ether_addr(u: mac, addr: dev->attrs.peer_addr);
591
592 dev->reflush_wq = alloc_workqueue("erdma-reflush-wq", WQ_UNBOUND,
593 WQ_UNBOUND_MAX_ACTIVE);
594 if (!dev->reflush_wq) {
595 ret = -ENOMEM;
596 goto err_alloc_workqueue;
597 }
598
599 ret = erdma_device_register(dev);
600 if (ret)
601 goto err_register;
602
603 return 0;
604
605err_register:
606 destroy_workqueue(wq: dev->reflush_wq);
607err_alloc_workqueue:
608 xa_destroy(&dev->qp_xa);
609 xa_destroy(&dev->cq_xa);
610
611 erdma_res_cb_free(dev);
612
613 return ret;
614}
615
616static void erdma_ib_device_remove(struct pci_dev *pdev)
617{
618 struct erdma_dev *dev = pci_get_drvdata(pdev);
619
620 unregister_netdevice_notifier(nb: &dev->netdev_nb);
621 ib_unregister_device(device: &dev->ibdev);
622
623 destroy_workqueue(wq: dev->reflush_wq);
624 erdma_res_cb_free(dev);
625 xa_destroy(&dev->qp_xa);
626 xa_destroy(&dev->cq_xa);
627}
628
629static int erdma_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
630{
631 int ret;
632
633 ret = erdma_probe_dev(pdev);
634 if (ret)
635 return ret;
636
637 ret = erdma_ib_device_add(pdev);
638 if (ret) {
639 erdma_remove_dev(pdev);
640 return ret;
641 }
642
643 return 0;
644}
645
646static void erdma_remove(struct pci_dev *pdev)
647{
648 erdma_ib_device_remove(pdev);
649 erdma_remove_dev(pdev);
650}
651
652static struct pci_driver erdma_pci_driver = {
653 .name = DRV_MODULE_NAME,
654 .id_table = erdma_pci_tbl,
655 .probe = erdma_probe,
656 .remove = erdma_remove
657};
658
659MODULE_DEVICE_TABLE(pci, erdma_pci_tbl);
660
661static __init int erdma_init_module(void)
662{
663 int ret;
664
665 ret = erdma_cm_init();
666 if (ret)
667 return ret;
668
669 ret = pci_register_driver(&erdma_pci_driver);
670 if (ret)
671 erdma_cm_exit();
672
673 return ret;
674}
675
676static void __exit erdma_exit_module(void)
677{
678 pci_unregister_driver(dev: &erdma_pci_driver);
679
680 erdma_cm_exit();
681}
682
683module_init(erdma_init_module);
684module_exit(erdma_exit_module);
685

source code of linux/drivers/infiniband/hw/erdma/erdma_main.c