1 | // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB |
2 | /* Copyright (c) 2015 - 2021 Intel Corporation */ |
3 | #include "main.h" |
4 | |
5 | /** |
6 | * irdma_arp_table -manage arp table |
7 | * @rf: RDMA PCI function |
8 | * @ip_addr: ip address for device |
9 | * @ipv4: IPv4 flag |
10 | * @mac_addr: mac address ptr |
11 | * @action: modify, delete or add |
12 | */ |
13 | int irdma_arp_table(struct irdma_pci_f *rf, u32 *ip_addr, bool ipv4, |
14 | const u8 *mac_addr, u32 action) |
15 | { |
16 | unsigned long flags; |
17 | int arp_index; |
18 | u32 ip[4] = {}; |
19 | |
20 | if (ipv4) |
21 | ip[0] = *ip_addr; |
22 | else |
23 | memcpy(ip, ip_addr, sizeof(ip)); |
24 | |
25 | spin_lock_irqsave(&rf->arp_lock, flags); |
26 | for (arp_index = 0; (u32)arp_index < rf->arp_table_size; arp_index++) { |
27 | if (!memcmp(p: rf->arp_table[arp_index].ip_addr, q: ip, size: sizeof(ip))) |
28 | break; |
29 | } |
30 | |
31 | switch (action) { |
32 | case IRDMA_ARP_ADD: |
33 | if (arp_index != rf->arp_table_size) { |
34 | arp_index = -1; |
35 | break; |
36 | } |
37 | |
38 | arp_index = 0; |
39 | if (irdma_alloc_rsrc(rf, rsrc_array: rf->allocated_arps, max_rsrc: rf->arp_table_size, |
40 | req_rsrc_num: (u32 *)&arp_index, next: &rf->next_arp_index)) { |
41 | arp_index = -1; |
42 | break; |
43 | } |
44 | |
45 | memcpy(rf->arp_table[arp_index].ip_addr, ip, |
46 | sizeof(rf->arp_table[arp_index].ip_addr)); |
47 | ether_addr_copy(dst: rf->arp_table[arp_index].mac_addr, src: mac_addr); |
48 | break; |
49 | case IRDMA_ARP_RESOLVE: |
50 | if (arp_index == rf->arp_table_size) |
51 | arp_index = -1; |
52 | break; |
53 | case IRDMA_ARP_DELETE: |
54 | if (arp_index == rf->arp_table_size) { |
55 | arp_index = -1; |
56 | break; |
57 | } |
58 | |
59 | memset(rf->arp_table[arp_index].ip_addr, 0, |
60 | sizeof(rf->arp_table[arp_index].ip_addr)); |
61 | eth_zero_addr(addr: rf->arp_table[arp_index].mac_addr); |
62 | irdma_free_rsrc(rf, rsrc_array: rf->allocated_arps, rsrc_num: arp_index); |
63 | break; |
64 | default: |
65 | arp_index = -1; |
66 | break; |
67 | } |
68 | |
69 | spin_unlock_irqrestore(lock: &rf->arp_lock, flags); |
70 | return arp_index; |
71 | } |
72 | |
73 | /** |
74 | * irdma_add_arp - add a new arp entry if needed |
75 | * @rf: RDMA function |
76 | * @ip: IP address |
77 | * @ipv4: IPv4 flag |
78 | * @mac: MAC address |
79 | */ |
80 | int irdma_add_arp(struct irdma_pci_f *rf, u32 *ip, bool ipv4, const u8 *mac) |
81 | { |
82 | int arpidx; |
83 | |
84 | arpidx = irdma_arp_table(rf, ip_addr: &ip[0], ipv4, NULL, IRDMA_ARP_RESOLVE); |
85 | if (arpidx >= 0) { |
86 | if (ether_addr_equal(addr1: rf->arp_table[arpidx].mac_addr, addr2: mac)) |
87 | return arpidx; |
88 | |
89 | irdma_manage_arp_cache(rf, mac_addr: rf->arp_table[arpidx].mac_addr, ip_addr: ip, |
90 | ipv4, IRDMA_ARP_DELETE); |
91 | } |
92 | |
93 | irdma_manage_arp_cache(rf, mac_addr: mac, ip_addr: ip, ipv4, IRDMA_ARP_ADD); |
94 | |
95 | return irdma_arp_table(rf, ip_addr: ip, ipv4, NULL, IRDMA_ARP_RESOLVE); |
96 | } |
97 | |
98 | /** |
99 | * wr32 - write 32 bits to hw register |
100 | * @hw: hardware information including registers |
101 | * @reg: register offset |
102 | * @val: value to write to register |
103 | */ |
104 | inline void wr32(struct irdma_hw *hw, u32 reg, u32 val) |
105 | { |
106 | writel(val, addr: hw->hw_addr + reg); |
107 | } |
108 | |
109 | /** |
110 | * rd32 - read a 32 bit hw register |
111 | * @hw: hardware information including registers |
112 | * @reg: register offset |
113 | * |
114 | * Return value of register content |
115 | */ |
116 | inline u32 rd32(struct irdma_hw *hw, u32 reg) |
117 | { |
118 | return readl(addr: hw->hw_addr + reg); |
119 | } |
120 | |
121 | /** |
122 | * rd64 - read a 64 bit hw register |
123 | * @hw: hardware information including registers |
124 | * @reg: register offset |
125 | * |
126 | * Return value of register content |
127 | */ |
128 | inline u64 rd64(struct irdma_hw *hw, u32 reg) |
129 | { |
130 | return readq(addr: hw->hw_addr + reg); |
131 | } |
132 | |
133 | static void irdma_gid_change_event(struct ib_device *ibdev) |
134 | { |
135 | struct ib_event ib_event; |
136 | |
137 | ib_event.event = IB_EVENT_GID_CHANGE; |
138 | ib_event.device = ibdev; |
139 | ib_event.element.port_num = 1; |
140 | ib_dispatch_event(event: &ib_event); |
141 | } |
142 | |
143 | /** |
144 | * irdma_inetaddr_event - system notifier for ipv4 addr events |
145 | * @notifier: not used |
146 | * @event: event for notifier |
147 | * @ptr: if address |
148 | */ |
149 | int irdma_inetaddr_event(struct notifier_block *notifier, unsigned long event, |
150 | void *ptr) |
151 | { |
152 | struct in_ifaddr *ifa = ptr; |
153 | struct net_device *real_dev, *netdev = ifa->ifa_dev->dev; |
154 | struct irdma_device *iwdev; |
155 | struct ib_device *ibdev; |
156 | u32 local_ipaddr; |
157 | |
158 | real_dev = rdma_vlan_dev_real_dev(dev: netdev); |
159 | if (!real_dev) |
160 | real_dev = netdev; |
161 | |
162 | ibdev = ib_device_get_by_netdev(ndev: real_dev, driver_id: RDMA_DRIVER_IRDMA); |
163 | if (!ibdev) |
164 | return NOTIFY_DONE; |
165 | |
166 | iwdev = to_iwdev(ibdev); |
167 | local_ipaddr = ntohl(ifa->ifa_address); |
168 | ibdev_dbg(&iwdev->ibdev, |
169 | "DEV: netdev %p event %lu local_ip=%pI4 MAC=%pM\n" , real_dev, |
170 | event, &local_ipaddr, real_dev->dev_addr); |
171 | switch (event) { |
172 | case NETDEV_DOWN: |
173 | irdma_manage_arp_cache(rf: iwdev->rf, mac_addr: real_dev->dev_addr, |
174 | ip_addr: &local_ipaddr, ipv4: true, IRDMA_ARP_DELETE); |
175 | irdma_if_notify(iwdev, netdev: real_dev, ipaddr: &local_ipaddr, ipv4: true, ifup: false); |
176 | irdma_gid_change_event(ibdev: &iwdev->ibdev); |
177 | break; |
178 | case NETDEV_UP: |
179 | case NETDEV_CHANGEADDR: |
180 | irdma_add_arp(rf: iwdev->rf, ip: &local_ipaddr, ipv4: true, mac: real_dev->dev_addr); |
181 | irdma_if_notify(iwdev, netdev: real_dev, ipaddr: &local_ipaddr, ipv4: true, ifup: true); |
182 | irdma_gid_change_event(ibdev: &iwdev->ibdev); |
183 | break; |
184 | default: |
185 | break; |
186 | } |
187 | |
188 | ib_device_put(device: ibdev); |
189 | |
190 | return NOTIFY_DONE; |
191 | } |
192 | |
193 | /** |
194 | * irdma_inet6addr_event - system notifier for ipv6 addr events |
195 | * @notifier: not used |
196 | * @event: event for notifier |
197 | * @ptr: if address |
198 | */ |
199 | int irdma_inet6addr_event(struct notifier_block *notifier, unsigned long event, |
200 | void *ptr) |
201 | { |
202 | struct inet6_ifaddr *ifa = ptr; |
203 | struct net_device *real_dev, *netdev = ifa->idev->dev; |
204 | struct irdma_device *iwdev; |
205 | struct ib_device *ibdev; |
206 | u32 local_ipaddr6[4]; |
207 | |
208 | real_dev = rdma_vlan_dev_real_dev(dev: netdev); |
209 | if (!real_dev) |
210 | real_dev = netdev; |
211 | |
212 | ibdev = ib_device_get_by_netdev(ndev: real_dev, driver_id: RDMA_DRIVER_IRDMA); |
213 | if (!ibdev) |
214 | return NOTIFY_DONE; |
215 | |
216 | iwdev = to_iwdev(ibdev); |
217 | irdma_copy_ip_ntohl(dst: local_ipaddr6, src: ifa->addr.in6_u.u6_addr32); |
218 | ibdev_dbg(&iwdev->ibdev, |
219 | "DEV: netdev %p event %lu local_ip=%pI6 MAC=%pM\n" , real_dev, |
220 | event, local_ipaddr6, real_dev->dev_addr); |
221 | switch (event) { |
222 | case NETDEV_DOWN: |
223 | irdma_manage_arp_cache(rf: iwdev->rf, mac_addr: real_dev->dev_addr, |
224 | ip_addr: local_ipaddr6, ipv4: false, IRDMA_ARP_DELETE); |
225 | irdma_if_notify(iwdev, netdev: real_dev, ipaddr: local_ipaddr6, ipv4: false, ifup: false); |
226 | irdma_gid_change_event(ibdev: &iwdev->ibdev); |
227 | break; |
228 | case NETDEV_UP: |
229 | case NETDEV_CHANGEADDR: |
230 | irdma_add_arp(rf: iwdev->rf, ip: local_ipaddr6, ipv4: false, |
231 | mac: real_dev->dev_addr); |
232 | irdma_if_notify(iwdev, netdev: real_dev, ipaddr: local_ipaddr6, ipv4: false, ifup: true); |
233 | irdma_gid_change_event(ibdev: &iwdev->ibdev); |
234 | break; |
235 | default: |
236 | break; |
237 | } |
238 | |
239 | ib_device_put(device: ibdev); |
240 | |
241 | return NOTIFY_DONE; |
242 | } |
243 | |
244 | /** |
245 | * irdma_net_event - system notifier for net events |
246 | * @notifier: not used |
247 | * @event: event for notifier |
248 | * @ptr: neighbor |
249 | */ |
250 | int irdma_net_event(struct notifier_block *notifier, unsigned long event, |
251 | void *ptr) |
252 | { |
253 | struct neighbour *neigh = ptr; |
254 | struct net_device *real_dev, *netdev = (struct net_device *)neigh->dev; |
255 | struct irdma_device *iwdev; |
256 | struct ib_device *ibdev; |
257 | __be32 *p; |
258 | u32 local_ipaddr[4] = {}; |
259 | bool ipv4 = true; |
260 | |
261 | switch (event) { |
262 | case NETEVENT_NEIGH_UPDATE: |
263 | real_dev = rdma_vlan_dev_real_dev(dev: netdev); |
264 | if (!real_dev) |
265 | real_dev = netdev; |
266 | ibdev = ib_device_get_by_netdev(ndev: real_dev, driver_id: RDMA_DRIVER_IRDMA); |
267 | if (!ibdev) |
268 | return NOTIFY_DONE; |
269 | |
270 | iwdev = to_iwdev(ibdev); |
271 | p = (__be32 *)neigh->primary_key; |
272 | if (neigh->tbl->family == AF_INET6) { |
273 | ipv4 = false; |
274 | irdma_copy_ip_ntohl(dst: local_ipaddr, src: p); |
275 | } else { |
276 | local_ipaddr[0] = ntohl(*p); |
277 | } |
278 | |
279 | ibdev_dbg(&iwdev->ibdev, |
280 | "DEV: netdev %p state %d local_ip=%pI4 MAC=%pM\n" , |
281 | iwdev->netdev, neigh->nud_state, local_ipaddr, |
282 | neigh->ha); |
283 | |
284 | if (neigh->nud_state & NUD_VALID) |
285 | irdma_add_arp(rf: iwdev->rf, ip: local_ipaddr, ipv4, mac: neigh->ha); |
286 | |
287 | else |
288 | irdma_manage_arp_cache(rf: iwdev->rf, mac_addr: neigh->ha, |
289 | ip_addr: local_ipaddr, ipv4, |
290 | IRDMA_ARP_DELETE); |
291 | ib_device_put(device: ibdev); |
292 | break; |
293 | default: |
294 | break; |
295 | } |
296 | |
297 | return NOTIFY_DONE; |
298 | } |
299 | |
300 | /** |
301 | * irdma_netdevice_event - system notifier for netdev events |
302 | * @notifier: not used |
303 | * @event: event for notifier |
304 | * @ptr: netdev |
305 | */ |
306 | int irdma_netdevice_event(struct notifier_block *notifier, unsigned long event, |
307 | void *ptr) |
308 | { |
309 | struct irdma_device *iwdev; |
310 | struct ib_device *ibdev; |
311 | struct net_device *netdev = netdev_notifier_info_to_dev(info: ptr); |
312 | |
313 | ibdev = ib_device_get_by_netdev(ndev: netdev, driver_id: RDMA_DRIVER_IRDMA); |
314 | if (!ibdev) |
315 | return NOTIFY_DONE; |
316 | |
317 | iwdev = to_iwdev(ibdev); |
318 | iwdev->iw_status = 1; |
319 | switch (event) { |
320 | case NETDEV_DOWN: |
321 | iwdev->iw_status = 0; |
322 | fallthrough; |
323 | case NETDEV_UP: |
324 | irdma_port_ibevent(iwdev); |
325 | break; |
326 | default: |
327 | break; |
328 | } |
329 | ib_device_put(device: ibdev); |
330 | |
331 | return NOTIFY_DONE; |
332 | } |
333 | |
334 | /** |
335 | * irdma_add_ipv6_addr - add ipv6 address to the hw arp table |
336 | * @iwdev: irdma device |
337 | */ |
338 | static void irdma_add_ipv6_addr(struct irdma_device *iwdev) |
339 | { |
340 | struct net_device *ip_dev; |
341 | struct inet6_dev *idev; |
342 | struct inet6_ifaddr *ifp, *tmp; |
343 | u32 local_ipaddr6[4]; |
344 | |
345 | rcu_read_lock(); |
346 | for_each_netdev_rcu (&init_net, ip_dev) { |
347 | if (((rdma_vlan_dev_vlan_id(dev: ip_dev) < 0xFFFF && |
348 | rdma_vlan_dev_real_dev(dev: ip_dev) == iwdev->netdev) || |
349 | ip_dev == iwdev->netdev) && |
350 | (READ_ONCE(ip_dev->flags) & IFF_UP)) { |
351 | idev = __in6_dev_get(dev: ip_dev); |
352 | if (!idev) { |
353 | ibdev_err(ibdev: &iwdev->ibdev, format: "ipv6 inet device not found\n" ); |
354 | break; |
355 | } |
356 | list_for_each_entry_safe (ifp, tmp, &idev->addr_list, |
357 | if_list) { |
358 | ibdev_dbg(&iwdev->ibdev, |
359 | "INIT: IP=%pI6, vlan_id=%d, MAC=%pM\n" , |
360 | &ifp->addr, |
361 | rdma_vlan_dev_vlan_id(ip_dev), |
362 | ip_dev->dev_addr); |
363 | |
364 | irdma_copy_ip_ntohl(dst: local_ipaddr6, |
365 | src: ifp->addr.in6_u.u6_addr32); |
366 | irdma_manage_arp_cache(rf: iwdev->rf, |
367 | mac_addr: ip_dev->dev_addr, |
368 | ip_addr: local_ipaddr6, ipv4: false, |
369 | IRDMA_ARP_ADD); |
370 | } |
371 | } |
372 | } |
373 | rcu_read_unlock(); |
374 | } |
375 | |
376 | /** |
377 | * irdma_add_ipv4_addr - add ipv4 address to the hw arp table |
378 | * @iwdev: irdma device |
379 | */ |
380 | static void irdma_add_ipv4_addr(struct irdma_device *iwdev) |
381 | { |
382 | struct net_device *dev; |
383 | struct in_device *idev; |
384 | u32 ip_addr; |
385 | |
386 | rcu_read_lock(); |
387 | for_each_netdev_rcu (&init_net, dev) { |
388 | if (((rdma_vlan_dev_vlan_id(dev) < 0xFFFF && |
389 | rdma_vlan_dev_real_dev(dev) == iwdev->netdev) || |
390 | dev == iwdev->netdev) && (READ_ONCE(dev->flags) & IFF_UP)) { |
391 | const struct in_ifaddr *ifa; |
392 | |
393 | idev = __in_dev_get_rcu(dev); |
394 | if (!idev) |
395 | continue; |
396 | |
397 | in_dev_for_each_ifa_rcu(ifa, idev) { |
398 | ibdev_dbg(&iwdev->ibdev, "CM: IP=%pI4, vlan_id=%d, MAC=%pM\n" , |
399 | &ifa->ifa_address, rdma_vlan_dev_vlan_id(dev), |
400 | dev->dev_addr); |
401 | |
402 | ip_addr = ntohl(ifa->ifa_address); |
403 | irdma_manage_arp_cache(rf: iwdev->rf, mac_addr: dev->dev_addr, |
404 | ip_addr: &ip_addr, ipv4: true, |
405 | IRDMA_ARP_ADD); |
406 | } |
407 | } |
408 | } |
409 | rcu_read_unlock(); |
410 | } |
411 | |
412 | /** |
413 | * irdma_add_ip - add ip addresses |
414 | * @iwdev: irdma device |
415 | * |
416 | * Add ipv4/ipv6 addresses to the arp cache |
417 | */ |
418 | void irdma_add_ip(struct irdma_device *iwdev) |
419 | { |
420 | irdma_add_ipv4_addr(iwdev); |
421 | irdma_add_ipv6_addr(iwdev); |
422 | } |
423 | |
424 | /** |
425 | * irdma_alloc_and_get_cqp_request - get cqp struct |
426 | * @cqp: device cqp ptr |
427 | * @wait: cqp to be used in wait mode |
428 | */ |
429 | struct irdma_cqp_request *irdma_alloc_and_get_cqp_request(struct irdma_cqp *cqp, |
430 | bool wait) |
431 | { |
432 | struct irdma_cqp_request *cqp_request = NULL; |
433 | unsigned long flags; |
434 | |
435 | spin_lock_irqsave(&cqp->req_lock, flags); |
436 | if (!list_empty(head: &cqp->cqp_avail_reqs)) { |
437 | cqp_request = list_first_entry(&cqp->cqp_avail_reqs, |
438 | struct irdma_cqp_request, list); |
439 | list_del_init(entry: &cqp_request->list); |
440 | } |
441 | spin_unlock_irqrestore(lock: &cqp->req_lock, flags); |
442 | if (!cqp_request) { |
443 | cqp_request = kzalloc(size: sizeof(*cqp_request), GFP_ATOMIC); |
444 | if (cqp_request) { |
445 | cqp_request->dynamic = true; |
446 | if (wait) |
447 | init_waitqueue_head(&cqp_request->waitq); |
448 | } |
449 | } |
450 | if (!cqp_request) { |
451 | ibdev_dbg(to_ibdev(cqp->sc_cqp.dev), "ERR: CQP Request Fail: No Memory" ); |
452 | return NULL; |
453 | } |
454 | |
455 | cqp_request->waiting = wait; |
456 | refcount_set(r: &cqp_request->refcnt, n: 1); |
457 | memset(&cqp_request->compl_info, 0, sizeof(cqp_request->compl_info)); |
458 | |
459 | return cqp_request; |
460 | } |
461 | |
462 | /** |
463 | * irdma_get_cqp_request - increase refcount for cqp_request |
464 | * @cqp_request: pointer to cqp_request instance |
465 | */ |
466 | static inline void irdma_get_cqp_request(struct irdma_cqp_request *cqp_request) |
467 | { |
468 | refcount_inc(r: &cqp_request->refcnt); |
469 | } |
470 | |
471 | /** |
472 | * irdma_free_cqp_request - free cqp request |
473 | * @cqp: cqp ptr |
474 | * @cqp_request: to be put back in cqp list |
475 | */ |
476 | void irdma_free_cqp_request(struct irdma_cqp *cqp, |
477 | struct irdma_cqp_request *cqp_request) |
478 | { |
479 | unsigned long flags; |
480 | |
481 | if (cqp_request->dynamic) { |
482 | kfree(objp: cqp_request); |
483 | } else { |
484 | WRITE_ONCE(cqp_request->request_done, false); |
485 | cqp_request->callback_fcn = NULL; |
486 | cqp_request->waiting = false; |
487 | |
488 | spin_lock_irqsave(&cqp->req_lock, flags); |
489 | list_add_tail(new: &cqp_request->list, head: &cqp->cqp_avail_reqs); |
490 | spin_unlock_irqrestore(lock: &cqp->req_lock, flags); |
491 | } |
492 | wake_up(&cqp->remove_wq); |
493 | } |
494 | |
495 | /** |
496 | * irdma_put_cqp_request - dec ref count and free if 0 |
497 | * @cqp: cqp ptr |
498 | * @cqp_request: to be put back in cqp list |
499 | */ |
500 | void irdma_put_cqp_request(struct irdma_cqp *cqp, |
501 | struct irdma_cqp_request *cqp_request) |
502 | { |
503 | if (refcount_dec_and_test(r: &cqp_request->refcnt)) |
504 | irdma_free_cqp_request(cqp, cqp_request); |
505 | } |
506 | |
507 | /** |
508 | * irdma_free_pending_cqp_request -free pending cqp request objs |
509 | * @cqp: cqp ptr |
510 | * @cqp_request: to be put back in cqp list |
511 | */ |
512 | static void |
513 | irdma_free_pending_cqp_request(struct irdma_cqp *cqp, |
514 | struct irdma_cqp_request *cqp_request) |
515 | { |
516 | if (cqp_request->waiting) { |
517 | cqp_request->compl_info.error = true; |
518 | WRITE_ONCE(cqp_request->request_done, true); |
519 | wake_up(&cqp_request->waitq); |
520 | } |
521 | wait_event_timeout(cqp->remove_wq, |
522 | refcount_read(&cqp_request->refcnt) == 1, 1000); |
523 | irdma_put_cqp_request(cqp, cqp_request); |
524 | } |
525 | |
526 | /** |
527 | * irdma_cleanup_pending_cqp_op - clean-up cqp with no |
528 | * completions |
529 | * @rf: RDMA PCI function |
530 | */ |
531 | void irdma_cleanup_pending_cqp_op(struct irdma_pci_f *rf) |
532 | { |
533 | struct irdma_sc_dev *dev = &rf->sc_dev; |
534 | struct irdma_cqp *cqp = &rf->cqp; |
535 | struct irdma_cqp_request *cqp_request = NULL; |
536 | struct cqp_cmds_info *pcmdinfo = NULL; |
537 | u32 i, pending_work, wqe_idx; |
538 | |
539 | pending_work = IRDMA_RING_USED_QUANTA(cqp->sc_cqp.sq_ring); |
540 | wqe_idx = IRDMA_RING_CURRENT_TAIL(cqp->sc_cqp.sq_ring); |
541 | for (i = 0; i < pending_work; i++) { |
542 | cqp_request = (struct irdma_cqp_request *)(unsigned long) |
543 | cqp->scratch_array[wqe_idx]; |
544 | if (cqp_request) |
545 | irdma_free_pending_cqp_request(cqp, cqp_request); |
546 | wqe_idx = (wqe_idx + 1) % IRDMA_RING_SIZE(cqp->sc_cqp.sq_ring); |
547 | } |
548 | |
549 | while (!list_empty(head: &dev->cqp_cmd_head)) { |
550 | pcmdinfo = irdma_remove_cqp_head(dev); |
551 | cqp_request = |
552 | container_of(pcmdinfo, struct irdma_cqp_request, info); |
553 | if (cqp_request) |
554 | irdma_free_pending_cqp_request(cqp, cqp_request); |
555 | } |
556 | } |
557 | |
558 | /** |
559 | * irdma_wait_event - wait for completion |
560 | * @rf: RDMA PCI function |
561 | * @cqp_request: cqp request to wait |
562 | */ |
563 | static int irdma_wait_event(struct irdma_pci_f *rf, |
564 | struct irdma_cqp_request *cqp_request) |
565 | { |
566 | struct irdma_cqp_timeout cqp_timeout = {}; |
567 | bool cqp_error = false; |
568 | int err_code = 0; |
569 | |
570 | cqp_timeout.compl_cqp_cmds = atomic64_read(v: &rf->sc_dev.cqp->completed_ops); |
571 | do { |
572 | irdma_cqp_ce_handler(rf, cq: &rf->ccq.sc_cq); |
573 | if (wait_event_timeout(cqp_request->waitq, |
574 | READ_ONCE(cqp_request->request_done), |
575 | msecs_to_jiffies(CQP_COMPL_WAIT_TIME_MS))) |
576 | break; |
577 | |
578 | irdma_check_cqp_progress(cqp_timeout: &cqp_timeout, dev: &rf->sc_dev); |
579 | |
580 | if (cqp_timeout.count < CQP_TIMEOUT_THRESHOLD) |
581 | continue; |
582 | |
583 | if (!rf->reset) { |
584 | rf->reset = true; |
585 | rf->gen_ops.request_reset(rf); |
586 | } |
587 | return -ETIMEDOUT; |
588 | } while (1); |
589 | |
590 | cqp_error = cqp_request->compl_info.error; |
591 | if (cqp_error) { |
592 | err_code = -EIO; |
593 | if (cqp_request->compl_info.maj_err_code == 0xFFFF) { |
594 | if (cqp_request->compl_info.min_err_code == 0x8002) |
595 | err_code = -EBUSY; |
596 | else if (cqp_request->compl_info.min_err_code == 0x8029) { |
597 | if (!rf->reset) { |
598 | rf->reset = true; |
599 | rf->gen_ops.request_reset(rf); |
600 | } |
601 | } |
602 | } |
603 | } |
604 | |
605 | return err_code; |
606 | } |
607 | |
608 | static const char *const irdma_cqp_cmd_names[IRDMA_MAX_CQP_OPS] = { |
609 | [IRDMA_OP_CEQ_DESTROY] = "Destroy CEQ Cmd" , |
610 | [IRDMA_OP_AEQ_DESTROY] = "Destroy AEQ Cmd" , |
611 | [IRDMA_OP_DELETE_ARP_CACHE_ENTRY] = "Delete ARP Cache Cmd" , |
612 | [IRDMA_OP_MANAGE_APBVT_ENTRY] = "Manage APBV Table Entry Cmd" , |
613 | [IRDMA_OP_CEQ_CREATE] = "CEQ Create Cmd" , |
614 | [IRDMA_OP_AEQ_CREATE] = "AEQ Destroy Cmd" , |
615 | [IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY] = "Manage Quad Hash Table Entry Cmd" , |
616 | [IRDMA_OP_QP_MODIFY] = "Modify QP Cmd" , |
617 | [IRDMA_OP_QP_UPLOAD_CONTEXT] = "Upload Context Cmd" , |
618 | [IRDMA_OP_CQ_CREATE] = "Create CQ Cmd" , |
619 | [IRDMA_OP_CQ_DESTROY] = "Destroy CQ Cmd" , |
620 | [IRDMA_OP_QP_CREATE] = "Create QP Cmd" , |
621 | [IRDMA_OP_QP_DESTROY] = "Destroy QP Cmd" , |
622 | [IRDMA_OP_ALLOC_STAG] = "Allocate STag Cmd" , |
623 | [IRDMA_OP_MR_REG_NON_SHARED] = "Register Non-Shared MR Cmd" , |
624 | [IRDMA_OP_DEALLOC_STAG] = "Deallocate STag Cmd" , |
625 | [IRDMA_OP_MW_ALLOC] = "Allocate Memory Window Cmd" , |
626 | [IRDMA_OP_QP_FLUSH_WQES] = "Flush QP Cmd" , |
627 | [IRDMA_OP_ADD_ARP_CACHE_ENTRY] = "Add ARP Cache Cmd" , |
628 | [IRDMA_OP_MANAGE_PUSH_PAGE] = "Manage Push Page Cmd" , |
629 | [IRDMA_OP_UPDATE_PE_SDS] = "Update PE SDs Cmd" , |
630 | [IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE] = "Manage HMC PM Function Table Cmd" , |
631 | [IRDMA_OP_SUSPEND] = "Suspend QP Cmd" , |
632 | [IRDMA_OP_RESUME] = "Resume QP Cmd" , |
633 | [IRDMA_OP_MANAGE_VF_PBLE_BP] = "Manage VF PBLE Backing Pages Cmd" , |
634 | [IRDMA_OP_QUERY_FPM_VAL] = "Query FPM Values Cmd" , |
635 | [IRDMA_OP_COMMIT_FPM_VAL] = "Commit FPM Values Cmd" , |
636 | [IRDMA_OP_AH_CREATE] = "Create Address Handle Cmd" , |
637 | [IRDMA_OP_AH_MODIFY] = "Modify Address Handle Cmd" , |
638 | [IRDMA_OP_AH_DESTROY] = "Destroy Address Handle Cmd" , |
639 | [IRDMA_OP_MC_CREATE] = "Create Multicast Group Cmd" , |
640 | [IRDMA_OP_MC_DESTROY] = "Destroy Multicast Group Cmd" , |
641 | [IRDMA_OP_MC_MODIFY] = "Modify Multicast Group Cmd" , |
642 | [IRDMA_OP_STATS_ALLOCATE] = "Add Statistics Instance Cmd" , |
643 | [IRDMA_OP_STATS_FREE] = "Free Statistics Instance Cmd" , |
644 | [IRDMA_OP_STATS_GATHER] = "Gather Statistics Cmd" , |
645 | [IRDMA_OP_WS_ADD_NODE] = "Add Work Scheduler Node Cmd" , |
646 | [IRDMA_OP_WS_MODIFY_NODE] = "Modify Work Scheduler Node Cmd" , |
647 | [IRDMA_OP_WS_DELETE_NODE] = "Delete Work Scheduler Node Cmd" , |
648 | [IRDMA_OP_SET_UP_MAP] = "Set UP-UP Mapping Cmd" , |
649 | [IRDMA_OP_GEN_AE] = "Generate AE Cmd" , |
650 | [IRDMA_OP_QUERY_RDMA_FEATURES] = "RDMA Get Features Cmd" , |
651 | [IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY] = "Allocate Local MAC Entry Cmd" , |
652 | [IRDMA_OP_ADD_LOCAL_MAC_ENTRY] = "Add Local MAC Entry Cmd" , |
653 | [IRDMA_OP_DELETE_LOCAL_MAC_ENTRY] = "Delete Local MAC Entry Cmd" , |
654 | [IRDMA_OP_CQ_MODIFY] = "CQ Modify Cmd" , |
655 | }; |
656 | |
657 | static const struct irdma_cqp_err_info irdma_noncrit_err_list[] = { |
658 | {0xffff, 0x8002, "Invalid State" }, |
659 | {0xffff, 0x8006, "Flush No Wqe Pending" }, |
660 | {0xffff, 0x8007, "Modify QP Bad Close" }, |
661 | {0xffff, 0x8009, "LLP Closed" }, |
662 | {0xffff, 0x800a, "Reset Not Sent" } |
663 | }; |
664 | |
665 | /** |
666 | * irdma_cqp_crit_err - check if CQP error is critical |
667 | * @dev: pointer to dev structure |
668 | * @cqp_cmd: code for last CQP operation |
669 | * @maj_err_code: major error code |
670 | * @min_err_code: minot error code |
671 | */ |
672 | bool irdma_cqp_crit_err(struct irdma_sc_dev *dev, u8 cqp_cmd, |
673 | u16 maj_err_code, u16 min_err_code) |
674 | { |
675 | int i; |
676 | |
677 | for (i = 0; i < ARRAY_SIZE(irdma_noncrit_err_list); ++i) { |
678 | if (maj_err_code == irdma_noncrit_err_list[i].maj && |
679 | min_err_code == irdma_noncrit_err_list[i].min) { |
680 | ibdev_dbg(to_ibdev(dev), |
681 | "CQP: [%s Error][%s] maj=0x%x min=0x%x\n" , |
682 | irdma_noncrit_err_list[i].desc, |
683 | irdma_cqp_cmd_names[cqp_cmd], maj_err_code, |
684 | min_err_code); |
685 | return false; |
686 | } |
687 | } |
688 | return true; |
689 | } |
690 | |
691 | /** |
692 | * irdma_handle_cqp_op - process cqp command |
693 | * @rf: RDMA PCI function |
694 | * @cqp_request: cqp request to process |
695 | */ |
696 | int irdma_handle_cqp_op(struct irdma_pci_f *rf, |
697 | struct irdma_cqp_request *cqp_request) |
698 | { |
699 | struct irdma_sc_dev *dev = &rf->sc_dev; |
700 | struct cqp_cmds_info *info = &cqp_request->info; |
701 | int status; |
702 | bool put_cqp_request = true; |
703 | |
704 | if (rf->reset) |
705 | return -EBUSY; |
706 | |
707 | irdma_get_cqp_request(cqp_request); |
708 | status = irdma_process_cqp_cmd(dev, pcmdinfo: info); |
709 | if (status) |
710 | goto err; |
711 | |
712 | if (cqp_request->waiting) { |
713 | put_cqp_request = false; |
714 | status = irdma_wait_event(rf, cqp_request); |
715 | if (status) |
716 | goto err; |
717 | } |
718 | |
719 | return 0; |
720 | |
721 | err: |
722 | if (irdma_cqp_crit_err(dev, cqp_cmd: info->cqp_cmd, |
723 | maj_err_code: cqp_request->compl_info.maj_err_code, |
724 | min_err_code: cqp_request->compl_info.min_err_code)) |
725 | ibdev_err(ibdev: &rf->iwdev->ibdev, |
726 | format: "[%s Error][op_code=%d] status=%d waiting=%d completion_err=%d maj=0x%x min=0x%x\n" , |
727 | irdma_cqp_cmd_names[info->cqp_cmd], info->cqp_cmd, status, cqp_request->waiting, |
728 | cqp_request->compl_info.error, cqp_request->compl_info.maj_err_code, |
729 | cqp_request->compl_info.min_err_code); |
730 | |
731 | if (put_cqp_request) |
732 | irdma_put_cqp_request(cqp: &rf->cqp, cqp_request); |
733 | |
734 | return status; |
735 | } |
736 | |
737 | void irdma_qp_add_ref(struct ib_qp *ibqp) |
738 | { |
739 | struct irdma_qp *iwqp = (struct irdma_qp *)ibqp; |
740 | |
741 | refcount_inc(r: &iwqp->refcnt); |
742 | } |
743 | |
744 | void irdma_qp_rem_ref(struct ib_qp *ibqp) |
745 | { |
746 | struct irdma_qp *iwqp = to_iwqp(ibqp); |
747 | struct irdma_device *iwdev = iwqp->iwdev; |
748 | u32 qp_num; |
749 | unsigned long flags; |
750 | |
751 | spin_lock_irqsave(&iwdev->rf->qptable_lock, flags); |
752 | if (!refcount_dec_and_test(r: &iwqp->refcnt)) { |
753 | spin_unlock_irqrestore(lock: &iwdev->rf->qptable_lock, flags); |
754 | return; |
755 | } |
756 | |
757 | qp_num = iwqp->ibqp.qp_num; |
758 | iwdev->rf->qp_table[qp_num] = NULL; |
759 | spin_unlock_irqrestore(lock: &iwdev->rf->qptable_lock, flags); |
760 | complete(&iwqp->free_qp); |
761 | } |
762 | |
763 | void irdma_cq_add_ref(struct ib_cq *ibcq) |
764 | { |
765 | struct irdma_cq *iwcq = to_iwcq(ibcq); |
766 | |
767 | refcount_inc(r: &iwcq->refcnt); |
768 | } |
769 | |
770 | void irdma_cq_rem_ref(struct ib_cq *ibcq) |
771 | { |
772 | struct ib_device *ibdev = ibcq->device; |
773 | struct irdma_device *iwdev = to_iwdev(ibdev); |
774 | struct irdma_cq *iwcq = to_iwcq(ibcq); |
775 | unsigned long flags; |
776 | |
777 | spin_lock_irqsave(&iwdev->rf->cqtable_lock, flags); |
778 | if (!refcount_dec_and_test(r: &iwcq->refcnt)) { |
779 | spin_unlock_irqrestore(lock: &iwdev->rf->cqtable_lock, flags); |
780 | return; |
781 | } |
782 | |
783 | iwdev->rf->cq_table[iwcq->cq_num] = NULL; |
784 | spin_unlock_irqrestore(lock: &iwdev->rf->cqtable_lock, flags); |
785 | complete(&iwcq->free_cq); |
786 | } |
787 | |
788 | struct ib_device *to_ibdev(struct irdma_sc_dev *dev) |
789 | { |
790 | return &(container_of(dev, struct irdma_pci_f, sc_dev))->iwdev->ibdev; |
791 | } |
792 | |
793 | /** |
794 | * irdma_get_qp - get qp address |
795 | * @device: iwarp device |
796 | * @qpn: qp number |
797 | */ |
798 | struct ib_qp *irdma_get_qp(struct ib_device *device, int qpn) |
799 | { |
800 | struct irdma_device *iwdev = to_iwdev(ibdev: device); |
801 | |
802 | if (qpn < IW_FIRST_QPN || qpn >= iwdev->rf->max_qp) |
803 | return NULL; |
804 | |
805 | return &iwdev->rf->qp_table[qpn]->ibqp; |
806 | } |
807 | |
808 | /** |
809 | * irdma_remove_cqp_head - return head entry and remove |
810 | * @dev: device |
811 | */ |
812 | void *irdma_remove_cqp_head(struct irdma_sc_dev *dev) |
813 | { |
814 | struct list_head *entry; |
815 | struct list_head *list = &dev->cqp_cmd_head; |
816 | |
817 | if (list_empty(head: list)) |
818 | return NULL; |
819 | |
820 | entry = list->next; |
821 | list_del(entry); |
822 | |
823 | return entry; |
824 | } |
825 | |
826 | /** |
827 | * irdma_cqp_sds_cmd - create cqp command for sd |
828 | * @dev: hardware control device structure |
829 | * @sdinfo: information for sd cqp |
830 | * |
831 | */ |
832 | int irdma_cqp_sds_cmd(struct irdma_sc_dev *dev, |
833 | struct irdma_update_sds_info *sdinfo) |
834 | { |
835 | struct irdma_cqp_request *cqp_request; |
836 | struct cqp_cmds_info *cqp_info; |
837 | struct irdma_pci_f *rf = dev_to_rf(dev); |
838 | int status; |
839 | |
840 | cqp_request = irdma_alloc_and_get_cqp_request(cqp: &rf->cqp, wait: true); |
841 | if (!cqp_request) |
842 | return -ENOMEM; |
843 | |
844 | cqp_info = &cqp_request->info; |
845 | memcpy(&cqp_info->in.u.update_pe_sds.info, sdinfo, |
846 | sizeof(cqp_info->in.u.update_pe_sds.info)); |
847 | cqp_info->cqp_cmd = IRDMA_OP_UPDATE_PE_SDS; |
848 | cqp_info->post_sq = 1; |
849 | cqp_info->in.u.update_pe_sds.dev = dev; |
850 | cqp_info->in.u.update_pe_sds.scratch = (uintptr_t)cqp_request; |
851 | |
852 | status = irdma_handle_cqp_op(rf, cqp_request); |
853 | irdma_put_cqp_request(cqp: &rf->cqp, cqp_request); |
854 | |
855 | return status; |
856 | } |
857 | |
858 | /** |
859 | * irdma_cqp_qp_suspend_resume - cqp command for suspend/resume |
860 | * @qp: hardware control qp |
861 | * @op: suspend or resume |
862 | */ |
863 | int irdma_cqp_qp_suspend_resume(struct irdma_sc_qp *qp, u8 op) |
864 | { |
865 | struct irdma_sc_dev *dev = qp->dev; |
866 | struct irdma_cqp_request *cqp_request; |
867 | struct irdma_sc_cqp *cqp = dev->cqp; |
868 | struct cqp_cmds_info *cqp_info; |
869 | struct irdma_pci_f *rf = dev_to_rf(dev); |
870 | int status; |
871 | |
872 | cqp_request = irdma_alloc_and_get_cqp_request(cqp: &rf->cqp, wait: false); |
873 | if (!cqp_request) |
874 | return -ENOMEM; |
875 | |
876 | cqp_info = &cqp_request->info; |
877 | cqp_info->cqp_cmd = op; |
878 | cqp_info->in.u.suspend_resume.cqp = cqp; |
879 | cqp_info->in.u.suspend_resume.qp = qp; |
880 | cqp_info->in.u.suspend_resume.scratch = (uintptr_t)cqp_request; |
881 | |
882 | status = irdma_handle_cqp_op(rf, cqp_request); |
883 | irdma_put_cqp_request(cqp: &rf->cqp, cqp_request); |
884 | |
885 | return status; |
886 | } |
887 | |
888 | /** |
889 | * irdma_term_modify_qp - modify qp for term message |
890 | * @qp: hardware control qp |
891 | * @next_state: qp's next state |
892 | * @term: terminate code |
893 | * @term_len: length |
894 | */ |
895 | void irdma_term_modify_qp(struct irdma_sc_qp *qp, u8 next_state, u8 term, |
896 | u8 term_len) |
897 | { |
898 | struct irdma_qp *iwqp; |
899 | |
900 | iwqp = qp->qp_uk.back_qp; |
901 | irdma_next_iw_state(iwqp, state: next_state, del_hash: 0, term, term_len); |
902 | }; |
903 | |
904 | /** |
905 | * irdma_terminate_done - after terminate is completed |
906 | * @qp: hardware control qp |
907 | * @timeout_occurred: indicates if terminate timer expired |
908 | */ |
909 | void irdma_terminate_done(struct irdma_sc_qp *qp, int timeout_occurred) |
910 | { |
911 | struct irdma_qp *iwqp; |
912 | u8 hte = 0; |
913 | bool first_time; |
914 | unsigned long flags; |
915 | |
916 | iwqp = qp->qp_uk.back_qp; |
917 | spin_lock_irqsave(&iwqp->lock, flags); |
918 | if (iwqp->hte_added) { |
919 | iwqp->hte_added = 0; |
920 | hte = 1; |
921 | } |
922 | first_time = !(qp->term_flags & IRDMA_TERM_DONE); |
923 | qp->term_flags |= IRDMA_TERM_DONE; |
924 | spin_unlock_irqrestore(lock: &iwqp->lock, flags); |
925 | if (first_time) { |
926 | if (!timeout_occurred) |
927 | irdma_terminate_del_timer(qp); |
928 | |
929 | irdma_next_iw_state(iwqp, IRDMA_QP_STATE_ERROR, del_hash: hte, term: 0, term_len: 0); |
930 | irdma_cm_disconn(qp: iwqp); |
931 | } |
932 | } |
933 | |
934 | static void irdma_terminate_timeout(struct timer_list *t) |
935 | { |
936 | struct irdma_qp *iwqp = from_timer(iwqp, t, terminate_timer); |
937 | struct irdma_sc_qp *qp = &iwqp->sc_qp; |
938 | |
939 | irdma_terminate_done(qp, timeout_occurred: 1); |
940 | irdma_qp_rem_ref(ibqp: &iwqp->ibqp); |
941 | } |
942 | |
943 | /** |
944 | * irdma_terminate_start_timer - start terminate timeout |
945 | * @qp: hardware control qp |
946 | */ |
947 | void irdma_terminate_start_timer(struct irdma_sc_qp *qp) |
948 | { |
949 | struct irdma_qp *iwqp; |
950 | |
951 | iwqp = qp->qp_uk.back_qp; |
952 | irdma_qp_add_ref(ibqp: &iwqp->ibqp); |
953 | timer_setup(&iwqp->terminate_timer, irdma_terminate_timeout, 0); |
954 | iwqp->terminate_timer.expires = jiffies + HZ; |
955 | |
956 | add_timer(timer: &iwqp->terminate_timer); |
957 | } |
958 | |
959 | /** |
960 | * irdma_terminate_del_timer - delete terminate timeout |
961 | * @qp: hardware control qp |
962 | */ |
963 | void irdma_terminate_del_timer(struct irdma_sc_qp *qp) |
964 | { |
965 | struct irdma_qp *iwqp; |
966 | int ret; |
967 | |
968 | iwqp = qp->qp_uk.back_qp; |
969 | ret = del_timer(timer: &iwqp->terminate_timer); |
970 | if (ret) |
971 | irdma_qp_rem_ref(ibqp: &iwqp->ibqp); |
972 | } |
973 | |
974 | /** |
975 | * irdma_cqp_query_fpm_val_cmd - send cqp command for fpm |
976 | * @dev: function device struct |
977 | * @val_mem: buffer for fpm |
978 | * @hmc_fn_id: function id for fpm |
979 | */ |
980 | int irdma_cqp_query_fpm_val_cmd(struct irdma_sc_dev *dev, |
981 | struct irdma_dma_mem *val_mem, u8 hmc_fn_id) |
982 | { |
983 | struct irdma_cqp_request *cqp_request; |
984 | struct cqp_cmds_info *cqp_info; |
985 | struct irdma_pci_f *rf = dev_to_rf(dev); |
986 | int status; |
987 | |
988 | cqp_request = irdma_alloc_and_get_cqp_request(cqp: &rf->cqp, wait: true); |
989 | if (!cqp_request) |
990 | return -ENOMEM; |
991 | |
992 | cqp_info = &cqp_request->info; |
993 | cqp_request->param = NULL; |
994 | cqp_info->in.u.query_fpm_val.cqp = dev->cqp; |
995 | cqp_info->in.u.query_fpm_val.fpm_val_pa = val_mem->pa; |
996 | cqp_info->in.u.query_fpm_val.fpm_val_va = val_mem->va; |
997 | cqp_info->in.u.query_fpm_val.hmc_fn_id = hmc_fn_id; |
998 | cqp_info->cqp_cmd = IRDMA_OP_QUERY_FPM_VAL; |
999 | cqp_info->post_sq = 1; |
1000 | cqp_info->in.u.query_fpm_val.scratch = (uintptr_t)cqp_request; |
1001 | |
1002 | status = irdma_handle_cqp_op(rf, cqp_request); |
1003 | irdma_put_cqp_request(cqp: &rf->cqp, cqp_request); |
1004 | |
1005 | return status; |
1006 | } |
1007 | |
1008 | /** |
1009 | * irdma_cqp_commit_fpm_val_cmd - commit fpm values in hw |
1010 | * @dev: hardware control device structure |
1011 | * @val_mem: buffer with fpm values |
1012 | * @hmc_fn_id: function id for fpm |
1013 | */ |
1014 | int irdma_cqp_commit_fpm_val_cmd(struct irdma_sc_dev *dev, |
1015 | struct irdma_dma_mem *val_mem, u8 hmc_fn_id) |
1016 | { |
1017 | struct irdma_cqp_request *cqp_request; |
1018 | struct cqp_cmds_info *cqp_info; |
1019 | struct irdma_pci_f *rf = dev_to_rf(dev); |
1020 | int status; |
1021 | |
1022 | cqp_request = irdma_alloc_and_get_cqp_request(cqp: &rf->cqp, wait: true); |
1023 | if (!cqp_request) |
1024 | return -ENOMEM; |
1025 | |
1026 | cqp_info = &cqp_request->info; |
1027 | cqp_request->param = NULL; |
1028 | cqp_info->in.u.commit_fpm_val.cqp = dev->cqp; |
1029 | cqp_info->in.u.commit_fpm_val.fpm_val_pa = val_mem->pa; |
1030 | cqp_info->in.u.commit_fpm_val.fpm_val_va = val_mem->va; |
1031 | cqp_info->in.u.commit_fpm_val.hmc_fn_id = hmc_fn_id; |
1032 | cqp_info->cqp_cmd = IRDMA_OP_COMMIT_FPM_VAL; |
1033 | cqp_info->post_sq = 1; |
1034 | cqp_info->in.u.commit_fpm_val.scratch = (uintptr_t)cqp_request; |
1035 | |
1036 | status = irdma_handle_cqp_op(rf, cqp_request); |
1037 | irdma_put_cqp_request(cqp: &rf->cqp, cqp_request); |
1038 | |
1039 | return status; |
1040 | } |
1041 | |
1042 | /** |
1043 | * irdma_cqp_cq_create_cmd - create a cq for the cqp |
1044 | * @dev: device pointer |
1045 | * @cq: pointer to created cq |
1046 | */ |
1047 | int irdma_cqp_cq_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq) |
1048 | { |
1049 | struct irdma_pci_f *rf = dev_to_rf(dev); |
1050 | struct irdma_cqp *iwcqp = &rf->cqp; |
1051 | struct irdma_cqp_request *cqp_request; |
1052 | struct cqp_cmds_info *cqp_info; |
1053 | int status; |
1054 | |
1055 | cqp_request = irdma_alloc_and_get_cqp_request(cqp: iwcqp, wait: true); |
1056 | if (!cqp_request) |
1057 | return -ENOMEM; |
1058 | |
1059 | cqp_info = &cqp_request->info; |
1060 | cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE; |
1061 | cqp_info->post_sq = 1; |
1062 | cqp_info->in.u.cq_create.cq = cq; |
1063 | cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request; |
1064 | |
1065 | status = irdma_handle_cqp_op(rf, cqp_request); |
1066 | irdma_put_cqp_request(cqp: iwcqp, cqp_request); |
1067 | |
1068 | return status; |
1069 | } |
1070 | |
1071 | /** |
1072 | * irdma_cqp_qp_create_cmd - create a qp for the cqp |
1073 | * @dev: device pointer |
1074 | * @qp: pointer to created qp |
1075 | */ |
1076 | int irdma_cqp_qp_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp) |
1077 | { |
1078 | struct irdma_pci_f *rf = dev_to_rf(dev); |
1079 | struct irdma_cqp *iwcqp = &rf->cqp; |
1080 | struct irdma_cqp_request *cqp_request; |
1081 | struct cqp_cmds_info *cqp_info; |
1082 | struct irdma_create_qp_info *qp_info; |
1083 | int status; |
1084 | |
1085 | cqp_request = irdma_alloc_and_get_cqp_request(cqp: iwcqp, wait: true); |
1086 | if (!cqp_request) |
1087 | return -ENOMEM; |
1088 | |
1089 | cqp_info = &cqp_request->info; |
1090 | qp_info = &cqp_request->info.in.u.qp_create.info; |
1091 | memset(qp_info, 0, sizeof(*qp_info)); |
1092 | qp_info->cq_num_valid = true; |
1093 | qp_info->next_iwarp_state = IRDMA_QP_STATE_RTS; |
1094 | cqp_info->cqp_cmd = IRDMA_OP_QP_CREATE; |
1095 | cqp_info->post_sq = 1; |
1096 | cqp_info->in.u.qp_create.qp = qp; |
1097 | cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request; |
1098 | |
1099 | status = irdma_handle_cqp_op(rf, cqp_request); |
1100 | irdma_put_cqp_request(cqp: iwcqp, cqp_request); |
1101 | |
1102 | return status; |
1103 | } |
1104 | |
1105 | /** |
1106 | * irdma_dealloc_push_page - free a push page for qp |
1107 | * @rf: RDMA PCI function |
1108 | * @qp: hardware control qp |
1109 | */ |
1110 | static void irdma_dealloc_push_page(struct irdma_pci_f *rf, |
1111 | struct irdma_sc_qp *qp) |
1112 | { |
1113 | struct irdma_cqp_request *cqp_request; |
1114 | struct cqp_cmds_info *cqp_info; |
1115 | int status; |
1116 | |
1117 | if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) |
1118 | return; |
1119 | |
1120 | cqp_request = irdma_alloc_and_get_cqp_request(cqp: &rf->cqp, wait: false); |
1121 | if (!cqp_request) |
1122 | return; |
1123 | |
1124 | cqp_info = &cqp_request->info; |
1125 | cqp_info->cqp_cmd = IRDMA_OP_MANAGE_PUSH_PAGE; |
1126 | cqp_info->post_sq = 1; |
1127 | cqp_info->in.u.manage_push_page.info.push_idx = qp->push_idx; |
1128 | cqp_info->in.u.manage_push_page.info.qs_handle = qp->qs_handle; |
1129 | cqp_info->in.u.manage_push_page.info.free_page = 1; |
1130 | cqp_info->in.u.manage_push_page.info.push_page_type = 0; |
1131 | cqp_info->in.u.manage_push_page.cqp = &rf->cqp.sc_cqp; |
1132 | cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request; |
1133 | status = irdma_handle_cqp_op(rf, cqp_request); |
1134 | if (!status) |
1135 | qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX; |
1136 | irdma_put_cqp_request(cqp: &rf->cqp, cqp_request); |
1137 | } |
1138 | |
1139 | /** |
1140 | * irdma_free_qp_rsrc - free up memory resources for qp |
1141 | * @iwqp: qp ptr (user or kernel) |
1142 | */ |
1143 | void irdma_free_qp_rsrc(struct irdma_qp *iwqp) |
1144 | { |
1145 | struct irdma_device *iwdev = iwqp->iwdev; |
1146 | struct irdma_pci_f *rf = iwdev->rf; |
1147 | u32 qp_num = iwqp->ibqp.qp_num; |
1148 | |
1149 | irdma_ieq_cleanup_qp(ieq: iwdev->vsi.ieq, qp: &iwqp->sc_qp); |
1150 | irdma_dealloc_push_page(rf, qp: &iwqp->sc_qp); |
1151 | if (iwqp->sc_qp.vsi) { |
1152 | irdma_qp_rem_qos(qp: &iwqp->sc_qp); |
1153 | iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi, |
1154 | iwqp->sc_qp.user_pri); |
1155 | } |
1156 | |
1157 | if (qp_num > 2) |
1158 | irdma_free_rsrc(rf, rsrc_array: rf->allocated_qps, rsrc_num: qp_num); |
1159 | dma_free_coherent(dev: rf->sc_dev.hw->device, size: iwqp->q2_ctx_mem.size, |
1160 | cpu_addr: iwqp->q2_ctx_mem.va, dma_handle: iwqp->q2_ctx_mem.pa); |
1161 | iwqp->q2_ctx_mem.va = NULL; |
1162 | dma_free_coherent(dev: rf->sc_dev.hw->device, size: iwqp->kqp.dma_mem.size, |
1163 | cpu_addr: iwqp->kqp.dma_mem.va, dma_handle: iwqp->kqp.dma_mem.pa); |
1164 | iwqp->kqp.dma_mem.va = NULL; |
1165 | kfree(objp: iwqp->kqp.sq_wrid_mem); |
1166 | kfree(objp: iwqp->kqp.rq_wrid_mem); |
1167 | } |
1168 | |
1169 | /** |
1170 | * irdma_cq_wq_destroy - send cq destroy cqp |
1171 | * @rf: RDMA PCI function |
1172 | * @cq: hardware control cq |
1173 | */ |
1174 | void irdma_cq_wq_destroy(struct irdma_pci_f *rf, struct irdma_sc_cq *cq) |
1175 | { |
1176 | struct irdma_cqp_request *cqp_request; |
1177 | struct cqp_cmds_info *cqp_info; |
1178 | |
1179 | cqp_request = irdma_alloc_and_get_cqp_request(cqp: &rf->cqp, wait: true); |
1180 | if (!cqp_request) |
1181 | return; |
1182 | |
1183 | cqp_info = &cqp_request->info; |
1184 | cqp_info->cqp_cmd = IRDMA_OP_CQ_DESTROY; |
1185 | cqp_info->post_sq = 1; |
1186 | cqp_info->in.u.cq_destroy.cq = cq; |
1187 | cqp_info->in.u.cq_destroy.scratch = (uintptr_t)cqp_request; |
1188 | |
1189 | irdma_handle_cqp_op(rf, cqp_request); |
1190 | irdma_put_cqp_request(cqp: &rf->cqp, cqp_request); |
1191 | } |
1192 | |
1193 | /** |
1194 | * irdma_hw_modify_qp_callback - handle state for modifyQPs that don't wait |
1195 | * @cqp_request: modify QP completion |
1196 | */ |
1197 | static void irdma_hw_modify_qp_callback(struct irdma_cqp_request *cqp_request) |
1198 | { |
1199 | struct cqp_cmds_info *cqp_info; |
1200 | struct irdma_qp *iwqp; |
1201 | |
1202 | cqp_info = &cqp_request->info; |
1203 | iwqp = cqp_info->in.u.qp_modify.qp->qp_uk.back_qp; |
1204 | atomic_dec(v: &iwqp->hw_mod_qp_pend); |
1205 | wake_up(&iwqp->mod_qp_waitq); |
1206 | } |
1207 | |
1208 | /** |
1209 | * irdma_hw_modify_qp - setup cqp for modify qp |
1210 | * @iwdev: RDMA device |
1211 | * @iwqp: qp ptr (user or kernel) |
1212 | * @info: info for modify qp |
1213 | * @wait: flag to wait or not for modify qp completion |
1214 | */ |
1215 | int irdma_hw_modify_qp(struct irdma_device *iwdev, struct irdma_qp *iwqp, |
1216 | struct irdma_modify_qp_info *info, bool wait) |
1217 | { |
1218 | int status; |
1219 | struct irdma_pci_f *rf = iwdev->rf; |
1220 | struct irdma_cqp_request *cqp_request; |
1221 | struct cqp_cmds_info *cqp_info; |
1222 | struct irdma_modify_qp_info *m_info; |
1223 | |
1224 | cqp_request = irdma_alloc_and_get_cqp_request(cqp: &rf->cqp, wait); |
1225 | if (!cqp_request) |
1226 | return -ENOMEM; |
1227 | |
1228 | if (!wait) { |
1229 | cqp_request->callback_fcn = irdma_hw_modify_qp_callback; |
1230 | atomic_inc(v: &iwqp->hw_mod_qp_pend); |
1231 | } |
1232 | cqp_info = &cqp_request->info; |
1233 | m_info = &cqp_info->in.u.qp_modify.info; |
1234 | memcpy(m_info, info, sizeof(*m_info)); |
1235 | cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY; |
1236 | cqp_info->post_sq = 1; |
1237 | cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp; |
1238 | cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request; |
1239 | status = irdma_handle_cqp_op(rf, cqp_request); |
1240 | irdma_put_cqp_request(cqp: &rf->cqp, cqp_request); |
1241 | if (status) { |
1242 | if (rdma_protocol_roce(device: &iwdev->ibdev, port_num: 1)) |
1243 | return status; |
1244 | |
1245 | switch (m_info->next_iwarp_state) { |
1246 | struct irdma_gen_ae_info ae_info; |
1247 | |
1248 | case IRDMA_QP_STATE_RTS: |
1249 | case IRDMA_QP_STATE_IDLE: |
1250 | case IRDMA_QP_STATE_TERMINATE: |
1251 | case IRDMA_QP_STATE_CLOSING: |
1252 | if (info->curr_iwarp_state == IRDMA_QP_STATE_IDLE) |
1253 | irdma_send_reset(cm_node: iwqp->cm_node); |
1254 | else |
1255 | iwqp->sc_qp.term_flags = IRDMA_TERM_DONE; |
1256 | if (!wait) { |
1257 | ae_info.ae_code = IRDMA_AE_BAD_CLOSE; |
1258 | ae_info.ae_src = 0; |
1259 | irdma_gen_ae(rf, qp: &iwqp->sc_qp, info: &ae_info, wait: false); |
1260 | } else { |
1261 | cqp_request = irdma_alloc_and_get_cqp_request(cqp: &rf->cqp, |
1262 | wait); |
1263 | if (!cqp_request) |
1264 | return -ENOMEM; |
1265 | |
1266 | cqp_info = &cqp_request->info; |
1267 | m_info = &cqp_info->in.u.qp_modify.info; |
1268 | memcpy(m_info, info, sizeof(*m_info)); |
1269 | cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY; |
1270 | cqp_info->post_sq = 1; |
1271 | cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp; |
1272 | cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request; |
1273 | m_info->next_iwarp_state = IRDMA_QP_STATE_ERROR; |
1274 | m_info->reset_tcp_conn = true; |
1275 | irdma_handle_cqp_op(rf, cqp_request); |
1276 | irdma_put_cqp_request(cqp: &rf->cqp, cqp_request); |
1277 | } |
1278 | break; |
1279 | case IRDMA_QP_STATE_ERROR: |
1280 | default: |
1281 | break; |
1282 | } |
1283 | } |
1284 | |
1285 | return status; |
1286 | } |
1287 | |
1288 | /** |
1289 | * irdma_cqp_cq_destroy_cmd - destroy the cqp cq |
1290 | * @dev: device pointer |
1291 | * @cq: pointer to cq |
1292 | */ |
1293 | void irdma_cqp_cq_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq) |
1294 | { |
1295 | struct irdma_pci_f *rf = dev_to_rf(dev); |
1296 | |
1297 | irdma_cq_wq_destroy(rf, cq); |
1298 | } |
1299 | |
1300 | /** |
1301 | * irdma_cqp_qp_destroy_cmd - destroy the cqp |
1302 | * @dev: device pointer |
1303 | * @qp: pointer to qp |
1304 | */ |
1305 | int irdma_cqp_qp_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp) |
1306 | { |
1307 | struct irdma_pci_f *rf = dev_to_rf(dev); |
1308 | struct irdma_cqp *iwcqp = &rf->cqp; |
1309 | struct irdma_cqp_request *cqp_request; |
1310 | struct cqp_cmds_info *cqp_info; |
1311 | int status; |
1312 | |
1313 | cqp_request = irdma_alloc_and_get_cqp_request(cqp: iwcqp, wait: true); |
1314 | if (!cqp_request) |
1315 | return -ENOMEM; |
1316 | |
1317 | cqp_info = &cqp_request->info; |
1318 | memset(cqp_info, 0, sizeof(*cqp_info)); |
1319 | cqp_info->cqp_cmd = IRDMA_OP_QP_DESTROY; |
1320 | cqp_info->post_sq = 1; |
1321 | cqp_info->in.u.qp_destroy.qp = qp; |
1322 | cqp_info->in.u.qp_destroy.scratch = (uintptr_t)cqp_request; |
1323 | cqp_info->in.u.qp_destroy.remove_hash_idx = true; |
1324 | |
1325 | status = irdma_handle_cqp_op(rf, cqp_request); |
1326 | irdma_put_cqp_request(cqp: &rf->cqp, cqp_request); |
1327 | |
1328 | return status; |
1329 | } |
1330 | |
1331 | /** |
1332 | * irdma_ieq_mpa_crc_ae - generate AE for crc error |
1333 | * @dev: hardware control device structure |
1334 | * @qp: hardware control qp |
1335 | */ |
1336 | void irdma_ieq_mpa_crc_ae(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp) |
1337 | { |
1338 | struct irdma_gen_ae_info info = {}; |
1339 | struct irdma_pci_f *rf = dev_to_rf(dev); |
1340 | |
1341 | ibdev_dbg(&rf->iwdev->ibdev, "AEQ: Generate MPA CRC AE\n" ); |
1342 | info.ae_code = IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR; |
1343 | info.ae_src = IRDMA_AE_SOURCE_RQ; |
1344 | irdma_gen_ae(rf, qp, info: &info, wait: false); |
1345 | } |
1346 | |
1347 | /** |
1348 | * irdma_init_hash_desc - initialize hash for crc calculation |
1349 | * @desc: cryption type |
1350 | */ |
1351 | int irdma_init_hash_desc(struct shash_desc **desc) |
1352 | { |
1353 | struct crypto_shash *tfm; |
1354 | struct shash_desc *tdesc; |
1355 | |
1356 | tfm = crypto_alloc_shash(alg_name: "crc32c" , type: 0, mask: 0); |
1357 | if (IS_ERR(ptr: tfm)) |
1358 | return -EINVAL; |
1359 | |
1360 | tdesc = kzalloc(size: sizeof(*tdesc) + crypto_shash_descsize(tfm), |
1361 | GFP_KERNEL); |
1362 | if (!tdesc) { |
1363 | crypto_free_shash(tfm); |
1364 | return -EINVAL; |
1365 | } |
1366 | |
1367 | tdesc->tfm = tfm; |
1368 | *desc = tdesc; |
1369 | |
1370 | return 0; |
1371 | } |
1372 | |
1373 | /** |
1374 | * irdma_free_hash_desc - free hash desc |
1375 | * @desc: to be freed |
1376 | */ |
1377 | void irdma_free_hash_desc(struct shash_desc *desc) |
1378 | { |
1379 | if (desc) { |
1380 | crypto_free_shash(tfm: desc->tfm); |
1381 | kfree(objp: desc); |
1382 | } |
1383 | } |
1384 | |
1385 | /** |
1386 | * irdma_ieq_check_mpacrc - check if mpa crc is OK |
1387 | * @desc: desc for hash |
1388 | * @addr: address of buffer for crc |
1389 | * @len: length of buffer |
1390 | * @val: value to be compared |
1391 | */ |
1392 | int irdma_ieq_check_mpacrc(struct shash_desc *desc, void *addr, u32 len, |
1393 | u32 val) |
1394 | { |
1395 | u32 crc = 0; |
1396 | |
1397 | crypto_shash_digest(desc, data: addr, len, out: (u8 *)&crc); |
1398 | if (crc != val) |
1399 | return -EINVAL; |
1400 | |
1401 | return 0; |
1402 | } |
1403 | |
1404 | /** |
1405 | * irdma_ieq_get_qp - get qp based on quad in puda buffer |
1406 | * @dev: hardware control device structure |
1407 | * @buf: receive puda buffer on exception q |
1408 | */ |
1409 | struct irdma_sc_qp *irdma_ieq_get_qp(struct irdma_sc_dev *dev, |
1410 | struct irdma_puda_buf *buf) |
1411 | { |
1412 | struct irdma_qp *iwqp; |
1413 | struct irdma_cm_node *cm_node; |
1414 | struct irdma_device *iwdev = buf->vsi->back_vsi; |
1415 | u32 loc_addr[4] = {}; |
1416 | u32 rem_addr[4] = {}; |
1417 | u16 loc_port, rem_port; |
1418 | struct ipv6hdr *ip6h; |
1419 | struct iphdr *iph = (struct iphdr *)buf->iph; |
1420 | struct tcphdr *tcph = (struct tcphdr *)buf->tcph; |
1421 | |
1422 | if (iph->version == 4) { |
1423 | loc_addr[0] = ntohl(iph->daddr); |
1424 | rem_addr[0] = ntohl(iph->saddr); |
1425 | } else { |
1426 | ip6h = (struct ipv6hdr *)buf->iph; |
1427 | irdma_copy_ip_ntohl(dst: loc_addr, src: ip6h->daddr.in6_u.u6_addr32); |
1428 | irdma_copy_ip_ntohl(dst: rem_addr, src: ip6h->saddr.in6_u.u6_addr32); |
1429 | } |
1430 | loc_port = ntohs(tcph->dest); |
1431 | rem_port = ntohs(tcph->source); |
1432 | cm_node = irdma_find_node(cm_core: &iwdev->cm_core, rem_port, rem_addr, loc_port, |
1433 | loc_addr, vlan_id: buf->vlan_valid ? buf->vlan_id : 0xFFFF); |
1434 | if (!cm_node) |
1435 | return NULL; |
1436 | |
1437 | iwqp = cm_node->iwqp; |
1438 | irdma_rem_ref_cm_node(cm_node); |
1439 | |
1440 | return &iwqp->sc_qp; |
1441 | } |
1442 | |
1443 | /** |
1444 | * irdma_send_ieq_ack - ACKs for duplicate or OOO partials FPDUs |
1445 | * @qp: qp ptr |
1446 | */ |
1447 | void irdma_send_ieq_ack(struct irdma_sc_qp *qp) |
1448 | { |
1449 | struct irdma_cm_node *cm_node = ((struct irdma_qp *)qp->qp_uk.back_qp)->cm_node; |
1450 | struct irdma_puda_buf *buf = qp->pfpdu.lastrcv_buf; |
1451 | struct tcphdr *tcph = (struct tcphdr *)buf->tcph; |
1452 | |
1453 | cm_node->tcp_cntxt.rcv_nxt = qp->pfpdu.nextseqnum; |
1454 | cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq); |
1455 | |
1456 | irdma_send_ack(cm_node); |
1457 | } |
1458 | |
1459 | /** |
1460 | * irdma_puda_ieq_get_ah_info - get AH info from IEQ buffer |
1461 | * @qp: qp pointer |
1462 | * @ah_info: AH info pointer |
1463 | */ |
1464 | void irdma_puda_ieq_get_ah_info(struct irdma_sc_qp *qp, |
1465 | struct irdma_ah_info *ah_info) |
1466 | { |
1467 | struct irdma_puda_buf *buf = qp->pfpdu.ah_buf; |
1468 | struct iphdr *iph; |
1469 | struct ipv6hdr *ip6h; |
1470 | |
1471 | memset(ah_info, 0, sizeof(*ah_info)); |
1472 | ah_info->do_lpbk = true; |
1473 | ah_info->vlan_tag = buf->vlan_id; |
1474 | ah_info->insert_vlan_tag = buf->vlan_valid; |
1475 | ah_info->ipv4_valid = buf->ipv4; |
1476 | ah_info->vsi = qp->vsi; |
1477 | |
1478 | if (buf->smac_valid) |
1479 | ether_addr_copy(dst: ah_info->mac_addr, src: buf->smac); |
1480 | |
1481 | if (buf->ipv4) { |
1482 | ah_info->ipv4_valid = true; |
1483 | iph = (struct iphdr *)buf->iph; |
1484 | ah_info->hop_ttl = iph->ttl; |
1485 | ah_info->tc_tos = iph->tos; |
1486 | ah_info->dest_ip_addr[0] = ntohl(iph->daddr); |
1487 | ah_info->src_ip_addr[0] = ntohl(iph->saddr); |
1488 | } else { |
1489 | ip6h = (struct ipv6hdr *)buf->iph; |
1490 | ah_info->hop_ttl = ip6h->hop_limit; |
1491 | ah_info->tc_tos = ip6h->priority; |
1492 | irdma_copy_ip_ntohl(dst: ah_info->dest_ip_addr, |
1493 | src: ip6h->daddr.in6_u.u6_addr32); |
1494 | irdma_copy_ip_ntohl(dst: ah_info->src_ip_addr, |
1495 | src: ip6h->saddr.in6_u.u6_addr32); |
1496 | } |
1497 | |
1498 | ah_info->dst_arpindex = irdma_arp_table(rf: dev_to_rf(dev: qp->dev), |
1499 | ip_addr: ah_info->dest_ip_addr, |
1500 | ipv4: ah_info->ipv4_valid, |
1501 | NULL, IRDMA_ARP_RESOLVE); |
1502 | } |
1503 | |
1504 | /** |
1505 | * irdma_gen1_ieq_update_tcpip_info - update tcpip in the buffer |
1506 | * @buf: puda to update |
1507 | * @len: length of buffer |
1508 | * @seqnum: seq number for tcp |
1509 | */ |
1510 | static void irdma_gen1_ieq_update_tcpip_info(struct irdma_puda_buf *buf, |
1511 | u16 len, u32 seqnum) |
1512 | { |
1513 | struct tcphdr *tcph; |
1514 | struct iphdr *iph; |
1515 | u16 iphlen; |
1516 | u16 pktsize; |
1517 | u8 *addr = buf->mem.va; |
1518 | |
1519 | iphlen = (buf->ipv4) ? 20 : 40; |
1520 | iph = (struct iphdr *)(addr + buf->maclen); |
1521 | tcph = (struct tcphdr *)(addr + buf->maclen + iphlen); |
1522 | pktsize = len + buf->tcphlen + iphlen; |
1523 | iph->tot_len = htons(pktsize); |
1524 | tcph->seq = htonl(seqnum); |
1525 | } |
1526 | |
1527 | /** |
1528 | * irdma_ieq_update_tcpip_info - update tcpip in the buffer |
1529 | * @buf: puda to update |
1530 | * @len: length of buffer |
1531 | * @seqnum: seq number for tcp |
1532 | */ |
1533 | void irdma_ieq_update_tcpip_info(struct irdma_puda_buf *buf, u16 len, |
1534 | u32 seqnum) |
1535 | { |
1536 | struct tcphdr *tcph; |
1537 | u8 *addr; |
1538 | |
1539 | if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) |
1540 | return irdma_gen1_ieq_update_tcpip_info(buf, len, seqnum); |
1541 | |
1542 | addr = buf->mem.va; |
1543 | tcph = (struct tcphdr *)addr; |
1544 | tcph->seq = htonl(seqnum); |
1545 | } |
1546 | |
1547 | /** |
1548 | * irdma_gen1_puda_get_tcpip_info - get tcpip info from puda |
1549 | * buffer |
1550 | * @info: to get information |
1551 | * @buf: puda buffer |
1552 | */ |
1553 | static int irdma_gen1_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info, |
1554 | struct irdma_puda_buf *buf) |
1555 | { |
1556 | struct iphdr *iph; |
1557 | struct ipv6hdr *ip6h; |
1558 | struct tcphdr *tcph; |
1559 | u16 iphlen; |
1560 | u16 pkt_len; |
1561 | u8 *mem = buf->mem.va; |
1562 | struct ethhdr *ethh = buf->mem.va; |
1563 | |
1564 | if (ethh->h_proto == htons(0x8100)) { |
1565 | info->vlan_valid = true; |
1566 | buf->vlan_id = ntohs(((struct vlan_ethhdr *)ethh)->h_vlan_TCI) & |
1567 | VLAN_VID_MASK; |
1568 | } |
1569 | |
1570 | buf->maclen = (info->vlan_valid) ? 18 : 14; |
1571 | iphlen = (info->l3proto) ? 40 : 20; |
1572 | buf->ipv4 = (info->l3proto) ? false : true; |
1573 | buf->iph = mem + buf->maclen; |
1574 | iph = (struct iphdr *)buf->iph; |
1575 | buf->tcph = buf->iph + iphlen; |
1576 | tcph = (struct tcphdr *)buf->tcph; |
1577 | |
1578 | if (buf->ipv4) { |
1579 | pkt_len = ntohs(iph->tot_len); |
1580 | } else { |
1581 | ip6h = (struct ipv6hdr *)buf->iph; |
1582 | pkt_len = ntohs(ip6h->payload_len) + iphlen; |
1583 | } |
1584 | |
1585 | buf->totallen = pkt_len + buf->maclen; |
1586 | |
1587 | if (info->payload_len < buf->totallen) { |
1588 | ibdev_dbg(to_ibdev(buf->vsi->dev), |
1589 | "ERR: payload_len = 0x%x totallen expected0x%x\n" , |
1590 | info->payload_len, buf->totallen); |
1591 | return -EINVAL; |
1592 | } |
1593 | |
1594 | buf->tcphlen = tcph->doff << 2; |
1595 | buf->datalen = pkt_len - iphlen - buf->tcphlen; |
1596 | buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL; |
1597 | buf->hdrlen = buf->maclen + iphlen + buf->tcphlen; |
1598 | buf->seqnum = ntohl(tcph->seq); |
1599 | |
1600 | return 0; |
1601 | } |
1602 | |
1603 | /** |
1604 | * irdma_puda_get_tcpip_info - get tcpip info from puda buffer |
1605 | * @info: to get information |
1606 | * @buf: puda buffer |
1607 | */ |
1608 | int irdma_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info, |
1609 | struct irdma_puda_buf *buf) |
1610 | { |
1611 | struct tcphdr *tcph; |
1612 | u32 pkt_len; |
1613 | u8 *mem; |
1614 | |
1615 | if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) |
1616 | return irdma_gen1_puda_get_tcpip_info(info, buf); |
1617 | |
1618 | mem = buf->mem.va; |
1619 | buf->vlan_valid = info->vlan_valid; |
1620 | if (info->vlan_valid) |
1621 | buf->vlan_id = info->vlan; |
1622 | |
1623 | buf->ipv4 = info->ipv4; |
1624 | if (buf->ipv4) |
1625 | buf->iph = mem + IRDMA_IPV4_PAD; |
1626 | else |
1627 | buf->iph = mem; |
1628 | |
1629 | buf->tcph = mem + IRDMA_TCP_OFFSET; |
1630 | tcph = (struct tcphdr *)buf->tcph; |
1631 | pkt_len = info->payload_len; |
1632 | buf->totallen = pkt_len; |
1633 | buf->tcphlen = tcph->doff << 2; |
1634 | buf->datalen = pkt_len - IRDMA_TCP_OFFSET - buf->tcphlen; |
1635 | buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL; |
1636 | buf->hdrlen = IRDMA_TCP_OFFSET + buf->tcphlen; |
1637 | buf->seqnum = ntohl(tcph->seq); |
1638 | |
1639 | if (info->smac_valid) { |
1640 | ether_addr_copy(dst: buf->smac, src: info->smac); |
1641 | buf->smac_valid = true; |
1642 | } |
1643 | |
1644 | return 0; |
1645 | } |
1646 | |
1647 | /** |
1648 | * irdma_hw_stats_timeout - Stats timer-handler which updates all HW stats |
1649 | * @t: timer_list pointer |
1650 | */ |
1651 | static void irdma_hw_stats_timeout(struct timer_list *t) |
1652 | { |
1653 | struct irdma_vsi_pestat *pf_devstat = |
1654 | from_timer(pf_devstat, t, stats_timer); |
1655 | struct irdma_sc_vsi *sc_vsi = pf_devstat->vsi; |
1656 | |
1657 | if (sc_vsi->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) |
1658 | irdma_cqp_gather_stats_cmd(dev: sc_vsi->dev, pestat: sc_vsi->pestat, wait: false); |
1659 | else |
1660 | irdma_cqp_gather_stats_gen1(dev: sc_vsi->dev, pestat: sc_vsi->pestat); |
1661 | |
1662 | mod_timer(timer: &pf_devstat->stats_timer, |
1663 | expires: jiffies + msecs_to_jiffies(STATS_TIMER_DELAY)); |
1664 | } |
1665 | |
1666 | /** |
1667 | * irdma_hw_stats_start_timer - Start periodic stats timer |
1668 | * @vsi: vsi structure pointer |
1669 | */ |
1670 | void irdma_hw_stats_start_timer(struct irdma_sc_vsi *vsi) |
1671 | { |
1672 | struct irdma_vsi_pestat *devstat = vsi->pestat; |
1673 | |
1674 | timer_setup(&devstat->stats_timer, irdma_hw_stats_timeout, 0); |
1675 | mod_timer(timer: &devstat->stats_timer, |
1676 | expires: jiffies + msecs_to_jiffies(STATS_TIMER_DELAY)); |
1677 | } |
1678 | |
1679 | /** |
1680 | * irdma_hw_stats_stop_timer - Delete periodic stats timer |
1681 | * @vsi: pointer to vsi structure |
1682 | */ |
1683 | void irdma_hw_stats_stop_timer(struct irdma_sc_vsi *vsi) |
1684 | { |
1685 | struct irdma_vsi_pestat *devstat = vsi->pestat; |
1686 | |
1687 | del_timer_sync(timer: &devstat->stats_timer); |
1688 | } |
1689 | |
1690 | /** |
1691 | * irdma_process_stats - Checking for wrap and update stats |
1692 | * @pestat: stats structure pointer |
1693 | */ |
1694 | static inline void irdma_process_stats(struct irdma_vsi_pestat *pestat) |
1695 | { |
1696 | sc_vsi_update_stats(vsi: pestat->vsi); |
1697 | } |
1698 | |
1699 | /** |
1700 | * irdma_cqp_gather_stats_gen1 - Gather stats |
1701 | * @dev: pointer to device structure |
1702 | * @pestat: statistics structure |
1703 | */ |
1704 | void irdma_cqp_gather_stats_gen1(struct irdma_sc_dev *dev, |
1705 | struct irdma_vsi_pestat *pestat) |
1706 | { |
1707 | struct irdma_gather_stats *gather_stats = |
1708 | pestat->gather_info.gather_stats_va; |
1709 | const struct irdma_hw_stat_map *map = dev->hw_stats_map; |
1710 | u16 max_stats_idx = dev->hw_attrs.max_stat_idx; |
1711 | u32 stats_inst_offset_32; |
1712 | u32 stats_inst_offset_64; |
1713 | u64 new_val; |
1714 | u16 i; |
1715 | |
1716 | stats_inst_offset_32 = (pestat->gather_info.use_stats_inst) ? |
1717 | pestat->gather_info.stats_inst_index : |
1718 | pestat->hw->hmc.hmc_fn_id; |
1719 | stats_inst_offset_32 *= 4; |
1720 | stats_inst_offset_64 = stats_inst_offset_32 * 2; |
1721 | |
1722 | for (i = 0; i < max_stats_idx; i++) { |
1723 | if (map[i].bitmask <= IRDMA_MAX_STATS_32) |
1724 | new_val = rd32(hw: dev->hw, |
1725 | reg: dev->hw_stats_regs[i] + stats_inst_offset_32); |
1726 | else |
1727 | new_val = rd64(hw: dev->hw, |
1728 | reg: dev->hw_stats_regs[i] + stats_inst_offset_64); |
1729 | gather_stats->val[map[i].byteoff / sizeof(u64)] = new_val; |
1730 | } |
1731 | |
1732 | irdma_process_stats(pestat); |
1733 | } |
1734 | |
1735 | /** |
1736 | * irdma_process_cqp_stats - Checking for wrap and update stats |
1737 | * @cqp_request: cqp_request structure pointer |
1738 | */ |
1739 | static void irdma_process_cqp_stats(struct irdma_cqp_request *cqp_request) |
1740 | { |
1741 | struct irdma_vsi_pestat *pestat = cqp_request->param; |
1742 | |
1743 | irdma_process_stats(pestat); |
1744 | } |
1745 | |
1746 | /** |
1747 | * irdma_cqp_gather_stats_cmd - Gather stats |
1748 | * @dev: pointer to device structure |
1749 | * @pestat: pointer to stats info |
1750 | * @wait: flag to wait or not wait for stats |
1751 | */ |
1752 | int irdma_cqp_gather_stats_cmd(struct irdma_sc_dev *dev, |
1753 | struct irdma_vsi_pestat *pestat, bool wait) |
1754 | |
1755 | { |
1756 | struct irdma_pci_f *rf = dev_to_rf(dev); |
1757 | struct irdma_cqp *iwcqp = &rf->cqp; |
1758 | struct irdma_cqp_request *cqp_request; |
1759 | struct cqp_cmds_info *cqp_info; |
1760 | int status; |
1761 | |
1762 | cqp_request = irdma_alloc_and_get_cqp_request(cqp: iwcqp, wait); |
1763 | if (!cqp_request) |
1764 | return -ENOMEM; |
1765 | |
1766 | cqp_info = &cqp_request->info; |
1767 | memset(cqp_info, 0, sizeof(*cqp_info)); |
1768 | cqp_info->cqp_cmd = IRDMA_OP_STATS_GATHER; |
1769 | cqp_info->post_sq = 1; |
1770 | cqp_info->in.u.stats_gather.info = pestat->gather_info; |
1771 | cqp_info->in.u.stats_gather.scratch = (uintptr_t)cqp_request; |
1772 | cqp_info->in.u.stats_gather.cqp = &rf->cqp.sc_cqp; |
1773 | cqp_request->param = pestat; |
1774 | if (!wait) |
1775 | cqp_request->callback_fcn = irdma_process_cqp_stats; |
1776 | status = irdma_handle_cqp_op(rf, cqp_request); |
1777 | if (wait) |
1778 | irdma_process_stats(pestat); |
1779 | irdma_put_cqp_request(cqp: &rf->cqp, cqp_request); |
1780 | |
1781 | return status; |
1782 | } |
1783 | |
1784 | /** |
1785 | * irdma_cqp_stats_inst_cmd - Allocate/free stats instance |
1786 | * @vsi: pointer to vsi structure |
1787 | * @cmd: command to allocate or free |
1788 | * @stats_info: pointer to allocate stats info |
1789 | */ |
1790 | int irdma_cqp_stats_inst_cmd(struct irdma_sc_vsi *vsi, u8 cmd, |
1791 | struct irdma_stats_inst_info *stats_info) |
1792 | { |
1793 | struct irdma_pci_f *rf = dev_to_rf(dev: vsi->dev); |
1794 | struct irdma_cqp *iwcqp = &rf->cqp; |
1795 | struct irdma_cqp_request *cqp_request; |
1796 | struct cqp_cmds_info *cqp_info; |
1797 | int status; |
1798 | bool wait = false; |
1799 | |
1800 | if (cmd == IRDMA_OP_STATS_ALLOCATE) |
1801 | wait = true; |
1802 | cqp_request = irdma_alloc_and_get_cqp_request(cqp: iwcqp, wait); |
1803 | if (!cqp_request) |
1804 | return -ENOMEM; |
1805 | |
1806 | cqp_info = &cqp_request->info; |
1807 | memset(cqp_info, 0, sizeof(*cqp_info)); |
1808 | cqp_info->cqp_cmd = cmd; |
1809 | cqp_info->post_sq = 1; |
1810 | cqp_info->in.u.stats_manage.info = *stats_info; |
1811 | cqp_info->in.u.stats_manage.scratch = (uintptr_t)cqp_request; |
1812 | cqp_info->in.u.stats_manage.cqp = &rf->cqp.sc_cqp; |
1813 | status = irdma_handle_cqp_op(rf, cqp_request); |
1814 | if (wait) |
1815 | stats_info->stats_idx = cqp_request->compl_info.op_ret_val; |
1816 | irdma_put_cqp_request(cqp: iwcqp, cqp_request); |
1817 | |
1818 | return status; |
1819 | } |
1820 | |
1821 | /** |
1822 | * irdma_cqp_ceq_cmd - Create/Destroy CEQ's after CEQ 0 |
1823 | * @dev: pointer to device info |
1824 | * @sc_ceq: pointer to ceq structure |
1825 | * @op: Create or Destroy |
1826 | */ |
1827 | int irdma_cqp_ceq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_ceq *sc_ceq, |
1828 | u8 op) |
1829 | { |
1830 | struct irdma_cqp_request *cqp_request; |
1831 | struct cqp_cmds_info *cqp_info; |
1832 | struct irdma_pci_f *rf = dev_to_rf(dev); |
1833 | int status; |
1834 | |
1835 | cqp_request = irdma_alloc_and_get_cqp_request(cqp: &rf->cqp, wait: true); |
1836 | if (!cqp_request) |
1837 | return -ENOMEM; |
1838 | |
1839 | cqp_info = &cqp_request->info; |
1840 | cqp_info->post_sq = 1; |
1841 | cqp_info->cqp_cmd = op; |
1842 | cqp_info->in.u.ceq_create.ceq = sc_ceq; |
1843 | cqp_info->in.u.ceq_create.scratch = (uintptr_t)cqp_request; |
1844 | |
1845 | status = irdma_handle_cqp_op(rf, cqp_request); |
1846 | irdma_put_cqp_request(cqp: &rf->cqp, cqp_request); |
1847 | |
1848 | return status; |
1849 | } |
1850 | |
1851 | /** |
1852 | * irdma_cqp_aeq_cmd - Create/Destroy AEQ |
1853 | * @dev: pointer to device info |
1854 | * @sc_aeq: pointer to aeq structure |
1855 | * @op: Create or Destroy |
1856 | */ |
1857 | int irdma_cqp_aeq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_aeq *sc_aeq, |
1858 | u8 op) |
1859 | { |
1860 | struct irdma_cqp_request *cqp_request; |
1861 | struct cqp_cmds_info *cqp_info; |
1862 | struct irdma_pci_f *rf = dev_to_rf(dev); |
1863 | int status; |
1864 | |
1865 | cqp_request = irdma_alloc_and_get_cqp_request(cqp: &rf->cqp, wait: true); |
1866 | if (!cqp_request) |
1867 | return -ENOMEM; |
1868 | |
1869 | cqp_info = &cqp_request->info; |
1870 | cqp_info->post_sq = 1; |
1871 | cqp_info->cqp_cmd = op; |
1872 | cqp_info->in.u.aeq_create.aeq = sc_aeq; |
1873 | cqp_info->in.u.aeq_create.scratch = (uintptr_t)cqp_request; |
1874 | |
1875 | status = irdma_handle_cqp_op(rf, cqp_request); |
1876 | irdma_put_cqp_request(cqp: &rf->cqp, cqp_request); |
1877 | |
1878 | return status; |
1879 | } |
1880 | |
1881 | /** |
1882 | * irdma_cqp_ws_node_cmd - Add/modify/delete ws node |
1883 | * @dev: pointer to device structure |
1884 | * @cmd: Add, modify or delete |
1885 | * @node_info: pointer to ws node info |
1886 | */ |
1887 | int irdma_cqp_ws_node_cmd(struct irdma_sc_dev *dev, u8 cmd, |
1888 | struct irdma_ws_node_info *node_info) |
1889 | { |
1890 | struct irdma_pci_f *rf = dev_to_rf(dev); |
1891 | struct irdma_cqp *iwcqp = &rf->cqp; |
1892 | struct irdma_sc_cqp *cqp = &iwcqp->sc_cqp; |
1893 | struct irdma_cqp_request *cqp_request; |
1894 | struct cqp_cmds_info *cqp_info; |
1895 | int status; |
1896 | bool poll; |
1897 | |
1898 | if (!rf->sc_dev.ceq_valid) |
1899 | poll = true; |
1900 | else |
1901 | poll = false; |
1902 | |
1903 | cqp_request = irdma_alloc_and_get_cqp_request(cqp: iwcqp, wait: !poll); |
1904 | if (!cqp_request) |
1905 | return -ENOMEM; |
1906 | |
1907 | cqp_info = &cqp_request->info; |
1908 | memset(cqp_info, 0, sizeof(*cqp_info)); |
1909 | cqp_info->cqp_cmd = cmd; |
1910 | cqp_info->post_sq = 1; |
1911 | cqp_info->in.u.ws_node.info = *node_info; |
1912 | cqp_info->in.u.ws_node.cqp = cqp; |
1913 | cqp_info->in.u.ws_node.scratch = (uintptr_t)cqp_request; |
1914 | status = irdma_handle_cqp_op(rf, cqp_request); |
1915 | if (status) |
1916 | goto exit; |
1917 | |
1918 | if (poll) { |
1919 | struct irdma_ccq_cqe_info compl_info; |
1920 | |
1921 | status = irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_WORK_SCHED_NODE, |
1922 | cmpl_info: &compl_info); |
1923 | node_info->qs_handle = compl_info.op_ret_val; |
1924 | ibdev_dbg(&rf->iwdev->ibdev, "DCB: opcode=%d, compl_info.retval=%d\n" , |
1925 | compl_info.op_code, compl_info.op_ret_val); |
1926 | } else { |
1927 | node_info->qs_handle = cqp_request->compl_info.op_ret_val; |
1928 | } |
1929 | |
1930 | exit: |
1931 | irdma_put_cqp_request(cqp: &rf->cqp, cqp_request); |
1932 | |
1933 | return status; |
1934 | } |
1935 | |
1936 | /** |
1937 | * irdma_ah_cqp_op - perform an AH cqp operation |
1938 | * @rf: RDMA PCI function |
1939 | * @sc_ah: address handle |
1940 | * @cmd: AH operation |
1941 | * @wait: wait if true |
1942 | * @callback_fcn: Callback function on CQP op completion |
1943 | * @cb_param: parameter for callback function |
1944 | * |
1945 | * returns errno |
1946 | */ |
1947 | int irdma_ah_cqp_op(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, u8 cmd, |
1948 | bool wait, |
1949 | void (*callback_fcn)(struct irdma_cqp_request *), |
1950 | void *cb_param) |
1951 | { |
1952 | struct irdma_cqp_request *cqp_request; |
1953 | struct cqp_cmds_info *cqp_info; |
1954 | int status; |
1955 | |
1956 | if (cmd != IRDMA_OP_AH_CREATE && cmd != IRDMA_OP_AH_DESTROY) |
1957 | return -EINVAL; |
1958 | |
1959 | cqp_request = irdma_alloc_and_get_cqp_request(cqp: &rf->cqp, wait); |
1960 | if (!cqp_request) |
1961 | return -ENOMEM; |
1962 | |
1963 | cqp_info = &cqp_request->info; |
1964 | cqp_info->cqp_cmd = cmd; |
1965 | cqp_info->post_sq = 1; |
1966 | if (cmd == IRDMA_OP_AH_CREATE) { |
1967 | cqp_info->in.u.ah_create.info = sc_ah->ah_info; |
1968 | cqp_info->in.u.ah_create.scratch = (uintptr_t)cqp_request; |
1969 | cqp_info->in.u.ah_create.cqp = &rf->cqp.sc_cqp; |
1970 | } else if (cmd == IRDMA_OP_AH_DESTROY) { |
1971 | cqp_info->in.u.ah_destroy.info = sc_ah->ah_info; |
1972 | cqp_info->in.u.ah_destroy.scratch = (uintptr_t)cqp_request; |
1973 | cqp_info->in.u.ah_destroy.cqp = &rf->cqp.sc_cqp; |
1974 | } |
1975 | |
1976 | if (!wait) { |
1977 | cqp_request->callback_fcn = callback_fcn; |
1978 | cqp_request->param = cb_param; |
1979 | } |
1980 | status = irdma_handle_cqp_op(rf, cqp_request); |
1981 | irdma_put_cqp_request(cqp: &rf->cqp, cqp_request); |
1982 | |
1983 | if (status) |
1984 | return -ENOMEM; |
1985 | |
1986 | if (wait) |
1987 | sc_ah->ah_info.ah_valid = (cmd == IRDMA_OP_AH_CREATE); |
1988 | |
1989 | return 0; |
1990 | } |
1991 | |
1992 | /** |
1993 | * irdma_ieq_ah_cb - callback after creation of AH for IEQ |
1994 | * @cqp_request: pointer to cqp_request of create AH |
1995 | */ |
1996 | static void irdma_ieq_ah_cb(struct irdma_cqp_request *cqp_request) |
1997 | { |
1998 | struct irdma_sc_qp *qp = cqp_request->param; |
1999 | struct irdma_sc_ah *sc_ah = qp->pfpdu.ah; |
2000 | unsigned long flags; |
2001 | |
2002 | spin_lock_irqsave(&qp->pfpdu.lock, flags); |
2003 | if (!cqp_request->compl_info.op_ret_val) { |
2004 | sc_ah->ah_info.ah_valid = true; |
2005 | irdma_ieq_process_fpdus(qp, ieq: qp->vsi->ieq); |
2006 | } else { |
2007 | sc_ah->ah_info.ah_valid = false; |
2008 | irdma_ieq_cleanup_qp(ieq: qp->vsi->ieq, qp); |
2009 | } |
2010 | spin_unlock_irqrestore(lock: &qp->pfpdu.lock, flags); |
2011 | } |
2012 | |
2013 | /** |
2014 | * irdma_ilq_ah_cb - callback after creation of AH for ILQ |
2015 | * @cqp_request: pointer to cqp_request of create AH |
2016 | */ |
2017 | static void irdma_ilq_ah_cb(struct irdma_cqp_request *cqp_request) |
2018 | { |
2019 | struct irdma_cm_node *cm_node = cqp_request->param; |
2020 | struct irdma_sc_ah *sc_ah = cm_node->ah; |
2021 | |
2022 | sc_ah->ah_info.ah_valid = !cqp_request->compl_info.op_ret_val; |
2023 | irdma_add_conn_est_qh(cm_node); |
2024 | } |
2025 | |
2026 | /** |
2027 | * irdma_puda_create_ah - create AH for ILQ/IEQ qp's |
2028 | * @dev: device pointer |
2029 | * @ah_info: Address handle info |
2030 | * @wait: When true will wait for operation to complete |
2031 | * @type: ILQ/IEQ |
2032 | * @cb_param: Callback param when not waiting |
2033 | * @ah_ret: Returned pointer to address handle if created |
2034 | * |
2035 | */ |
2036 | int irdma_puda_create_ah(struct irdma_sc_dev *dev, |
2037 | struct irdma_ah_info *ah_info, bool wait, |
2038 | enum puda_rsrc_type type, void *cb_param, |
2039 | struct irdma_sc_ah **ah_ret) |
2040 | { |
2041 | struct irdma_sc_ah *ah; |
2042 | struct irdma_pci_f *rf = dev_to_rf(dev); |
2043 | int err; |
2044 | |
2045 | ah = kzalloc(size: sizeof(*ah), GFP_ATOMIC); |
2046 | *ah_ret = ah; |
2047 | if (!ah) |
2048 | return -ENOMEM; |
2049 | |
2050 | err = irdma_alloc_rsrc(rf, rsrc_array: rf->allocated_ahs, max_rsrc: rf->max_ah, |
2051 | req_rsrc_num: &ah_info->ah_idx, next: &rf->next_ah); |
2052 | if (err) |
2053 | goto err_free; |
2054 | |
2055 | ah->dev = dev; |
2056 | ah->ah_info = *ah_info; |
2057 | |
2058 | if (type == IRDMA_PUDA_RSRC_TYPE_ILQ) |
2059 | err = irdma_ah_cqp_op(rf, sc_ah: ah, cmd: IRDMA_OP_AH_CREATE, wait, |
2060 | callback_fcn: irdma_ilq_ah_cb, cb_param); |
2061 | else |
2062 | err = irdma_ah_cqp_op(rf, sc_ah: ah, cmd: IRDMA_OP_AH_CREATE, wait, |
2063 | callback_fcn: irdma_ieq_ah_cb, cb_param); |
2064 | |
2065 | if (err) |
2066 | goto error; |
2067 | return 0; |
2068 | |
2069 | error: |
2070 | irdma_free_rsrc(rf, rsrc_array: rf->allocated_ahs, rsrc_num: ah->ah_info.ah_idx); |
2071 | err_free: |
2072 | kfree(objp: ah); |
2073 | *ah_ret = NULL; |
2074 | return -ENOMEM; |
2075 | } |
2076 | |
2077 | /** |
2078 | * irdma_puda_free_ah - free a puda address handle |
2079 | * @dev: device pointer |
2080 | * @ah: The address handle to free |
2081 | */ |
2082 | void irdma_puda_free_ah(struct irdma_sc_dev *dev, struct irdma_sc_ah *ah) |
2083 | { |
2084 | struct irdma_pci_f *rf = dev_to_rf(dev); |
2085 | |
2086 | if (!ah) |
2087 | return; |
2088 | |
2089 | if (ah->ah_info.ah_valid) { |
2090 | irdma_ah_cqp_op(rf, sc_ah: ah, cmd: IRDMA_OP_AH_DESTROY, wait: false, NULL, NULL); |
2091 | irdma_free_rsrc(rf, rsrc_array: rf->allocated_ahs, rsrc_num: ah->ah_info.ah_idx); |
2092 | } |
2093 | |
2094 | kfree(objp: ah); |
2095 | } |
2096 | |
2097 | /** |
2098 | * irdma_gsi_ud_qp_ah_cb - callback after creation of AH for GSI/ID QP |
2099 | * @cqp_request: pointer to cqp_request of create AH |
2100 | */ |
2101 | void irdma_gsi_ud_qp_ah_cb(struct irdma_cqp_request *cqp_request) |
2102 | { |
2103 | struct irdma_sc_ah *sc_ah = cqp_request->param; |
2104 | |
2105 | if (!cqp_request->compl_info.op_ret_val) |
2106 | sc_ah->ah_info.ah_valid = true; |
2107 | else |
2108 | sc_ah->ah_info.ah_valid = false; |
2109 | } |
2110 | |
2111 | /** |
2112 | * irdma_prm_add_pble_mem - add moemory to pble resources |
2113 | * @pprm: pble resource manager |
2114 | * @pchunk: chunk of memory to add |
2115 | */ |
2116 | int irdma_prm_add_pble_mem(struct irdma_pble_prm *pprm, |
2117 | struct irdma_chunk *pchunk) |
2118 | { |
2119 | u64 sizeofbitmap; |
2120 | |
2121 | if (pchunk->size & 0xfff) |
2122 | return -EINVAL; |
2123 | |
2124 | sizeofbitmap = (u64)pchunk->size >> pprm->pble_shift; |
2125 | |
2126 | pchunk->bitmapbuf = bitmap_zalloc(nbits: sizeofbitmap, GFP_KERNEL); |
2127 | if (!pchunk->bitmapbuf) |
2128 | return -ENOMEM; |
2129 | |
2130 | pchunk->sizeofbitmap = sizeofbitmap; |
2131 | /* each pble is 8 bytes hence shift by 3 */ |
2132 | pprm->total_pble_alloc += pchunk->size >> 3; |
2133 | pprm->free_pble_cnt += pchunk->size >> 3; |
2134 | |
2135 | return 0; |
2136 | } |
2137 | |
2138 | /** |
2139 | * irdma_prm_get_pbles - get pble's from prm |
2140 | * @pprm: pble resource manager |
2141 | * @chunkinfo: nformation about chunk where pble's were acquired |
2142 | * @mem_size: size of pble memory needed |
2143 | * @vaddr: returns virtual address of pble memory |
2144 | * @fpm_addr: returns fpm address of pble memory |
2145 | */ |
2146 | int irdma_prm_get_pbles(struct irdma_pble_prm *pprm, |
2147 | struct irdma_pble_chunkinfo *chunkinfo, u64 mem_size, |
2148 | u64 **vaddr, u64 *fpm_addr) |
2149 | { |
2150 | u64 bits_needed; |
2151 | u64 bit_idx = PBLE_INVALID_IDX; |
2152 | struct irdma_chunk *pchunk = NULL; |
2153 | struct list_head *chunk_entry = pprm->clist.next; |
2154 | u32 offset; |
2155 | unsigned long flags; |
2156 | *vaddr = NULL; |
2157 | *fpm_addr = 0; |
2158 | |
2159 | bits_needed = DIV_ROUND_UP_ULL(mem_size, BIT_ULL(pprm->pble_shift)); |
2160 | |
2161 | spin_lock_irqsave(&pprm->prm_lock, flags); |
2162 | while (chunk_entry != &pprm->clist) { |
2163 | pchunk = (struct irdma_chunk *)chunk_entry; |
2164 | bit_idx = bitmap_find_next_zero_area(map: pchunk->bitmapbuf, |
2165 | size: pchunk->sizeofbitmap, start: 0, |
2166 | nr: bits_needed, align_mask: 0); |
2167 | if (bit_idx < pchunk->sizeofbitmap) |
2168 | break; |
2169 | |
2170 | /* list.next used macro */ |
2171 | chunk_entry = pchunk->list.next; |
2172 | } |
2173 | |
2174 | if (!pchunk || bit_idx >= pchunk->sizeofbitmap) { |
2175 | spin_unlock_irqrestore(lock: &pprm->prm_lock, flags); |
2176 | return -ENOMEM; |
2177 | } |
2178 | |
2179 | bitmap_set(map: pchunk->bitmapbuf, start: bit_idx, nbits: bits_needed); |
2180 | offset = bit_idx << pprm->pble_shift; |
2181 | *vaddr = pchunk->vaddr + offset; |
2182 | *fpm_addr = pchunk->fpm_addr + offset; |
2183 | |
2184 | chunkinfo->pchunk = pchunk; |
2185 | chunkinfo->bit_idx = bit_idx; |
2186 | chunkinfo->bits_used = bits_needed; |
2187 | /* 3 is sizeof pble divide */ |
2188 | pprm->free_pble_cnt -= chunkinfo->bits_used << (pprm->pble_shift - 3); |
2189 | spin_unlock_irqrestore(lock: &pprm->prm_lock, flags); |
2190 | |
2191 | return 0; |
2192 | } |
2193 | |
2194 | /** |
2195 | * irdma_prm_return_pbles - return pbles back to prm |
2196 | * @pprm: pble resource manager |
2197 | * @chunkinfo: chunk where pble's were acquired and to be freed |
2198 | */ |
2199 | void irdma_prm_return_pbles(struct irdma_pble_prm *pprm, |
2200 | struct irdma_pble_chunkinfo *chunkinfo) |
2201 | { |
2202 | unsigned long flags; |
2203 | |
2204 | spin_lock_irqsave(&pprm->prm_lock, flags); |
2205 | pprm->free_pble_cnt += chunkinfo->bits_used << (pprm->pble_shift - 3); |
2206 | bitmap_clear(map: chunkinfo->pchunk->bitmapbuf, start: chunkinfo->bit_idx, |
2207 | nbits: chunkinfo->bits_used); |
2208 | spin_unlock_irqrestore(lock: &pprm->prm_lock, flags); |
2209 | } |
2210 | |
2211 | int irdma_map_vm_page_list(struct irdma_hw *hw, void *va, dma_addr_t *pg_dma, |
2212 | u32 pg_cnt) |
2213 | { |
2214 | struct page *vm_page; |
2215 | int i; |
2216 | u8 *addr; |
2217 | |
2218 | addr = (u8 *)(uintptr_t)va; |
2219 | for (i = 0; i < pg_cnt; i++) { |
2220 | vm_page = vmalloc_to_page(addr); |
2221 | if (!vm_page) |
2222 | goto err; |
2223 | |
2224 | pg_dma[i] = dma_map_page(hw->device, vm_page, 0, PAGE_SIZE, |
2225 | DMA_BIDIRECTIONAL); |
2226 | if (dma_mapping_error(dev: hw->device, dma_addr: pg_dma[i])) |
2227 | goto err; |
2228 | |
2229 | addr += PAGE_SIZE; |
2230 | } |
2231 | |
2232 | return 0; |
2233 | |
2234 | err: |
2235 | irdma_unmap_vm_page_list(hw, pg_dma, pg_cnt: i); |
2236 | return -ENOMEM; |
2237 | } |
2238 | |
2239 | void irdma_unmap_vm_page_list(struct irdma_hw *hw, dma_addr_t *pg_dma, u32 pg_cnt) |
2240 | { |
2241 | int i; |
2242 | |
2243 | for (i = 0; i < pg_cnt; i++) |
2244 | dma_unmap_page(hw->device, pg_dma[i], PAGE_SIZE, DMA_BIDIRECTIONAL); |
2245 | } |
2246 | |
2247 | /** |
2248 | * irdma_pble_free_paged_mem - free virtual paged memory |
2249 | * @chunk: chunk to free with paged memory |
2250 | */ |
2251 | void irdma_pble_free_paged_mem(struct irdma_chunk *chunk) |
2252 | { |
2253 | if (!chunk->pg_cnt) |
2254 | goto done; |
2255 | |
2256 | irdma_unmap_vm_page_list(hw: chunk->dev->hw, pg_dma: chunk->dmainfo.dmaaddrs, |
2257 | pg_cnt: chunk->pg_cnt); |
2258 | |
2259 | done: |
2260 | kfree(objp: chunk->dmainfo.dmaaddrs); |
2261 | chunk->dmainfo.dmaaddrs = NULL; |
2262 | vfree(addr: chunk->vaddr); |
2263 | chunk->vaddr = NULL; |
2264 | chunk->type = 0; |
2265 | } |
2266 | |
2267 | /** |
2268 | * irdma_pble_get_paged_mem -allocate paged memory for pbles |
2269 | * @chunk: chunk to add for paged memory |
2270 | * @pg_cnt: number of pages needed |
2271 | */ |
2272 | int irdma_pble_get_paged_mem(struct irdma_chunk *chunk, u32 pg_cnt) |
2273 | { |
2274 | u32 size; |
2275 | void *va; |
2276 | |
2277 | chunk->dmainfo.dmaaddrs = kzalloc(size: pg_cnt << 3, GFP_KERNEL); |
2278 | if (!chunk->dmainfo.dmaaddrs) |
2279 | return -ENOMEM; |
2280 | |
2281 | size = PAGE_SIZE * pg_cnt; |
2282 | va = vmalloc(size); |
2283 | if (!va) |
2284 | goto err; |
2285 | |
2286 | if (irdma_map_vm_page_list(hw: chunk->dev->hw, va, pg_dma: chunk->dmainfo.dmaaddrs, |
2287 | pg_cnt)) { |
2288 | vfree(addr: va); |
2289 | goto err; |
2290 | } |
2291 | chunk->vaddr = va; |
2292 | chunk->size = size; |
2293 | chunk->pg_cnt = pg_cnt; |
2294 | chunk->type = PBLE_SD_PAGED; |
2295 | |
2296 | return 0; |
2297 | err: |
2298 | kfree(objp: chunk->dmainfo.dmaaddrs); |
2299 | chunk->dmainfo.dmaaddrs = NULL; |
2300 | |
2301 | return -ENOMEM; |
2302 | } |
2303 | |
2304 | /** |
2305 | * irdma_alloc_ws_node_id - Allocate a tx scheduler node ID |
2306 | * @dev: device pointer |
2307 | */ |
2308 | u16 irdma_alloc_ws_node_id(struct irdma_sc_dev *dev) |
2309 | { |
2310 | struct irdma_pci_f *rf = dev_to_rf(dev); |
2311 | u32 next = 1; |
2312 | u32 node_id; |
2313 | |
2314 | if (irdma_alloc_rsrc(rf, rsrc_array: rf->allocated_ws_nodes, max_rsrc: rf->max_ws_node_id, |
2315 | req_rsrc_num: &node_id, next: &next)) |
2316 | return IRDMA_WS_NODE_INVALID; |
2317 | |
2318 | return (u16)node_id; |
2319 | } |
2320 | |
2321 | /** |
2322 | * irdma_free_ws_node_id - Free a tx scheduler node ID |
2323 | * @dev: device pointer |
2324 | * @node_id: Work scheduler node ID |
2325 | */ |
2326 | void irdma_free_ws_node_id(struct irdma_sc_dev *dev, u16 node_id) |
2327 | { |
2328 | struct irdma_pci_f *rf = dev_to_rf(dev); |
2329 | |
2330 | irdma_free_rsrc(rf, rsrc_array: rf->allocated_ws_nodes, rsrc_num: (u32)node_id); |
2331 | } |
2332 | |
2333 | /** |
2334 | * irdma_modify_qp_to_err - Modify a QP to error |
2335 | * @sc_qp: qp structure |
2336 | */ |
2337 | void irdma_modify_qp_to_err(struct irdma_sc_qp *sc_qp) |
2338 | { |
2339 | struct irdma_qp *qp = sc_qp->qp_uk.back_qp; |
2340 | struct ib_qp_attr attr; |
2341 | |
2342 | if (qp->iwdev->rf->reset) |
2343 | return; |
2344 | attr.qp_state = IB_QPS_ERR; |
2345 | |
2346 | if (rdma_protocol_roce(device: qp->ibqp.device, port_num: 1)) |
2347 | irdma_modify_qp_roce(ibqp: &qp->ibqp, attr: &attr, attr_mask: IB_QP_STATE, NULL); |
2348 | else |
2349 | irdma_modify_qp(ibqp: &qp->ibqp, attr: &attr, attr_mask: IB_QP_STATE, NULL); |
2350 | } |
2351 | |
2352 | void irdma_ib_qp_event(struct irdma_qp *iwqp, enum irdma_qp_event_type event) |
2353 | { |
2354 | struct ib_event ibevent; |
2355 | |
2356 | if (!iwqp->ibqp.event_handler) |
2357 | return; |
2358 | |
2359 | switch (event) { |
2360 | case IRDMA_QP_EVENT_CATASTROPHIC: |
2361 | ibevent.event = IB_EVENT_QP_FATAL; |
2362 | break; |
2363 | case IRDMA_QP_EVENT_ACCESS_ERR: |
2364 | ibevent.event = IB_EVENT_QP_ACCESS_ERR; |
2365 | break; |
2366 | case IRDMA_QP_EVENT_REQ_ERR: |
2367 | ibevent.event = IB_EVENT_QP_REQ_ERR; |
2368 | break; |
2369 | } |
2370 | ibevent.device = iwqp->ibqp.device; |
2371 | ibevent.element.qp = &iwqp->ibqp; |
2372 | iwqp->ibqp.event_handler(&ibevent, iwqp->ibqp.qp_context); |
2373 | } |
2374 | |
2375 | bool irdma_cq_empty(struct irdma_cq *iwcq) |
2376 | { |
2377 | struct irdma_cq_uk *ukcq; |
2378 | u64 qword3; |
2379 | __le64 *cqe; |
2380 | u8 polarity; |
2381 | |
2382 | ukcq = &iwcq->sc_cq.cq_uk; |
2383 | cqe = IRDMA_GET_CURRENT_CQ_ELEM(ukcq); |
2384 | get_64bit_val(wqe_words: cqe, byte_index: 24, val: &qword3); |
2385 | polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3); |
2386 | |
2387 | return polarity != ukcq->polarity; |
2388 | } |
2389 | |
2390 | void irdma_remove_cmpls_list(struct irdma_cq *iwcq) |
2391 | { |
2392 | struct irdma_cmpl_gen *cmpl_node; |
2393 | struct list_head *tmp_node, *list_node; |
2394 | |
2395 | list_for_each_safe (list_node, tmp_node, &iwcq->cmpl_generated) { |
2396 | cmpl_node = list_entry(list_node, struct irdma_cmpl_gen, list); |
2397 | list_del(entry: &cmpl_node->list); |
2398 | kfree(objp: cmpl_node); |
2399 | } |
2400 | } |
2401 | |
2402 | int irdma_generated_cmpls(struct irdma_cq *iwcq, struct irdma_cq_poll_info *cq_poll_info) |
2403 | { |
2404 | struct irdma_cmpl_gen *cmpl; |
2405 | |
2406 | if (list_empty(head: &iwcq->cmpl_generated)) |
2407 | return -ENOENT; |
2408 | cmpl = list_first_entry_or_null(&iwcq->cmpl_generated, struct irdma_cmpl_gen, list); |
2409 | list_del(entry: &cmpl->list); |
2410 | memcpy(cq_poll_info, &cmpl->cpi, sizeof(*cq_poll_info)); |
2411 | kfree(objp: cmpl); |
2412 | |
2413 | ibdev_dbg(iwcq->ibcq.device, |
2414 | "VERBS: %s: Poll artificially generated completion for QP 0x%X, op %u, wr_id=0x%llx\n" , |
2415 | __func__, cq_poll_info->qp_id, cq_poll_info->op_type, |
2416 | cq_poll_info->wr_id); |
2417 | |
2418 | return 0; |
2419 | } |
2420 | |
2421 | /** |
2422 | * irdma_set_cpi_common_values - fill in values for polling info struct |
2423 | * @cpi: resulting structure of cq_poll_info type |
2424 | * @qp: QPair |
2425 | * @qp_num: id of the QP |
2426 | */ |
2427 | static void irdma_set_cpi_common_values(struct irdma_cq_poll_info *cpi, |
2428 | struct irdma_qp_uk *qp, u32 qp_num) |
2429 | { |
2430 | cpi->comp_status = IRDMA_COMPL_STATUS_FLUSHED; |
2431 | cpi->error = true; |
2432 | cpi->major_err = IRDMA_FLUSH_MAJOR_ERR; |
2433 | cpi->minor_err = FLUSH_GENERAL_ERR; |
2434 | cpi->qp_handle = (irdma_qp_handle)(uintptr_t)qp; |
2435 | cpi->qp_id = qp_num; |
2436 | } |
2437 | |
2438 | static inline void irdma_comp_handler(struct irdma_cq *cq) |
2439 | { |
2440 | if (!cq->ibcq.comp_handler) |
2441 | return; |
2442 | if (atomic_cmpxchg(v: &cq->armed, old: 1, new: 0)) |
2443 | cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); |
2444 | } |
2445 | |
2446 | void irdma_generate_flush_completions(struct irdma_qp *iwqp) |
2447 | { |
2448 | struct irdma_qp_uk *qp = &iwqp->sc_qp.qp_uk; |
2449 | struct irdma_ring *sq_ring = &qp->sq_ring; |
2450 | struct irdma_ring *rq_ring = &qp->rq_ring; |
2451 | struct irdma_cmpl_gen *cmpl; |
2452 | __le64 *sw_wqe; |
2453 | u64 wqe_qword; |
2454 | u32 wqe_idx; |
2455 | bool compl_generated = false; |
2456 | unsigned long flags1; |
2457 | |
2458 | spin_lock_irqsave(&iwqp->iwscq->lock, flags1); |
2459 | if (irdma_cq_empty(iwcq: iwqp->iwscq)) { |
2460 | unsigned long flags2; |
2461 | |
2462 | spin_lock_irqsave(&iwqp->lock, flags2); |
2463 | while (IRDMA_RING_MORE_WORK(*sq_ring)) { |
2464 | cmpl = kzalloc(size: sizeof(*cmpl), GFP_ATOMIC); |
2465 | if (!cmpl) { |
2466 | spin_unlock_irqrestore(lock: &iwqp->lock, flags: flags2); |
2467 | spin_unlock_irqrestore(lock: &iwqp->iwscq->lock, flags: flags1); |
2468 | return; |
2469 | } |
2470 | |
2471 | wqe_idx = sq_ring->tail; |
2472 | irdma_set_cpi_common_values(cpi: &cmpl->cpi, qp, qp_num: qp->qp_id); |
2473 | |
2474 | cmpl->cpi.wr_id = qp->sq_wrtrk_array[wqe_idx].wrid; |
2475 | sw_wqe = qp->sq_base[wqe_idx].elem; |
2476 | get_64bit_val(wqe_words: sw_wqe, byte_index: 24, val: &wqe_qword); |
2477 | cmpl->cpi.op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, IRDMAQPSQ_OPCODE); |
2478 | cmpl->cpi.q_type = IRDMA_CQE_QTYPE_SQ; |
2479 | /* remove the SQ WR by moving SQ tail*/ |
2480 | IRDMA_RING_SET_TAIL(*sq_ring, |
2481 | sq_ring->tail + qp->sq_wrtrk_array[sq_ring->tail].quanta); |
2482 | if (cmpl->cpi.op_type == IRDMAQP_OP_NOP) { |
2483 | kfree(objp: cmpl); |
2484 | continue; |
2485 | } |
2486 | ibdev_dbg(iwqp->iwscq->ibcq.device, |
2487 | "DEV: %s: adding wr_id = 0x%llx SQ Completion to list qp_id=%d\n" , |
2488 | __func__, cmpl->cpi.wr_id, qp->qp_id); |
2489 | list_add_tail(new: &cmpl->list, head: &iwqp->iwscq->cmpl_generated); |
2490 | compl_generated = true; |
2491 | } |
2492 | spin_unlock_irqrestore(lock: &iwqp->lock, flags: flags2); |
2493 | spin_unlock_irqrestore(lock: &iwqp->iwscq->lock, flags: flags1); |
2494 | if (compl_generated) |
2495 | irdma_comp_handler(cq: iwqp->iwscq); |
2496 | } else { |
2497 | spin_unlock_irqrestore(lock: &iwqp->iwscq->lock, flags: flags1); |
2498 | mod_delayed_work(wq: iwqp->iwdev->cleanup_wq, dwork: &iwqp->dwork_flush, |
2499 | delay: msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS)); |
2500 | } |
2501 | |
2502 | spin_lock_irqsave(&iwqp->iwrcq->lock, flags1); |
2503 | if (irdma_cq_empty(iwcq: iwqp->iwrcq)) { |
2504 | unsigned long flags2; |
2505 | |
2506 | spin_lock_irqsave(&iwqp->lock, flags2); |
2507 | while (IRDMA_RING_MORE_WORK(*rq_ring)) { |
2508 | cmpl = kzalloc(size: sizeof(*cmpl), GFP_ATOMIC); |
2509 | if (!cmpl) { |
2510 | spin_unlock_irqrestore(lock: &iwqp->lock, flags: flags2); |
2511 | spin_unlock_irqrestore(lock: &iwqp->iwrcq->lock, flags: flags1); |
2512 | return; |
2513 | } |
2514 | |
2515 | wqe_idx = rq_ring->tail; |
2516 | irdma_set_cpi_common_values(cpi: &cmpl->cpi, qp, qp_num: qp->qp_id); |
2517 | |
2518 | cmpl->cpi.wr_id = qp->rq_wrid_array[wqe_idx]; |
2519 | cmpl->cpi.op_type = IRDMA_OP_TYPE_REC; |
2520 | cmpl->cpi.q_type = IRDMA_CQE_QTYPE_RQ; |
2521 | /* remove the RQ WR by moving RQ tail */ |
2522 | IRDMA_RING_SET_TAIL(*rq_ring, rq_ring->tail + 1); |
2523 | ibdev_dbg(iwqp->iwrcq->ibcq.device, |
2524 | "DEV: %s: adding wr_id = 0x%llx RQ Completion to list qp_id=%d, wqe_idx=%d\n" , |
2525 | __func__, cmpl->cpi.wr_id, qp->qp_id, |
2526 | wqe_idx); |
2527 | list_add_tail(new: &cmpl->list, head: &iwqp->iwrcq->cmpl_generated); |
2528 | |
2529 | compl_generated = true; |
2530 | } |
2531 | spin_unlock_irqrestore(lock: &iwqp->lock, flags: flags2); |
2532 | spin_unlock_irqrestore(lock: &iwqp->iwrcq->lock, flags: flags1); |
2533 | if (compl_generated) |
2534 | irdma_comp_handler(cq: iwqp->iwrcq); |
2535 | } else { |
2536 | spin_unlock_irqrestore(lock: &iwqp->iwrcq->lock, flags: flags1); |
2537 | mod_delayed_work(wq: iwqp->iwdev->cleanup_wq, dwork: &iwqp->dwork_flush, |
2538 | delay: msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS)); |
2539 | } |
2540 | } |
2541 | |