1	// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2	/* Copyright (c) 2015 - 2021 Intel Corporation */
3	#include "main.h"
4
5	/**
6	* irdma_arp_table -manage arp table
7	* @rf: RDMA PCI function
8	* @ip_addr: ip address for device
9	* @ipv4: IPv4 flag
10	* @mac_addr: mac address ptr
11	* @action: modify, delete or add
12	*/
13	int irdma_arp_table(struct irdma_pci_f *rf, u32 *ip_addr, bool ipv4,
14	const u8 *mac_addr, u32 action)
15	{
16	unsigned long flags;
17	int arp_index;
18	u32 ip[4] = {};
19
20	if (ipv4)
21	ip[0] = *ip_addr;
22	else
23	memcpy(ip, ip_addr, sizeof(ip));
24
25	spin_lock_irqsave(&rf->arp_lock, flags);
26	for (arp_index = 0; (u32)arp_index < rf->arp_table_size; arp_index++) {
27	if (!memcmp(p: rf->arp_table[arp_index].ip_addr, q: ip, size: sizeof(ip)))
28	break;
29	}
30
31	switch (action) {
32	case IRDMA_ARP_ADD:
33	if (arp_index != rf->arp_table_size) {
34	arp_index = -1;
35	break;
36	}
37
38	arp_index = 0;
39	if (irdma_alloc_rsrc(rf, rsrc_array: rf->allocated_arps, max_rsrc: rf->arp_table_size,
40	req_rsrc_num: (u32 *)&arp_index, next: &rf->next_arp_index)) {
41	arp_index = -1;
42	break;
43	}
44
45	memcpy(rf->arp_table[arp_index].ip_addr, ip,
46	sizeof(rf->arp_table[arp_index].ip_addr));
47	ether_addr_copy(dst: rf->arp_table[arp_index].mac_addr, src: mac_addr);
48	break;
49	case IRDMA_ARP_RESOLVE:
50	if (arp_index == rf->arp_table_size)
51	arp_index = -1;
52	break;
53	case IRDMA_ARP_DELETE:
54	if (arp_index == rf->arp_table_size) {
55	arp_index = -1;
56	break;
57	}
58
59	memset(rf->arp_table[arp_index].ip_addr, 0,
60	sizeof(rf->arp_table[arp_index].ip_addr));
61	eth_zero_addr(addr: rf->arp_table[arp_index].mac_addr);
62	irdma_free_rsrc(rf, rsrc_array: rf->allocated_arps, rsrc_num: arp_index);
63	break;
64	default:
65	arp_index = -1;
66	break;
67	}
68
69	spin_unlock_irqrestore(lock: &rf->arp_lock, flags);
70	return arp_index;
71	}
72
73	/**
74	* irdma_add_arp - add a new arp entry if needed
75	* @rf: RDMA function
76	* @ip: IP address
77	* @ipv4: IPv4 flag
78	* @mac: MAC address
79	*/
80	int irdma_add_arp(struct irdma_pci_f *rf, u32 *ip, bool ipv4, const u8 *mac)
81	{
82	int arpidx;
83
84	arpidx = irdma_arp_table(rf, ip_addr: &ip[0], ipv4, NULL, IRDMA_ARP_RESOLVE);
85	if (arpidx >= 0) {
86	if (ether_addr_equal(addr1: rf->arp_table[arpidx].mac_addr, addr2: mac))
87	return arpidx;
88
89	irdma_manage_arp_cache(rf, mac_addr: rf->arp_table[arpidx].mac_addr, ip_addr: ip,
90	ipv4, IRDMA_ARP_DELETE);
91	}
92
93	irdma_manage_arp_cache(rf, mac_addr: mac, ip_addr: ip, ipv4, IRDMA_ARP_ADD);
94
95	return irdma_arp_table(rf, ip_addr: ip, ipv4, NULL, IRDMA_ARP_RESOLVE);
96	}
97
98	/**
99	* wr32 - write 32 bits to hw register
100	* @hw: hardware information including registers
101	* @reg: register offset
102	* @val: value to write to register
103	*/
104	inline void wr32(struct irdma_hw *hw, u32 reg, u32 val)
105	{
106	writel(val, addr: hw->hw_addr + reg);
107	}
108
109	/**
110	* rd32 - read a 32 bit hw register
111	* @hw: hardware information including registers
112	* @reg: register offset
113	*
114	* Return value of register content
115	*/
116	inline u32 rd32(struct irdma_hw *hw, u32 reg)
117	{
118	return readl(addr: hw->hw_addr + reg);
119	}
120
121	/**
122	* rd64 - read a 64 bit hw register
123	* @hw: hardware information including registers
124	* @reg: register offset
125	*
126	* Return value of register content
127	*/
128	inline u64 rd64(struct irdma_hw *hw, u32 reg)
129	{
130	return readq(addr: hw->hw_addr + reg);
131	}
132
133	static void irdma_gid_change_event(struct ib_device *ibdev)
134	{
135	struct ib_event ib_event;
136
137	ib_event.event = IB_EVENT_GID_CHANGE;
138	ib_event.device = ibdev;
139	ib_event.element.port_num = 1;
140	ib_dispatch_event(event: &ib_event);
141	}
142
143	/**
144	* irdma_inetaddr_event - system notifier for ipv4 addr events
145	* @notifier: not used
146	* @event: event for notifier
147	* @ptr: if address
148	*/
149	int irdma_inetaddr_event(struct notifier_block *notifier, unsigned long event,
150	void *ptr)
151	{
152	struct in_ifaddr *ifa = ptr;
153	struct net_device *real_dev, *netdev = ifa->ifa_dev->dev;
154	struct irdma_device *iwdev;
155	struct ib_device *ibdev;
156	u32 local_ipaddr;
157
158	real_dev = rdma_vlan_dev_real_dev(dev: netdev);
159	if (!real_dev)
160	real_dev = netdev;
161
162	ibdev = ib_device_get_by_netdev(ndev: real_dev, driver_id: RDMA_DRIVER_IRDMA);
163	if (!ibdev)
164	return NOTIFY_DONE;
165
166	iwdev = to_iwdev(ibdev);
167	local_ipaddr = ntohl(ifa->ifa_address);
168	ibdev_dbg(&iwdev->ibdev,
169	"DEV: netdev %p event %lu local_ip=%pI4 MAC=%pM\n", real_dev,
170	event, &local_ipaddr, real_dev->dev_addr);
171	switch (event) {
172	case NETDEV_DOWN:
173	irdma_manage_arp_cache(rf: iwdev->rf, mac_addr: real_dev->dev_addr,
174	ip_addr: &local_ipaddr, ipv4: true, IRDMA_ARP_DELETE);
175	irdma_if_notify(iwdev, netdev: real_dev, ipaddr: &local_ipaddr, ipv4: true, ifup: false);
176	irdma_gid_change_event(ibdev: &iwdev->ibdev);
177	break;
178	case NETDEV_UP:
179	case NETDEV_CHANGEADDR:
180	irdma_add_arp(rf: iwdev->rf, ip: &local_ipaddr, ipv4: true, mac: real_dev->dev_addr);
181	irdma_if_notify(iwdev, netdev: real_dev, ipaddr: &local_ipaddr, ipv4: true, ifup: true);
182	irdma_gid_change_event(ibdev: &iwdev->ibdev);
183	break;
184	default:
185	break;
186	}
187
188	ib_device_put(device: ibdev);
189
190	return NOTIFY_DONE;
191	}
192
193	/**
194	* irdma_inet6addr_event - system notifier for ipv6 addr events
195	* @notifier: not used
196	* @event: event for notifier
197	* @ptr: if address
198	*/
199	int irdma_inet6addr_event(struct notifier_block *notifier, unsigned long event,
200	void *ptr)
201	{
202	struct inet6_ifaddr *ifa = ptr;
203	struct net_device *real_dev, *netdev = ifa->idev->dev;
204	struct irdma_device *iwdev;
205	struct ib_device *ibdev;
206	u32 local_ipaddr6[4];
207
208	real_dev = rdma_vlan_dev_real_dev(dev: netdev);
209	if (!real_dev)
210	real_dev = netdev;
211
212	ibdev = ib_device_get_by_netdev(ndev: real_dev, driver_id: RDMA_DRIVER_IRDMA);
213	if (!ibdev)
214	return NOTIFY_DONE;
215
216	iwdev = to_iwdev(ibdev);
217	irdma_copy_ip_ntohl(dst: local_ipaddr6, src: ifa->addr.in6_u.u6_addr32);
218	ibdev_dbg(&iwdev->ibdev,
219	"DEV: netdev %p event %lu local_ip=%pI6 MAC=%pM\n", real_dev,
220	event, local_ipaddr6, real_dev->dev_addr);
221	switch (event) {
222	case NETDEV_DOWN:
223	irdma_manage_arp_cache(rf: iwdev->rf, mac_addr: real_dev->dev_addr,
224	ip_addr: local_ipaddr6, ipv4: false, IRDMA_ARP_DELETE);
225	irdma_if_notify(iwdev, netdev: real_dev, ipaddr: local_ipaddr6, ipv4: false, ifup: false);
226	irdma_gid_change_event(ibdev: &iwdev->ibdev);
227	break;
228	case NETDEV_UP:
229	case NETDEV_CHANGEADDR:
230	irdma_add_arp(rf: iwdev->rf, ip: local_ipaddr6, ipv4: false,
231	mac: real_dev->dev_addr);
232	irdma_if_notify(iwdev, netdev: real_dev, ipaddr: local_ipaddr6, ipv4: false, ifup: true);
233	irdma_gid_change_event(ibdev: &iwdev->ibdev);
234	break;
235	default:
236	break;
237	}
238
239	ib_device_put(device: ibdev);
240
241	return NOTIFY_DONE;
242	}
243
244	/**
245	* irdma_net_event - system notifier for net events
246	* @notifier: not used
247	* @event: event for notifier
248	* @ptr: neighbor
249	*/
250	int irdma_net_event(struct notifier_block *notifier, unsigned long event,
251	void *ptr)
252	{
253	struct neighbour *neigh = ptr;
254	struct net_device *real_dev, *netdev;
255	struct irdma_device *iwdev;
256	struct ib_device *ibdev;
257	__be32 *p;
258	u32 local_ipaddr[4] = {};
259	bool ipv4 = true;
260
261	switch (event) {
262	case NETEVENT_NEIGH_UPDATE:
263	netdev = neigh->dev;
264	real_dev = rdma_vlan_dev_real_dev(dev: netdev);
265	if (!real_dev)
266	real_dev = netdev;
267	ibdev = ib_device_get_by_netdev(ndev: real_dev, driver_id: RDMA_DRIVER_IRDMA);
268	if (!ibdev)
269	return NOTIFY_DONE;
270
271	iwdev = to_iwdev(ibdev);
272	p = (__be32 *)neigh->primary_key;
273	if (neigh->tbl->family == AF_INET6) {
274	ipv4 = false;
275	irdma_copy_ip_ntohl(dst: local_ipaddr, src: p);
276	} else {
277	local_ipaddr[0] = ntohl(*p);
278	}
279
280	ibdev_dbg(&iwdev->ibdev,
281	"DEV: netdev %p state %d local_ip=%pI4 MAC=%pM\n",
282	iwdev->netdev, neigh->nud_state, local_ipaddr,
283	neigh->ha);
284
285	if (neigh->nud_state & NUD_VALID)
286	irdma_add_arp(rf: iwdev->rf, ip: local_ipaddr, ipv4, mac: neigh->ha);
287
288	else
289	irdma_manage_arp_cache(rf: iwdev->rf, mac_addr: neigh->ha,
290	ip_addr: local_ipaddr, ipv4,
291	IRDMA_ARP_DELETE);
292	ib_device_put(device: ibdev);
293	break;
294	default:
295	break;
296	}
297
298	return NOTIFY_DONE;
299	}
300
301	/**
302	* irdma_netdevice_event - system notifier for netdev events
303	* @notifier: not used
304	* @event: event for notifier
305	* @ptr: netdev
306	*/
307	int irdma_netdevice_event(struct notifier_block *notifier, unsigned long event,
308	void *ptr)
309	{
310	struct irdma_device *iwdev;
311	struct ib_device *ibdev;
312	struct net_device *netdev = netdev_notifier_info_to_dev(info: ptr);
313
314	ibdev = ib_device_get_by_netdev(ndev: netdev, driver_id: RDMA_DRIVER_IRDMA);
315	if (!ibdev)
316	return NOTIFY_DONE;
317
318	iwdev = to_iwdev(ibdev);
319	iwdev->iw_status = 1;
320	switch (event) {
321	case NETDEV_DOWN:
322	iwdev->iw_status = 0;
323	fallthrough;
324	default:
325	break;
326	}
327	ib_device_put(device: ibdev);
328
329	return NOTIFY_DONE;
330	}
331
332	/**
333	* irdma_add_ipv6_addr - add ipv6 address to the hw arp table
334	* @iwdev: irdma device
335	*/
336	static void irdma_add_ipv6_addr(struct irdma_device *iwdev)
337	{
338	struct net_device *ip_dev;
339	struct inet6_dev *idev;
340	struct inet6_ifaddr *ifp, *tmp;
341	u32 local_ipaddr6[4];
342
343	rcu_read_lock();
344	for_each_netdev_rcu (&init_net, ip_dev) {
345	if (((rdma_vlan_dev_vlan_id(dev: ip_dev) < 0xFFFF &&
346	rdma_vlan_dev_real_dev(dev: ip_dev) == iwdev->netdev) ||
347	ip_dev == iwdev->netdev) &&
348	(READ_ONCE(ip_dev->flags) & IFF_UP)) {
349	idev = __in6_dev_get(dev: ip_dev);
350	if (!idev) {
351	ibdev_err(ibdev: &iwdev->ibdev, format: "ipv6 inet device not found\n");
352	break;
353	}
354	list_for_each_entry_safe (ifp, tmp, &idev->addr_list,
355	if_list) {
356	ibdev_dbg(&iwdev->ibdev,
357	"INIT: IP=%pI6, vlan_id=%d, MAC=%pM\n",
358	&ifp->addr,
359	rdma_vlan_dev_vlan_id(ip_dev),
360	ip_dev->dev_addr);
361
362	irdma_copy_ip_ntohl(dst: local_ipaddr6,
363	src: ifp->addr.in6_u.u6_addr32);
364	irdma_manage_arp_cache(rf: iwdev->rf,
365	mac_addr: ip_dev->dev_addr,
366	ip_addr: local_ipaddr6, ipv4: false,
367	IRDMA_ARP_ADD);
368	}
369	}
370	}
371	rcu_read_unlock();
372	}
373
374	/**
375	* irdma_add_ipv4_addr - add ipv4 address to the hw arp table
376	* @iwdev: irdma device
377	*/
378	static void irdma_add_ipv4_addr(struct irdma_device *iwdev)
379	{
380	struct net_device *dev;
381	struct in_device *idev;
382	u32 ip_addr;
383
384	rcu_read_lock();
385	for_each_netdev_rcu (&init_net, dev) {
386	if (((rdma_vlan_dev_vlan_id(dev) < 0xFFFF &&
387	rdma_vlan_dev_real_dev(dev) == iwdev->netdev) ||
388	dev == iwdev->netdev) && (READ_ONCE(dev->flags) & IFF_UP)) {
389	const struct in_ifaddr *ifa;
390
391	idev = __in_dev_get_rcu(dev);
392	if (!idev)
393	continue;
394
395	in_dev_for_each_ifa_rcu(ifa, idev) {
396	ibdev_dbg(&iwdev->ibdev, "CM: IP=%pI4, vlan_id=%d, MAC=%pM\n",
397	&ifa->ifa_address, rdma_vlan_dev_vlan_id(dev),
398	dev->dev_addr);
399
400	ip_addr = ntohl(ifa->ifa_address);
401	irdma_manage_arp_cache(rf: iwdev->rf, mac_addr: dev->dev_addr,
402	ip_addr: &ip_addr, ipv4: true,
403	IRDMA_ARP_ADD);
404	}
405	}
406	}
407	rcu_read_unlock();
408	}
409
410	/**
411	* irdma_add_ip - add ip addresses
412	* @iwdev: irdma device
413	*
414	* Add ipv4/ipv6 addresses to the arp cache
415	*/
416	void irdma_add_ip(struct irdma_device *iwdev)
417	{
418	irdma_add_ipv4_addr(iwdev);
419	irdma_add_ipv6_addr(iwdev);
420	}
421
422	/**
423	* irdma_alloc_and_get_cqp_request - get cqp struct
424	* @cqp: device cqp ptr
425	* @wait: cqp to be used in wait mode
426	*/
427	struct irdma_cqp_request *irdma_alloc_and_get_cqp_request(struct irdma_cqp *cqp,
428	bool wait)
429	{
430	struct irdma_cqp_request *cqp_request = NULL;
431	unsigned long flags;
432
433	spin_lock_irqsave(&cqp->req_lock, flags);
434	if (!list_empty(head: &cqp->cqp_avail_reqs)) {
435	cqp_request = list_first_entry(&cqp->cqp_avail_reqs,
436	struct irdma_cqp_request, list);
437	list_del_init(entry: &cqp_request->list);
438	}
439	spin_unlock_irqrestore(lock: &cqp->req_lock, flags);
440	if (!cqp_request) {
441	cqp_request = kzalloc(sizeof(*cqp_request), GFP_ATOMIC);
442	if (cqp_request) {
443	cqp_request->dynamic = true;
444	if (wait)
445	init_waitqueue_head(&cqp_request->waitq);
446	}
447	}
448	if (!cqp_request) {
449	ibdev_dbg(to_ibdev(cqp->sc_cqp.dev), "ERR: CQP Request Fail: No Memory");
450	return NULL;
451	}
452
453	cqp_request->waiting = wait;
454	refcount_set(r: &cqp_request->refcnt, n: 1);
455	memset(&cqp_request->compl_info, 0, sizeof(cqp_request->compl_info));
456	memset(&cqp_request->info, 0, sizeof(cqp_request->info));
457
458	return cqp_request;
459	}
460
461	/**
462	* irdma_get_cqp_request - increase refcount for cqp_request
463	* @cqp_request: pointer to cqp_request instance
464	*/
465	static inline void irdma_get_cqp_request(struct irdma_cqp_request *cqp_request)
466	{
467	refcount_inc(r: &cqp_request->refcnt);
468	}
469
470	/**
471	* irdma_free_cqp_request - free cqp request
472	* @cqp: cqp ptr
473	* @cqp_request: to be put back in cqp list
474	*/
475	void irdma_free_cqp_request(struct irdma_cqp *cqp,
476	struct irdma_cqp_request *cqp_request)
477	{
478	unsigned long flags;
479
480	if (cqp_request->dynamic) {
481	kfree(objp: cqp_request);
482	} else {
483	WRITE_ONCE(cqp_request->request_done, false);
484	cqp_request->callback_fcn = NULL;
485	cqp_request->waiting = false;
486	cqp_request->pending = false;
487
488	spin_lock_irqsave(&cqp->req_lock, flags);
489	list_add_tail(new: &cqp_request->list, head: &cqp->cqp_avail_reqs);
490	spin_unlock_irqrestore(lock: &cqp->req_lock, flags);
491	}
492	wake_up(&cqp->remove_wq);
493	}
494
495	/**
496	* irdma_put_cqp_request - dec ref count and free if 0
497	* @cqp: cqp ptr
498	* @cqp_request: to be put back in cqp list
499	*/
500	void irdma_put_cqp_request(struct irdma_cqp *cqp,
501	struct irdma_cqp_request *cqp_request)
502	{
503	if (refcount_dec_and_test(r: &cqp_request->refcnt))
504	irdma_free_cqp_request(cqp, cqp_request);
505	}
506
507	/**
508	* irdma_free_pending_cqp_request -free pending cqp request objs
509	* @cqp: cqp ptr
510	* @cqp_request: to be put back in cqp list
511	*/
512	static void
513	irdma_free_pending_cqp_request(struct irdma_cqp *cqp,
514	struct irdma_cqp_request *cqp_request)
515	{
516	if (cqp_request->waiting) {
517	cqp_request->compl_info.error = true;
518	WRITE_ONCE(cqp_request->request_done, true);
519	wake_up(&cqp_request->waitq);
520	}
521	wait_event_timeout(cqp->remove_wq,
522	refcount_read(&cqp_request->refcnt) == 1, 1000);
523	irdma_put_cqp_request(cqp, cqp_request);
524	}
525
526	/**
527	* irdma_cleanup_deferred_cqp_ops - clean-up cqp with no completions
528	* @dev: sc_dev
529	* @cqp: cqp
530	*/
531	static void irdma_cleanup_deferred_cqp_ops(struct irdma_sc_dev *dev,
532	struct irdma_cqp *cqp)
533	{
534	u64 scratch;
535
536	/* process all CQP requests with deferred/pending completions */
537	while ((scratch = irdma_sc_cqp_cleanup_handler(dev)))
538	irdma_free_pending_cqp_request(cqp, cqp_request: (struct irdma_cqp_request *)
539	(uintptr_t)scratch);
540	}
541
542	/**
543	* irdma_cleanup_pending_cqp_op - clean-up cqp with no
544	* completions
545	* @rf: RDMA PCI function
546	*/
547	void irdma_cleanup_pending_cqp_op(struct irdma_pci_f *rf)
548	{
549	struct irdma_sc_dev *dev = &rf->sc_dev;
550	struct irdma_cqp *cqp = &rf->cqp;
551	struct irdma_cqp_request *cqp_request = NULL;
552	struct cqp_cmds_info *pcmdinfo = NULL;
553	u32 i, pending_work, wqe_idx;
554
555	if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_3)
556	irdma_cleanup_deferred_cqp_ops(dev, cqp);
557	pending_work = IRDMA_RING_USED_QUANTA(cqp->sc_cqp.sq_ring);
558	wqe_idx = IRDMA_RING_CURRENT_TAIL(cqp->sc_cqp.sq_ring);
559	for (i = 0; i < pending_work; i++) {
560	cqp_request = (struct irdma_cqp_request *)(unsigned long)
561	cqp->scratch_array[wqe_idx];
562	if (cqp_request)
563	irdma_free_pending_cqp_request(cqp, cqp_request);
564	wqe_idx = (wqe_idx + 1) % IRDMA_RING_SIZE(cqp->sc_cqp.sq_ring);
565	}
566
567	while (!list_empty(head: &dev->cqp_cmd_head)) {
568	pcmdinfo = irdma_remove_cqp_head(dev);
569	cqp_request =
570	container_of(pcmdinfo, struct irdma_cqp_request, info);
571	if (cqp_request)
572	irdma_free_pending_cqp_request(cqp, cqp_request);
573	}
574	}
575
576	static int irdma_get_timeout_threshold(struct irdma_sc_dev *dev)
577	{
578	u16 time_s = dev->vc_caps.cqp_timeout_s;
579
580	if (!time_s)
581	return CQP_TIMEOUT_THRESHOLD;
582
583	return time_s * 1000 / dev->hw_attrs.max_cqp_compl_wait_time_ms;
584	}
585
586	static int irdma_get_def_timeout_threshold(struct irdma_sc_dev *dev)
587	{
588	u16 time_s = dev->vc_caps.cqp_def_timeout_s;
589
590	if (!time_s)
591	return CQP_DEF_CMPL_TIMEOUT_THRESHOLD;
592
593	return time_s * 1000 / dev->hw_attrs.max_cqp_compl_wait_time_ms;
594	}
595
596	/**
597	* irdma_wait_event - wait for completion
598	* @rf: RDMA PCI function
599	* @cqp_request: cqp request to wait
600	*/
601	static int irdma_wait_event(struct irdma_pci_f *rf,
602	struct irdma_cqp_request *cqp_request)
603	{
604	struct irdma_cqp_timeout cqp_timeout = {};
605	int timeout_threshold = irdma_get_timeout_threshold(dev: &rf->sc_dev);
606	bool cqp_error = false;
607	int err_code = 0;
608
609	cqp_timeout.compl_cqp_cmds = atomic64_read(v: &rf->sc_dev.cqp->completed_ops);
610	do {
611	irdma_cqp_ce_handler(rf, cq: &rf->ccq.sc_cq);
612	if (wait_event_timeout(cqp_request->waitq,
613	READ_ONCE(cqp_request->request_done),
614	msecs_to_jiffies(CQP_COMPL_WAIT_TIME_MS)))
615	break;
616
617	if (cqp_request->pending)
618	/* There was a deferred or pending completion
619	* received for this CQP request, so we need
620	* to wait longer than usual.
621	*/
622	timeout_threshold =
623	irdma_get_def_timeout_threshold(dev: &rf->sc_dev);
624
625	irdma_check_cqp_progress(cqp_timeout: &cqp_timeout, dev: &rf->sc_dev);
626
627	if (cqp_timeout.count < timeout_threshold)
628	continue;
629
630	if (!rf->reset) {
631	rf->reset = true;
632	rf->gen_ops.request_reset(rf);
633	}
634	return -ETIMEDOUT;
635	} while (1);
636
637	cqp_error = cqp_request->compl_info.error;
638	if (cqp_error) {
639	err_code = -EIO;
640	if (cqp_request->compl_info.maj_err_code == 0xFFFF) {
641	if (cqp_request->compl_info.min_err_code == 0x8002)
642	err_code = -EBUSY;
643	else if (cqp_request->compl_info.min_err_code == 0x8029) {
644	if (!rf->reset) {
645	rf->reset = true;
646	rf->gen_ops.request_reset(rf);
647	}
648	}
649	}
650	}
651
652	return err_code;
653	}
654
655	static const char *const irdma_cqp_cmd_names[IRDMA_MAX_CQP_OPS] = {
656	[IRDMA_OP_CEQ_DESTROY] = "Destroy CEQ Cmd",
657	[IRDMA_OP_AEQ_DESTROY] = "Destroy AEQ Cmd",
658	[IRDMA_OP_DELETE_ARP_CACHE_ENTRY] = "Delete ARP Cache Cmd",
659	[IRDMA_OP_MANAGE_APBVT_ENTRY] = "Manage APBV Table Entry Cmd",
660	[IRDMA_OP_CEQ_CREATE] = "CEQ Create Cmd",
661	[IRDMA_OP_AEQ_CREATE] = "AEQ Destroy Cmd",
662	[IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY] = "Manage Quad Hash Table Entry Cmd",
663	[IRDMA_OP_QP_MODIFY] = "Modify QP Cmd",
664	[IRDMA_OP_QP_UPLOAD_CONTEXT] = "Upload Context Cmd",
665	[IRDMA_OP_CQ_CREATE] = "Create CQ Cmd",
666	[IRDMA_OP_CQ_DESTROY] = "Destroy CQ Cmd",
667	[IRDMA_OP_QP_CREATE] = "Create QP Cmd",
668	[IRDMA_OP_QP_DESTROY] = "Destroy QP Cmd",
669	[IRDMA_OP_ALLOC_STAG] = "Allocate STag Cmd",
670	[IRDMA_OP_MR_REG_NON_SHARED] = "Register Non-Shared MR Cmd",
671	[IRDMA_OP_DEALLOC_STAG] = "Deallocate STag Cmd",
672	[IRDMA_OP_MW_ALLOC] = "Allocate Memory Window Cmd",
673	[IRDMA_OP_QP_FLUSH_WQES] = "Flush QP Cmd",
674	[IRDMA_OP_ADD_ARP_CACHE_ENTRY] = "Add ARP Cache Cmd",
675	[IRDMA_OP_MANAGE_PUSH_PAGE] = "Manage Push Page Cmd",
676	[IRDMA_OP_UPDATE_PE_SDS] = "Update PE SDs Cmd",
677	[IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE] = "Manage HMC PM Function Table Cmd",
678	[IRDMA_OP_SUSPEND] = "Suspend QP Cmd",
679	[IRDMA_OP_RESUME] = "Resume QP Cmd",
680	[IRDMA_OP_MANAGE_VF_PBLE_BP] = "Manage VF PBLE Backing Pages Cmd",
681	[IRDMA_OP_QUERY_FPM_VAL] = "Query FPM Values Cmd",
682	[IRDMA_OP_COMMIT_FPM_VAL] = "Commit FPM Values Cmd",
683	[IRDMA_OP_AH_CREATE] = "Create Address Handle Cmd",
684	[IRDMA_OP_AH_MODIFY] = "Modify Address Handle Cmd",
685	[IRDMA_OP_AH_DESTROY] = "Destroy Address Handle Cmd",
686	[IRDMA_OP_MC_CREATE] = "Create Multicast Group Cmd",
687	[IRDMA_OP_MC_DESTROY] = "Destroy Multicast Group Cmd",
688	[IRDMA_OP_MC_MODIFY] = "Modify Multicast Group Cmd",
689	[IRDMA_OP_STATS_ALLOCATE] = "Add Statistics Instance Cmd",
690	[IRDMA_OP_STATS_FREE] = "Free Statistics Instance Cmd",
691	[IRDMA_OP_STATS_GATHER] = "Gather Statistics Cmd",
692	[IRDMA_OP_WS_ADD_NODE] = "Add Work Scheduler Node Cmd",
693	[IRDMA_OP_WS_MODIFY_NODE] = "Modify Work Scheduler Node Cmd",
694	[IRDMA_OP_WS_DELETE_NODE] = "Delete Work Scheduler Node Cmd",
695	[IRDMA_OP_SET_UP_MAP] = "Set UP-UP Mapping Cmd",
696	[IRDMA_OP_GEN_AE] = "Generate AE Cmd",
697	[IRDMA_OP_QUERY_RDMA_FEATURES] = "RDMA Get Features Cmd",
698	[IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY] = "Allocate Local MAC Entry Cmd",
699	[IRDMA_OP_ADD_LOCAL_MAC_ENTRY] = "Add Local MAC Entry Cmd",
700	[IRDMA_OP_DELETE_LOCAL_MAC_ENTRY] = "Delete Local MAC Entry Cmd",
701	[IRDMA_OP_CQ_MODIFY] = "CQ Modify Cmd",
702	[IRDMA_OP_SRQ_CREATE] = "Create SRQ Cmd",
703	[IRDMA_OP_SRQ_MODIFY] = "Modify SRQ Cmd",
704	[IRDMA_OP_SRQ_DESTROY] = "Destroy SRQ Cmd",
705	};
706
707	static const struct irdma_cqp_err_info irdma_noncrit_err_list[] = {
708	{0xffff, 0x8002, "Invalid State"},
709	{0xffff, 0x8006, "Flush No Wqe Pending"},
710	{0xffff, 0x8007, "Modify QP Bad Close"},
711	{0xffff, 0x8009, "LLP Closed"},
712	{0xffff, 0x800a, "Reset Not Sent"}
713	};
714
715	/**
716	* irdma_cqp_crit_err - check if CQP error is critical
717	* @dev: pointer to dev structure
718	* @cqp_cmd: code for last CQP operation
719	* @maj_err_code: major error code
720	* @min_err_code: minot error code
721	*/
722	bool irdma_cqp_crit_err(struct irdma_sc_dev *dev, u8 cqp_cmd,
723	u16 maj_err_code, u16 min_err_code)
724	{
725	int i;
726
727	for (i = 0; i < ARRAY_SIZE(irdma_noncrit_err_list); ++i) {
728	if (maj_err_code == irdma_noncrit_err_list[i].maj &&
729	min_err_code == irdma_noncrit_err_list[i].min) {
730	ibdev_dbg(to_ibdev(dev),
731	"CQP: [%s Error][%s] maj=0x%x min=0x%x\n",
732	irdma_noncrit_err_list[i].desc,
733	irdma_cqp_cmd_names[cqp_cmd], maj_err_code,
734	min_err_code);
735	return false;
736	}
737	}
738	return true;
739	}
740
741	/**
742	* irdma_handle_cqp_op - process cqp command
743	* @rf: RDMA PCI function
744	* @cqp_request: cqp request to process
745	*/
746	int irdma_handle_cqp_op(struct irdma_pci_f *rf,
747	struct irdma_cqp_request *cqp_request)
748	{
749	struct irdma_sc_dev *dev = &rf->sc_dev;
750	struct cqp_cmds_info *info = &cqp_request->info;
751	int status;
752	bool put_cqp_request = true;
753
754	if (rf->reset)
755	return -EBUSY;
756
757	irdma_get_cqp_request(cqp_request);
758	status = irdma_process_cqp_cmd(dev, pcmdinfo: info);
759	if (status)
760	goto err;
761
762	if (cqp_request->waiting) {
763	put_cqp_request = false;
764	status = irdma_wait_event(rf, cqp_request);
765	if (status)
766	goto err;
767	}
768
769	return 0;
770
771	err:
772	if (irdma_cqp_crit_err(dev, cqp_cmd: info->cqp_cmd,
773	maj_err_code: cqp_request->compl_info.maj_err_code,
774	min_err_code: cqp_request->compl_info.min_err_code))
775	ibdev_err(ibdev: &rf->iwdev->ibdev,
776	format: "[%s Error][op_code=%d] status=%d waiting=%d completion_err=%d maj=0x%x min=0x%x\n",
777	irdma_cqp_cmd_names[info->cqp_cmd], info->cqp_cmd, status, cqp_request->waiting,
778	cqp_request->compl_info.error, cqp_request->compl_info.maj_err_code,
779	cqp_request->compl_info.min_err_code);
780
781	if (put_cqp_request)
782	irdma_put_cqp_request(cqp: &rf->cqp, cqp_request);
783
784	return status;
785	}
786
787	void irdma_qp_add_ref(struct ib_qp *ibqp)
788	{
789	struct irdma_qp *iwqp = (struct irdma_qp *)ibqp;
790
791	refcount_inc(r: &iwqp->refcnt);
792	}
793
794	void irdma_qp_rem_ref(struct ib_qp *ibqp)
795	{
796	struct irdma_qp *iwqp = to_iwqp(ibqp);
797	struct irdma_device *iwdev = iwqp->iwdev;
798	u32 qp_num;
799	unsigned long flags;
800
801	spin_lock_irqsave(&iwdev->rf->qptable_lock, flags);
802	if (!refcount_dec_and_test(r: &iwqp->refcnt)) {
803	spin_unlock_irqrestore(lock: &iwdev->rf->qptable_lock, flags);
804	return;
805	}
806
807	qp_num = iwqp->ibqp.qp_num;
808	iwdev->rf->qp_table[qp_num] = NULL;
809	spin_unlock_irqrestore(lock: &iwdev->rf->qptable_lock, flags);
810	complete(&iwqp->free_qp);
811	}
812
813	void irdma_cq_add_ref(struct ib_cq *ibcq)
814	{
815	struct irdma_cq *iwcq = to_iwcq(ibcq);
816
817	refcount_inc(r: &iwcq->refcnt);
818	}
819
820	void irdma_cq_rem_ref(struct ib_cq *ibcq)
821	{
822	struct ib_device *ibdev = ibcq->device;
823	struct irdma_device *iwdev = to_iwdev(ibdev);
824	struct irdma_cq *iwcq = to_iwcq(ibcq);
825	unsigned long flags;
826
827	spin_lock_irqsave(&iwdev->rf->cqtable_lock, flags);
828	if (!refcount_dec_and_test(r: &iwcq->refcnt)) {
829	spin_unlock_irqrestore(lock: &iwdev->rf->cqtable_lock, flags);
830	return;
831	}
832
833	iwdev->rf->cq_table[iwcq->cq_num] = NULL;
834	spin_unlock_irqrestore(lock: &iwdev->rf->cqtable_lock, flags);
835	complete(&iwcq->free_cq);
836	}
837
838	struct ib_device *to_ibdev(struct irdma_sc_dev *dev)
839	{
840	return &(container_of(dev, struct irdma_pci_f, sc_dev))->iwdev->ibdev;
841	}
842
843	/**
844	* irdma_get_qp - get qp address
845	* @device: iwarp device
846	* @qpn: qp number
847	*/
848	struct ib_qp *irdma_get_qp(struct ib_device *device, int qpn)
849	{
850	struct irdma_device *iwdev = to_iwdev(ibdev: device);
851
852	if (qpn < IW_FIRST_QPN || qpn >= iwdev->rf->max_qp)
853	return NULL;
854
855	return &iwdev->rf->qp_table[qpn]->ibqp;
856	}
857
858	/**
859	* irdma_remove_cqp_head - return head entry and remove
860	* @dev: device
861	*/
862	void *irdma_remove_cqp_head(struct irdma_sc_dev *dev)
863	{
864	struct list_head *entry;
865	struct list_head *list = &dev->cqp_cmd_head;
866
867	if (list_empty(head: list))
868	return NULL;
869
870	entry = list->next;
871	list_del(entry);
872
873	return entry;
874	}
875
876	/**
877	* irdma_cqp_sds_cmd - create cqp command for sd
878	* @dev: hardware control device structure
879	* @sdinfo: information for sd cqp
880	*
881	*/
882	int irdma_cqp_sds_cmd(struct irdma_sc_dev *dev,
883	struct irdma_update_sds_info *sdinfo)
884	{
885	struct irdma_cqp_request *cqp_request;
886	struct cqp_cmds_info *cqp_info;
887	struct irdma_pci_f *rf = dev_to_rf(dev);
888	int status;
889
890	cqp_request = irdma_alloc_and_get_cqp_request(cqp: &rf->cqp, wait: true);
891	if (!cqp_request)
892	return -ENOMEM;
893
894	cqp_info = &cqp_request->info;
895	memcpy(&cqp_info->in.u.update_pe_sds.info, sdinfo,
896	sizeof(cqp_info->in.u.update_pe_sds.info));
897	cqp_info->cqp_cmd = IRDMA_OP_UPDATE_PE_SDS;
898	cqp_info->post_sq = 1;
899	cqp_info->in.u.update_pe_sds.dev = dev;
900	cqp_info->in.u.update_pe_sds.scratch = (uintptr_t)cqp_request;
901
902	status = irdma_handle_cqp_op(rf, cqp_request);
903	irdma_put_cqp_request(cqp: &rf->cqp, cqp_request);
904
905	return status;
906	}
907
908	/**
909	* irdma_cqp_qp_suspend_resume - cqp command for suspend/resume
910	* @qp: hardware control qp
911	* @op: suspend or resume
912	*/
913	int irdma_cqp_qp_suspend_resume(struct irdma_sc_qp *qp, u8 op)
914	{
915	struct irdma_sc_dev *dev = qp->dev;
916	struct irdma_cqp_request *cqp_request;
917	struct irdma_sc_cqp *cqp = dev->cqp;
918	struct cqp_cmds_info *cqp_info;
919	struct irdma_pci_f *rf = dev_to_rf(dev);
920	int status;
921
922	cqp_request = irdma_alloc_and_get_cqp_request(cqp: &rf->cqp, wait: false);
923	if (!cqp_request)
924	return -ENOMEM;
925
926	cqp_info = &cqp_request->info;
927	cqp_info->cqp_cmd = op;
928	cqp_info->in.u.suspend_resume.cqp = cqp;
929	cqp_info->in.u.suspend_resume.qp = qp;
930	cqp_info->in.u.suspend_resume.scratch = (uintptr_t)cqp_request;
931
932	status = irdma_handle_cqp_op(rf, cqp_request);
933	irdma_put_cqp_request(cqp: &rf->cqp, cqp_request);
934
935	return status;
936	}
937
938	/**
939	* irdma_term_modify_qp - modify qp for term message
940	* @qp: hardware control qp
941	* @next_state: qp's next state
942	* @term: terminate code
943	* @term_len: length
944	*/
945	void irdma_term_modify_qp(struct irdma_sc_qp *qp, u8 next_state, u8 term,
946	u8 term_len)
947	{
948	struct irdma_qp *iwqp;
949
950	iwqp = qp->qp_uk.back_qp;
951	irdma_next_iw_state(iwqp, state: next_state, del_hash: 0, term, term_len);
952	};
953
954	/**
955	* irdma_terminate_done - after terminate is completed
956	* @qp: hardware control qp
957	* @timeout_occurred: indicates if terminate timer expired
958	*/
959	void irdma_terminate_done(struct irdma_sc_qp *qp, int timeout_occurred)
960	{
961	struct irdma_qp *iwqp;
962	u8 hte = 0;
963	bool first_time;
964	unsigned long flags;
965
966	iwqp = qp->qp_uk.back_qp;
967	spin_lock_irqsave(&iwqp->lock, flags);
968	if (iwqp->hte_added) {
969	iwqp->hte_added = 0;
970	hte = 1;
971	}
972	first_time = !(qp->term_flags & IRDMA_TERM_DONE);
973	qp->term_flags |= IRDMA_TERM_DONE;
974	spin_unlock_irqrestore(lock: &iwqp->lock, flags);
975	if (first_time) {
976	if (!timeout_occurred)
977	irdma_terminate_del_timer(qp);
978
979	irdma_next_iw_state(iwqp, IRDMA_QP_STATE_ERROR, del_hash: hte, term: 0, term_len: 0);
980	irdma_cm_disconn(qp: iwqp);
981	}
982	}
983
984	static void irdma_terminate_timeout(struct timer_list *t)
985	{
986	struct irdma_qp *iwqp = timer_container_of(iwqp, t, terminate_timer);
987	struct irdma_sc_qp *qp = &iwqp->sc_qp;
988
989	irdma_terminate_done(qp, timeout_occurred: 1);
990	irdma_qp_rem_ref(ibqp: &iwqp->ibqp);
991	}
992
993	/**
994	* irdma_terminate_start_timer - start terminate timeout
995	* @qp: hardware control qp
996	*/
997	void irdma_terminate_start_timer(struct irdma_sc_qp *qp)
998	{
999	struct irdma_qp *iwqp;
1000
1001	iwqp = qp->qp_uk.back_qp;
1002	irdma_qp_add_ref(ibqp: &iwqp->ibqp);
1003	timer_setup(&iwqp->terminate_timer, irdma_terminate_timeout, 0);
1004	iwqp->terminate_timer.expires = jiffies + HZ;
1005
1006	add_timer(timer: &iwqp->terminate_timer);
1007	}
1008
1009	/**
1010	* irdma_terminate_del_timer - delete terminate timeout
1011	* @qp: hardware control qp
1012	*/
1013	void irdma_terminate_del_timer(struct irdma_sc_qp *qp)
1014	{
1015	struct irdma_qp *iwqp;
1016	int ret;
1017
1018	iwqp = qp->qp_uk.back_qp;
1019	ret = timer_delete(timer: &iwqp->terminate_timer);
1020	if (ret)
1021	irdma_qp_rem_ref(ibqp: &iwqp->ibqp);
1022	}
1023
1024	/**
1025	* irdma_cqp_cq_create_cmd - create a cq for the cqp
1026	* @dev: device pointer
1027	* @cq: pointer to created cq
1028	*/
1029	int irdma_cqp_cq_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq)
1030	{
1031	struct irdma_pci_f *rf = dev_to_rf(dev);
1032	struct irdma_cqp *iwcqp = &rf->cqp;
1033	struct irdma_cqp_request *cqp_request;
1034	struct cqp_cmds_info *cqp_info;
1035	int status;
1036
1037	cqp_request = irdma_alloc_and_get_cqp_request(cqp: iwcqp, wait: true);
1038	if (!cqp_request)
1039	return -ENOMEM;
1040
1041	cqp_info = &cqp_request->info;
1042	cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE;
1043	cqp_info->post_sq = 1;
1044	cqp_info->in.u.cq_create.cq = cq;
1045	cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request;
1046
1047	status = irdma_handle_cqp_op(rf, cqp_request);
1048	irdma_put_cqp_request(cqp: iwcqp, cqp_request);
1049
1050	return status;
1051	}
1052
1053	/**
1054	* irdma_cqp_qp_create_cmd - create a qp for the cqp
1055	* @dev: device pointer
1056	* @qp: pointer to created qp
1057	*/
1058	int irdma_cqp_qp_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
1059	{
1060	struct irdma_pci_f *rf = dev_to_rf(dev);
1061	struct irdma_cqp *iwcqp = &rf->cqp;
1062	struct irdma_cqp_request *cqp_request;
1063	struct cqp_cmds_info *cqp_info;
1064	struct irdma_create_qp_info *qp_info;
1065	int status;
1066
1067	cqp_request = irdma_alloc_and_get_cqp_request(cqp: iwcqp, wait: true);
1068	if (!cqp_request)
1069	return -ENOMEM;
1070
1071	cqp_info = &cqp_request->info;
1072	qp_info = &cqp_request->info.in.u.qp_create.info;
1073	qp_info->cq_num_valid = true;
1074	qp_info->next_iwarp_state = IRDMA_QP_STATE_RTS;
1075	cqp_info->cqp_cmd = IRDMA_OP_QP_CREATE;
1076	cqp_info->post_sq = 1;
1077	cqp_info->in.u.qp_create.qp = qp;
1078	cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request;
1079
1080	status = irdma_handle_cqp_op(rf, cqp_request);
1081	irdma_put_cqp_request(cqp: iwcqp, cqp_request);
1082
1083	return status;
1084	}
1085
1086	/**
1087	* irdma_dealloc_push_page - free a push page for qp
1088	* @rf: RDMA PCI function
1089	* @qp: hardware control qp
1090	*/
1091	static void irdma_dealloc_push_page(struct irdma_pci_f *rf,
1092	struct irdma_sc_qp *qp)
1093	{
1094	struct irdma_cqp_request *cqp_request;
1095	struct cqp_cmds_info *cqp_info;
1096	int status;
1097
1098	if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX)
1099	return;
1100
1101	cqp_request = irdma_alloc_and_get_cqp_request(cqp: &rf->cqp, wait: false);
1102	if (!cqp_request)
1103	return;
1104
1105	cqp_info = &cqp_request->info;
1106	cqp_info->cqp_cmd = IRDMA_OP_MANAGE_PUSH_PAGE;
1107	cqp_info->post_sq = 1;
1108	cqp_info->in.u.manage_push_page.info.push_idx = qp->push_idx;
1109	cqp_info->in.u.manage_push_page.info.qs_handle = qp->qs_handle;
1110	cqp_info->in.u.manage_push_page.info.free_page = 1;
1111	cqp_info->in.u.manage_push_page.info.push_page_type = 0;
1112	cqp_info->in.u.manage_push_page.cqp = &rf->cqp.sc_cqp;
1113	cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request;
1114	status = irdma_handle_cqp_op(rf, cqp_request);
1115	if (!status)
1116	qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX;
1117	irdma_put_cqp_request(cqp: &rf->cqp, cqp_request);
1118	}
1119
1120	static void irdma_free_gsi_qp_rsrc(struct irdma_qp *iwqp, u32 qp_num)
1121	{
1122	struct irdma_device *iwdev = iwqp->iwdev;
1123	struct irdma_pci_f *rf = iwdev->rf;
1124	unsigned long flags;
1125
1126	if (rf->sc_dev.hw_attrs.uk_attrs.hw_rev < IRDMA_GEN_3)
1127	return;
1128
1129	irdma_vchnl_req_del_vport(dev: &rf->sc_dev, vport_id: iwdev->vport_id, qp1_id: qp_num);
1130
1131	if (qp_num == 1) {
1132	spin_lock_irqsave(&rf->rsrc_lock, flags);
1133	rf->hwqp1_rsvd = false;
1134	spin_unlock_irqrestore(lock: &rf->rsrc_lock, flags);
1135	} else if (qp_num > 2) {
1136	irdma_free_rsrc(rf, rsrc_array: rf->allocated_qps, rsrc_num: qp_num);
1137	}
1138	}
1139
1140	/**
1141	* irdma_free_qp_rsrc - free up memory resources for qp
1142	* @iwqp: qp ptr (user or kernel)
1143	*/
1144	void irdma_free_qp_rsrc(struct irdma_qp *iwqp)
1145	{
1146	struct irdma_device *iwdev = iwqp->iwdev;
1147	struct irdma_pci_f *rf = iwdev->rf;
1148	u32 qp_num = iwqp->sc_qp.qp_uk.qp_id;
1149
1150	irdma_ieq_cleanup_qp(ieq: iwdev->vsi.ieq, qp: &iwqp->sc_qp);
1151	irdma_dealloc_push_page(rf, qp: &iwqp->sc_qp);
1152	if (iwqp->sc_qp.vsi) {
1153	irdma_qp_rem_qos(qp: &iwqp->sc_qp);
1154	iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi,
1155	iwqp->sc_qp.user_pri);
1156	}
1157
1158	if (iwqp->ibqp.qp_type == IB_QPT_GSI) {
1159	irdma_free_gsi_qp_rsrc(iwqp, qp_num);
1160	} else {
1161	if (qp_num > 2)
1162	irdma_free_rsrc(rf, rsrc_array: rf->allocated_qps, rsrc_num: qp_num);
1163	}
1164	dma_free_coherent(dev: rf->sc_dev.hw->device, size: iwqp->q2_ctx_mem.size,
1165	cpu_addr: iwqp->q2_ctx_mem.va, dma_handle: iwqp->q2_ctx_mem.pa);
1166	iwqp->q2_ctx_mem.va = NULL;
1167	dma_free_coherent(dev: rf->sc_dev.hw->device, size: iwqp->kqp.dma_mem.size,
1168	cpu_addr: iwqp->kqp.dma_mem.va, dma_handle: iwqp->kqp.dma_mem.pa);
1169	iwqp->kqp.dma_mem.va = NULL;
1170	kfree(objp: iwqp->kqp.sq_wrid_mem);
1171	kfree(objp: iwqp->kqp.rq_wrid_mem);
1172	}
1173
1174	/**
1175	* irdma_srq_wq_destroy - send srq destroy cqp
1176	* @rf: RDMA PCI function
1177	* @srq: hardware control srq
1178	*/
1179	void irdma_srq_wq_destroy(struct irdma_pci_f *rf, struct irdma_sc_srq *srq)
1180	{
1181	struct irdma_cqp_request *cqp_request;
1182	struct cqp_cmds_info *cqp_info;
1183
1184	cqp_request = irdma_alloc_and_get_cqp_request(cqp: &rf->cqp, wait: true);
1185	if (!cqp_request)
1186	return;
1187
1188	cqp_info = &cqp_request->info;
1189	cqp_info->cqp_cmd = IRDMA_OP_SRQ_DESTROY;
1190	cqp_info->post_sq = 1;
1191	cqp_info->in.u.srq_destroy.srq = srq;
1192	cqp_info->in.u.srq_destroy.scratch = (uintptr_t)cqp_request;
1193
1194	irdma_handle_cqp_op(rf, cqp_request);
1195	irdma_put_cqp_request(cqp: &rf->cqp, cqp_request);
1196	}
1197
1198	/**
1199	* irdma_cq_wq_destroy - send cq destroy cqp
1200	* @rf: RDMA PCI function
1201	* @cq: hardware control cq
1202	*/
1203	void irdma_cq_wq_destroy(struct irdma_pci_f *rf, struct irdma_sc_cq *cq)
1204	{
1205	struct irdma_cqp_request *cqp_request;
1206	struct cqp_cmds_info *cqp_info;
1207
1208	cqp_request = irdma_alloc_and_get_cqp_request(cqp: &rf->cqp, wait: true);
1209	if (!cqp_request)
1210	return;
1211
1212	cqp_info = &cqp_request->info;
1213	cqp_info->cqp_cmd = IRDMA_OP_CQ_DESTROY;
1214	cqp_info->post_sq = 1;
1215	cqp_info->in.u.cq_destroy.cq = cq;
1216	cqp_info->in.u.cq_destroy.scratch = (uintptr_t)cqp_request;
1217
1218	irdma_handle_cqp_op(rf, cqp_request);
1219	irdma_put_cqp_request(cqp: &rf->cqp, cqp_request);
1220	}
1221
1222	/**
1223	* irdma_hw_modify_qp_callback - handle state for modifyQPs that don't wait
1224	* @cqp_request: modify QP completion
1225	*/
1226	static void irdma_hw_modify_qp_callback(struct irdma_cqp_request *cqp_request)
1227	{
1228	struct cqp_cmds_info *cqp_info;
1229	struct irdma_qp *iwqp;
1230
1231	cqp_info = &cqp_request->info;
1232	iwqp = cqp_info->in.u.qp_modify.qp->qp_uk.back_qp;
1233	atomic_dec(v: &iwqp->hw_mod_qp_pend);
1234	wake_up(&iwqp->mod_qp_waitq);
1235	}
1236
1237	/**
1238	* irdma_hw_modify_qp - setup cqp for modify qp
1239	* @iwdev: RDMA device
1240	* @iwqp: qp ptr (user or kernel)
1241	* @info: info for modify qp
1242	* @wait: flag to wait or not for modify qp completion
1243	*/
1244	int irdma_hw_modify_qp(struct irdma_device *iwdev, struct irdma_qp *iwqp,
1245	struct irdma_modify_qp_info *info, bool wait)
1246	{
1247	int status;
1248	struct irdma_pci_f *rf = iwdev->rf;
1249	struct irdma_cqp_request *cqp_request;
1250	struct cqp_cmds_info *cqp_info;
1251	struct irdma_modify_qp_info *m_info;
1252
1253	cqp_request = irdma_alloc_and_get_cqp_request(cqp: &rf->cqp, wait);
1254	if (!cqp_request)
1255	return -ENOMEM;
1256
1257	if (!wait) {
1258	cqp_request->callback_fcn = irdma_hw_modify_qp_callback;
1259	atomic_inc(v: &iwqp->hw_mod_qp_pend);
1260	}
1261	cqp_info = &cqp_request->info;
1262	m_info = &cqp_info->in.u.qp_modify.info;
1263	memcpy(m_info, info, sizeof(*m_info));
1264	cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY;
1265	cqp_info->post_sq = 1;
1266	cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp;
1267	cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request;
1268	status = irdma_handle_cqp_op(rf, cqp_request);
1269	irdma_put_cqp_request(cqp: &rf->cqp, cqp_request);
1270	if (status) {
1271	if (rdma_protocol_roce(device: &iwdev->ibdev, port_num: 1))
1272	return status;
1273
1274	switch (m_info->next_iwarp_state) {
1275	struct irdma_gen_ae_info ae_info;
1276
1277	case IRDMA_QP_STATE_RTS:
1278	case IRDMA_QP_STATE_IDLE:
1279	case IRDMA_QP_STATE_TERMINATE:
1280	case IRDMA_QP_STATE_CLOSING:
1281	if (info->curr_iwarp_state == IRDMA_QP_STATE_IDLE)
1282	irdma_send_reset(cm_node: iwqp->cm_node);
1283	else
1284	iwqp->sc_qp.term_flags = IRDMA_TERM_DONE;
1285	if (!wait) {
1286	ae_info.ae_code = IRDMA_AE_BAD_CLOSE;
1287	ae_info.ae_src = 0;
1288	irdma_gen_ae(rf, qp: &iwqp->sc_qp, info: &ae_info, wait: false);
1289	} else {
1290	cqp_request = irdma_alloc_and_get_cqp_request(cqp: &rf->cqp,
1291	wait);
1292	if (!cqp_request)
1293	return -ENOMEM;
1294
1295	cqp_info = &cqp_request->info;
1296	m_info = &cqp_info->in.u.qp_modify.info;
1297	memcpy(m_info, info, sizeof(*m_info));
1298	cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY;
1299	cqp_info->post_sq = 1;
1300	cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp;
1301	cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request;
1302	m_info->next_iwarp_state = IRDMA_QP_STATE_ERROR;
1303	m_info->reset_tcp_conn = true;
1304	irdma_handle_cqp_op(rf, cqp_request);
1305	irdma_put_cqp_request(cqp: &rf->cqp, cqp_request);
1306	}
1307	break;
1308	case IRDMA_QP_STATE_ERROR:
1309	default:
1310	break;
1311	}
1312	}
1313
1314	return status;
1315	}
1316
1317	/**
1318	* irdma_cqp_cq_destroy_cmd - destroy the cqp cq
1319	* @dev: device pointer
1320	* @cq: pointer to cq
1321	*/
1322	void irdma_cqp_cq_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq)
1323	{
1324	struct irdma_pci_f *rf = dev_to_rf(dev);
1325
1326	irdma_cq_wq_destroy(rf, cq);
1327	}
1328
1329	/**
1330	* irdma_cqp_qp_destroy_cmd - destroy the cqp
1331	* @dev: device pointer
1332	* @qp: pointer to qp
1333	*/
1334	int irdma_cqp_qp_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
1335	{
1336	struct irdma_pci_f *rf = dev_to_rf(dev);
1337	struct irdma_cqp *iwcqp = &rf->cqp;
1338	struct irdma_cqp_request *cqp_request;
1339	struct cqp_cmds_info *cqp_info;
1340	int status;
1341
1342	cqp_request = irdma_alloc_and_get_cqp_request(cqp: iwcqp, wait: true);
1343	if (!cqp_request)
1344	return -ENOMEM;
1345
1346	cqp_info = &cqp_request->info;
1347	cqp_info->cqp_cmd = IRDMA_OP_QP_DESTROY;
1348	cqp_info->post_sq = 1;
1349	cqp_info->in.u.qp_destroy.qp = qp;
1350	cqp_info->in.u.qp_destroy.scratch = (uintptr_t)cqp_request;
1351	cqp_info->in.u.qp_destroy.remove_hash_idx = true;
1352
1353	status = irdma_handle_cqp_op(rf, cqp_request);
1354	irdma_put_cqp_request(cqp: &rf->cqp, cqp_request);
1355
1356	return status;
1357	}
1358
1359	/**
1360	* irdma_ieq_mpa_crc_ae - generate AE for crc error
1361	* @dev: hardware control device structure
1362	* @qp: hardware control qp
1363	*/
1364	void irdma_ieq_mpa_crc_ae(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
1365	{
1366	struct irdma_gen_ae_info info = {};
1367	struct irdma_pci_f *rf = dev_to_rf(dev);
1368
1369	ibdev_dbg(&rf->iwdev->ibdev, "AEQ: Generate MPA CRC AE\n");
1370	info.ae_code = IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR;
1371	info.ae_src = IRDMA_AE_SOURCE_RQ;
1372	irdma_gen_ae(rf, qp, info: &info, wait: false);
1373	}
1374
1375	/**
1376	* irdma_ieq_check_mpacrc - check if mpa crc is OK
1377	* @addr: address of buffer for crc
1378	* @len: length of buffer
1379	* @val: value to be compared
1380	*/
1381	int irdma_ieq_check_mpacrc(const void *addr, u32 len, u32 val)
1382	{
1383	if ((__force u32)cpu_to_le32(~crc32c(~0, addr, len)) != val)
1384	return -EINVAL;
1385
1386	return 0;
1387	}
1388
1389	/**
1390	* irdma_ieq_get_qp - get qp based on quad in puda buffer
1391	* @dev: hardware control device structure
1392	* @buf: receive puda buffer on exception q
1393	*/
1394	struct irdma_sc_qp *irdma_ieq_get_qp(struct irdma_sc_dev *dev,
1395	struct irdma_puda_buf *buf)
1396	{
1397	struct irdma_qp *iwqp;
1398	struct irdma_cm_node *cm_node;
1399	struct irdma_device *iwdev = buf->vsi->back_vsi;
1400	u32 loc_addr[4] = {};
1401	u32 rem_addr[4] = {};
1402	u16 loc_port, rem_port;
1403	struct ipv6hdr *ip6h;
1404	struct iphdr *iph = (struct iphdr *)buf->iph;
1405	struct tcphdr *tcph = (struct tcphdr *)buf->tcph;
1406
1407	if (iph->version == 4) {
1408	loc_addr[0] = ntohl(iph->daddr);
1409	rem_addr[0] = ntohl(iph->saddr);
1410	} else {
1411	ip6h = (struct ipv6hdr *)buf->iph;
1412	irdma_copy_ip_ntohl(dst: loc_addr, src: ip6h->daddr.in6_u.u6_addr32);
1413	irdma_copy_ip_ntohl(dst: rem_addr, src: ip6h->saddr.in6_u.u6_addr32);
1414	}
1415	loc_port = ntohs(tcph->dest);
1416	rem_port = ntohs(tcph->source);
1417	cm_node = irdma_find_node(cm_core: &iwdev->cm_core, rem_port, rem_addr, loc_port,
1418	loc_addr, vlan_id: buf->vlan_valid ? buf->vlan_id : 0xFFFF);
1419	if (!cm_node)
1420	return NULL;
1421
1422	iwqp = cm_node->iwqp;
1423	irdma_rem_ref_cm_node(cm_node);
1424
1425	return &iwqp->sc_qp;
1426	}
1427
1428	/**
1429	* irdma_send_ieq_ack - ACKs for duplicate or OOO partials FPDUs
1430	* @qp: qp ptr
1431	*/
1432	void irdma_send_ieq_ack(struct irdma_sc_qp *qp)
1433	{
1434	struct irdma_cm_node *cm_node = ((struct irdma_qp *)qp->qp_uk.back_qp)->cm_node;
1435	struct irdma_puda_buf *buf = qp->pfpdu.lastrcv_buf;
1436	struct tcphdr *tcph = (struct tcphdr *)buf->tcph;
1437
1438	cm_node->tcp_cntxt.rcv_nxt = qp->pfpdu.nextseqnum;
1439	cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq);
1440
1441	irdma_send_ack(cm_node);
1442	}
1443
1444	/**
1445	* irdma_puda_ieq_get_ah_info - get AH info from IEQ buffer
1446	* @qp: qp pointer
1447	* @ah_info: AH info pointer
1448	*/
1449	void irdma_puda_ieq_get_ah_info(struct irdma_sc_qp *qp,
1450	struct irdma_ah_info *ah_info)
1451	{
1452	struct irdma_puda_buf *buf = qp->pfpdu.ah_buf;
1453	struct iphdr *iph;
1454	struct ipv6hdr *ip6h;
1455
1456	memset(ah_info, 0, sizeof(*ah_info));
1457	ah_info->do_lpbk = true;
1458	ah_info->vlan_tag = buf->vlan_id;
1459	ah_info->insert_vlan_tag = buf->vlan_valid;
1460	ah_info->ipv4_valid = buf->ipv4;
1461	ah_info->vsi = qp->vsi;
1462
1463	if (buf->smac_valid)
1464	ether_addr_copy(dst: ah_info->mac_addr, src: buf->smac);
1465
1466	if (buf->ipv4) {
1467	ah_info->ipv4_valid = true;
1468	iph = (struct iphdr *)buf->iph;
1469	ah_info->hop_ttl = iph->ttl;
1470	ah_info->tc_tos = iph->tos;
1471	ah_info->dest_ip_addr[0] = ntohl(iph->daddr);
1472	ah_info->src_ip_addr[0] = ntohl(iph->saddr);
1473	} else {
1474	ip6h = (struct ipv6hdr *)buf->iph;
1475	ah_info->hop_ttl = ip6h->hop_limit;
1476	ah_info->tc_tos = ip6h->priority;
1477	irdma_copy_ip_ntohl(dst: ah_info->dest_ip_addr,
1478	src: ip6h->daddr.in6_u.u6_addr32);
1479	irdma_copy_ip_ntohl(dst: ah_info->src_ip_addr,
1480	src: ip6h->saddr.in6_u.u6_addr32);
1481	}
1482
1483	ah_info->dst_arpindex = irdma_arp_table(rf: dev_to_rf(dev: qp->dev),
1484	ip_addr: ah_info->dest_ip_addr,
1485	ipv4: ah_info->ipv4_valid,
1486	NULL, IRDMA_ARP_RESOLVE);
1487	}
1488
1489	/**
1490	* irdma_gen1_ieq_update_tcpip_info - update tcpip in the buffer
1491	* @buf: puda to update
1492	* @len: length of buffer
1493	* @seqnum: seq number for tcp
1494	*/
1495	static void irdma_gen1_ieq_update_tcpip_info(struct irdma_puda_buf *buf,
1496	u16 len, u32 seqnum)
1497	{
1498	struct tcphdr *tcph;
1499	struct iphdr *iph;
1500	u16 iphlen;
1501	u16 pktsize;
1502	u8 *addr = buf->mem.va;
1503
1504	iphlen = (buf->ipv4) ? 20 : 40;
1505	iph = (struct iphdr *)(addr + buf->maclen);
1506	tcph = (struct tcphdr *)(addr + buf->maclen + iphlen);
1507	pktsize = len + buf->tcphlen + iphlen;
1508	iph->tot_len = htons(pktsize);
1509	tcph->seq = htonl(seqnum);
1510	}
1511
1512	/**
1513	* irdma_ieq_update_tcpip_info - update tcpip in the buffer
1514	* @buf: puda to update
1515	* @len: length of buffer
1516	* @seqnum: seq number for tcp
1517	*/
1518	void irdma_ieq_update_tcpip_info(struct irdma_puda_buf *buf, u16 len,
1519	u32 seqnum)
1520	{
1521	struct tcphdr *tcph;
1522	u8 *addr;
1523
1524	if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
1525	return irdma_gen1_ieq_update_tcpip_info(buf, len, seqnum);
1526
1527	addr = buf->mem.va;
1528	tcph = (struct tcphdr *)addr;
1529	tcph->seq = htonl(seqnum);
1530	}
1531
1532	/**
1533	* irdma_gen1_puda_get_tcpip_info - get tcpip info from puda
1534	* buffer
1535	* @info: to get information
1536	* @buf: puda buffer
1537	*/
1538	static int irdma_gen1_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,
1539	struct irdma_puda_buf *buf)
1540	{
1541	struct iphdr *iph;
1542	struct ipv6hdr *ip6h;
1543	struct tcphdr *tcph;
1544	u16 iphlen;
1545	u16 pkt_len;
1546	u8 *mem = buf->mem.va;
1547	struct ethhdr *ethh = buf->mem.va;
1548
1549	if (ethh->h_proto == htons(0x8100)) {
1550	info->vlan_valid = true;
1551	buf->vlan_id = ntohs(((struct vlan_ethhdr *)ethh)->h_vlan_TCI) &
1552	VLAN_VID_MASK;
1553	}
1554
1555	buf->maclen = (info->vlan_valid) ? 18 : 14;
1556	iphlen = (info->l3proto) ? 40 : 20;
1557	buf->ipv4 = (info->l3proto) ? false : true;
1558	buf->iph = mem + buf->maclen;
1559	iph = (struct iphdr *)buf->iph;
1560	buf->tcph = buf->iph + iphlen;
1561	tcph = (struct tcphdr *)buf->tcph;
1562
1563	if (buf->ipv4) {
1564	pkt_len = ntohs(iph->tot_len);
1565	} else {
1566	ip6h = (struct ipv6hdr *)buf->iph;
1567	pkt_len = ntohs(ip6h->payload_len) + iphlen;
1568	}
1569
1570	buf->totallen = pkt_len + buf->maclen;
1571
1572	if (info->payload_len < buf->totallen) {
1573	ibdev_dbg(to_ibdev(buf->vsi->dev),
1574	"ERR: payload_len = 0x%x totallen expected0x%x\n",
1575	info->payload_len, buf->totallen);
1576	return -EINVAL;
1577	}
1578
1579	buf->tcphlen = tcph->doff << 2;
1580	buf->datalen = pkt_len - iphlen - buf->tcphlen;
1581	buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL;
1582	buf->hdrlen = buf->maclen + iphlen + buf->tcphlen;
1583	buf->seqnum = ntohl(tcph->seq);
1584
1585	return 0;
1586	}
1587
1588	/**
1589	* irdma_puda_get_tcpip_info - get tcpip info from puda buffer
1590	* @info: to get information
1591	* @buf: puda buffer
1592	*/
1593	int irdma_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,
1594	struct irdma_puda_buf *buf)
1595	{
1596	struct tcphdr *tcph;
1597	u32 pkt_len;
1598	u8 *mem;
1599
1600	if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
1601	return irdma_gen1_puda_get_tcpip_info(info, buf);
1602
1603	mem = buf->mem.va;
1604	buf->vlan_valid = info->vlan_valid;
1605	if (info->vlan_valid)
1606	buf->vlan_id = info->vlan;
1607
1608	buf->ipv4 = info->ipv4;
1609	if (buf->ipv4)
1610	buf->iph = mem + IRDMA_IPV4_PAD;
1611	else
1612	buf->iph = mem;
1613
1614	buf->tcph = mem + IRDMA_TCP_OFFSET;
1615	tcph = (struct tcphdr *)buf->tcph;
1616	pkt_len = info->payload_len;
1617	buf->totallen = pkt_len;
1618	buf->tcphlen = tcph->doff << 2;
1619	buf->datalen = pkt_len - IRDMA_TCP_OFFSET - buf->tcphlen;
1620	buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL;
1621	buf->hdrlen = IRDMA_TCP_OFFSET + buf->tcphlen;
1622	buf->seqnum = ntohl(tcph->seq);
1623
1624	if (info->smac_valid) {
1625	ether_addr_copy(dst: buf->smac, src: info->smac);
1626	buf->smac_valid = true;
1627	}
1628
1629	return 0;
1630	}
1631
1632	/**
1633	* irdma_hw_stats_timeout - Stats timer-handler which updates all HW stats
1634	* @t: timer_list pointer
1635	*/
1636	static void irdma_hw_stats_timeout(struct timer_list *t)
1637	{
1638	struct irdma_vsi_pestat *pf_devstat =
1639	timer_container_of(pf_devstat, t, stats_timer);
1640	struct irdma_sc_vsi *sc_vsi = pf_devstat->vsi;
1641
1642	if (sc_vsi->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
1643	irdma_cqp_gather_stats_cmd(dev: sc_vsi->dev, pestat: sc_vsi->pestat, wait: false);
1644	else
1645	irdma_cqp_gather_stats_gen1(dev: sc_vsi->dev, pestat: sc_vsi->pestat);
1646
1647	mod_timer(timer: &pf_devstat->stats_timer,
1648	expires: jiffies + msecs_to_jiffies(STATS_TIMER_DELAY));
1649	}
1650
1651	/**
1652	* irdma_hw_stats_start_timer - Start periodic stats timer
1653	* @vsi: vsi structure pointer
1654	*/
1655	void irdma_hw_stats_start_timer(struct irdma_sc_vsi *vsi)
1656	{
1657	struct irdma_vsi_pestat *devstat = vsi->pestat;
1658
1659	timer_setup(&devstat->stats_timer, irdma_hw_stats_timeout, 0);
1660	mod_timer(timer: &devstat->stats_timer,
1661	expires: jiffies + msecs_to_jiffies(STATS_TIMER_DELAY));
1662	}
1663
1664	/**
1665	* irdma_hw_stats_stop_timer - Delete periodic stats timer
1666	* @vsi: pointer to vsi structure
1667	*/
1668	void irdma_hw_stats_stop_timer(struct irdma_sc_vsi *vsi)
1669	{
1670	struct irdma_vsi_pestat *devstat = vsi->pestat;
1671
1672	timer_delete_sync(timer: &devstat->stats_timer);
1673	}
1674
1675	/**
1676	* irdma_process_stats - Checking for wrap and update stats
1677	* @pestat: stats structure pointer
1678	*/
1679	static inline void irdma_process_stats(struct irdma_vsi_pestat *pestat)
1680	{
1681	sc_vsi_update_stats(vsi: pestat->vsi);
1682	}
1683
1684	/**
1685	* irdma_cqp_gather_stats_gen1 - Gather stats
1686	* @dev: pointer to device structure
1687	* @pestat: statistics structure
1688	*/
1689	void irdma_cqp_gather_stats_gen1(struct irdma_sc_dev *dev,
1690	struct irdma_vsi_pestat *pestat)
1691	{
1692	struct irdma_gather_stats *gather_stats =
1693	pestat->gather_info.gather_stats_va;
1694	const struct irdma_hw_stat_map *map = dev->hw_stats_map;
1695	u16 max_stats_idx = dev->hw_attrs.max_stat_idx;
1696	u32 stats_inst_offset_32;
1697	u32 stats_inst_offset_64;
1698	u64 new_val;
1699	u16 i;
1700
1701	stats_inst_offset_32 = (pestat->gather_info.use_stats_inst) ?
1702	pestat->gather_info.stats_inst_index :
1703	pestat->hw->hmc.hmc_fn_id;
1704	stats_inst_offset_32 *= 4;
1705	stats_inst_offset_64 = stats_inst_offset_32 * 2;
1706
1707	for (i = 0; i < max_stats_idx; i++) {
1708	if (map[i].bitmask <= IRDMA_MAX_STATS_32)
1709	new_val = rd32(hw: dev->hw,
1710	reg: dev->hw_stats_regs[i] + stats_inst_offset_32);
1711	else
1712	new_val = rd64(hw: dev->hw,
1713	reg: dev->hw_stats_regs[i] + stats_inst_offset_64);
1714	gather_stats->val[map[i].byteoff / sizeof(u64)] = new_val;
1715	}
1716
1717	irdma_process_stats(pestat);
1718	}
1719
1720	/**
1721	* irdma_process_cqp_stats - Checking for wrap and update stats
1722	* @cqp_request: cqp_request structure pointer
1723	*/
1724	static void irdma_process_cqp_stats(struct irdma_cqp_request *cqp_request)
1725	{
1726	struct irdma_vsi_pestat *pestat = cqp_request->param;
1727
1728	irdma_process_stats(pestat);
1729	}
1730
1731	/**
1732	* irdma_cqp_gather_stats_cmd - Gather stats
1733	* @dev: pointer to device structure
1734	* @pestat: pointer to stats info
1735	* @wait: flag to wait or not wait for stats
1736	*/
1737	int irdma_cqp_gather_stats_cmd(struct irdma_sc_dev *dev,
1738	struct irdma_vsi_pestat *pestat, bool wait)
1739
1740	{
1741	struct irdma_pci_f *rf = dev_to_rf(dev);
1742	struct irdma_cqp *iwcqp = &rf->cqp;
1743	struct irdma_cqp_request *cqp_request;
1744	struct cqp_cmds_info *cqp_info;
1745	int status;
1746
1747	cqp_request = irdma_alloc_and_get_cqp_request(cqp: iwcqp, wait);
1748	if (!cqp_request)
1749	return -ENOMEM;
1750
1751	cqp_info = &cqp_request->info;
1752	cqp_info->cqp_cmd = IRDMA_OP_STATS_GATHER;
1753	cqp_info->post_sq = 1;
1754	cqp_info->in.u.stats_gather.info = pestat->gather_info;
1755	cqp_info->in.u.stats_gather.scratch = (uintptr_t)cqp_request;
1756	cqp_info->in.u.stats_gather.cqp = &rf->cqp.sc_cqp;
1757	cqp_request->param = pestat;
1758	if (!wait)
1759	cqp_request->callback_fcn = irdma_process_cqp_stats;
1760	status = irdma_handle_cqp_op(rf, cqp_request);
1761	if (wait)
1762	irdma_process_stats(pestat);
1763	irdma_put_cqp_request(cqp: &rf->cqp, cqp_request);
1764
1765	return status;
1766	}
1767
1768	/**
1769	* irdma_cqp_stats_inst_cmd - Allocate/free stats instance
1770	* @vsi: pointer to vsi structure
1771	* @cmd: command to allocate or free
1772	* @stats_info: pointer to allocate stats info
1773	*/
1774	int irdma_cqp_stats_inst_cmd(struct irdma_sc_vsi *vsi, u8 cmd,
1775	struct irdma_stats_inst_info *stats_info)
1776	{
1777	struct irdma_pci_f *rf = dev_to_rf(dev: vsi->dev);
1778	struct irdma_cqp *iwcqp = &rf->cqp;
1779	struct irdma_cqp_request *cqp_request;
1780	struct cqp_cmds_info *cqp_info;
1781	int status;
1782	bool wait = false;
1783
1784	if (cmd == IRDMA_OP_STATS_ALLOCATE)
1785	wait = true;
1786	cqp_request = irdma_alloc_and_get_cqp_request(cqp: iwcqp, wait);
1787	if (!cqp_request)
1788	return -ENOMEM;
1789
1790	cqp_info = &cqp_request->info;
1791	cqp_info->cqp_cmd = cmd;
1792	cqp_info->post_sq = 1;
1793	cqp_info->in.u.stats_manage.info = *stats_info;
1794	cqp_info->in.u.stats_manage.scratch = (uintptr_t)cqp_request;
1795	cqp_info->in.u.stats_manage.cqp = &rf->cqp.sc_cqp;
1796	status = irdma_handle_cqp_op(rf, cqp_request);
1797	if (wait)
1798	stats_info->stats_idx = cqp_request->compl_info.op_ret_val;
1799	irdma_put_cqp_request(cqp: iwcqp, cqp_request);
1800
1801	return status;
1802	}
1803
1804	/**
1805	* irdma_cqp_ceq_cmd - Create/Destroy CEQ's after CEQ 0
1806	* @dev: pointer to device info
1807	* @sc_ceq: pointer to ceq structure
1808	* @op: Create or Destroy
1809	*/
1810	int irdma_cqp_ceq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_ceq *sc_ceq,
1811	u8 op)
1812	{
1813	struct irdma_cqp_request *cqp_request;
1814	struct cqp_cmds_info *cqp_info;
1815	struct irdma_pci_f *rf = dev_to_rf(dev);
1816	int status;
1817
1818	cqp_request = irdma_alloc_and_get_cqp_request(cqp: &rf->cqp, wait: true);
1819	if (!cqp_request)
1820	return -ENOMEM;
1821
1822	cqp_info = &cqp_request->info;
1823	cqp_info->post_sq = 1;
1824	cqp_info->cqp_cmd = op;
1825	cqp_info->in.u.ceq_create.ceq = sc_ceq;
1826	cqp_info->in.u.ceq_create.scratch = (uintptr_t)cqp_request;
1827
1828	status = irdma_handle_cqp_op(rf, cqp_request);
1829	irdma_put_cqp_request(cqp: &rf->cqp, cqp_request);
1830
1831	return status;
1832	}
1833
1834	/**
1835	* irdma_cqp_aeq_cmd - Create/Destroy AEQ
1836	* @dev: pointer to device info
1837	* @sc_aeq: pointer to aeq structure
1838	* @op: Create or Destroy
1839	*/
1840	int irdma_cqp_aeq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_aeq *sc_aeq,
1841	u8 op)
1842	{
1843	struct irdma_cqp_request *cqp_request;
1844	struct cqp_cmds_info *cqp_info;
1845	struct irdma_pci_f *rf = dev_to_rf(dev);
1846	int status;
1847
1848	cqp_request = irdma_alloc_and_get_cqp_request(cqp: &rf->cqp, wait: true);
1849	if (!cqp_request)
1850	return -ENOMEM;
1851
1852	cqp_info = &cqp_request->info;
1853	cqp_info->post_sq = 1;
1854	cqp_info->cqp_cmd = op;
1855	cqp_info->in.u.aeq_create.aeq = sc_aeq;
1856	cqp_info->in.u.aeq_create.scratch = (uintptr_t)cqp_request;
1857
1858	status = irdma_handle_cqp_op(rf, cqp_request);
1859	irdma_put_cqp_request(cqp: &rf->cqp, cqp_request);
1860
1861	return status;
1862	}
1863
1864	/**
1865	* irdma_cqp_ws_node_cmd - Add/modify/delete ws node
1866	* @dev: pointer to device structure
1867	* @cmd: Add, modify or delete
1868	* @node_info: pointer to ws node info
1869	*/
1870	int irdma_cqp_ws_node_cmd(struct irdma_sc_dev *dev, u8 cmd,
1871	struct irdma_ws_node_info *node_info)
1872	{
1873	struct irdma_pci_f *rf = dev_to_rf(dev);
1874	struct irdma_cqp *iwcqp = &rf->cqp;
1875	struct irdma_sc_cqp *cqp = &iwcqp->sc_cqp;
1876	struct irdma_cqp_request *cqp_request;
1877	struct cqp_cmds_info *cqp_info;
1878	int status;
1879	bool poll;
1880
1881	if (!rf->sc_dev.ceq_valid)
1882	poll = true;
1883	else
1884	poll = false;
1885
1886	cqp_request = irdma_alloc_and_get_cqp_request(cqp: iwcqp, wait: !poll);
1887	if (!cqp_request)
1888	return -ENOMEM;
1889
1890	cqp_info = &cqp_request->info;
1891	cqp_info->cqp_cmd = cmd;
1892	cqp_info->post_sq = 1;
1893	cqp_info->in.u.ws_node.info = *node_info;
1894	cqp_info->in.u.ws_node.cqp = cqp;
1895	cqp_info->in.u.ws_node.scratch = (uintptr_t)cqp_request;
1896	status = irdma_handle_cqp_op(rf, cqp_request);
1897	if (status)
1898	goto exit;
1899
1900	if (poll) {
1901	struct irdma_ccq_cqe_info compl_info;
1902
1903	status = irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_WORK_SCHED_NODE,
1904	cmpl_info: &compl_info);
1905	node_info->qs_handle = compl_info.op_ret_val;
1906	ibdev_dbg(&rf->iwdev->ibdev, "DCB: opcode=%d, compl_info.retval=%d\n",
1907	compl_info.op_code, compl_info.op_ret_val);
1908	} else {
1909	node_info->qs_handle = cqp_request->compl_info.op_ret_val;
1910	}
1911
1912	exit:
1913	irdma_put_cqp_request(cqp: &rf->cqp, cqp_request);
1914
1915	return status;
1916	}
1917
1918	/**
1919	* irdma_ah_cqp_op - perform an AH cqp operation
1920	* @rf: RDMA PCI function
1921	* @sc_ah: address handle
1922	* @cmd: AH operation
1923	* @wait: wait if true
1924	* @callback_fcn: Callback function on CQP op completion
1925	* @cb_param: parameter for callback function
1926	*
1927	* returns errno
1928	*/
1929	int irdma_ah_cqp_op(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, u8 cmd,
1930	bool wait,
1931	void (*callback_fcn)(struct irdma_cqp_request *),
1932	void *cb_param)
1933	{
1934	struct irdma_cqp_request *cqp_request;
1935	struct cqp_cmds_info *cqp_info;
1936	int status;
1937
1938	if (cmd != IRDMA_OP_AH_CREATE && cmd != IRDMA_OP_AH_DESTROY)
1939	return -EINVAL;
1940
1941	cqp_request = irdma_alloc_and_get_cqp_request(cqp: &rf->cqp, wait);
1942	if (!cqp_request)
1943	return -ENOMEM;
1944
1945	cqp_info = &cqp_request->info;
1946	cqp_info->cqp_cmd = cmd;
1947	cqp_info->post_sq = 1;
1948	if (cmd == IRDMA_OP_AH_CREATE) {
1949	cqp_info->in.u.ah_create.info = sc_ah->ah_info;
1950	cqp_info->in.u.ah_create.scratch = (uintptr_t)cqp_request;
1951	cqp_info->in.u.ah_create.cqp = &rf->cqp.sc_cqp;
1952	} else if (cmd == IRDMA_OP_AH_DESTROY) {
1953	cqp_info->in.u.ah_destroy.info = sc_ah->ah_info;
1954	cqp_info->in.u.ah_destroy.scratch = (uintptr_t)cqp_request;
1955	cqp_info->in.u.ah_destroy.cqp = &rf->cqp.sc_cqp;
1956	}
1957
1958	if (!wait) {
1959	cqp_request->callback_fcn = callback_fcn;
1960	cqp_request->param = cb_param;
1961	}
1962	status = irdma_handle_cqp_op(rf, cqp_request);
1963	irdma_put_cqp_request(cqp: &rf->cqp, cqp_request);
1964
1965	if (status)
1966	return -ENOMEM;
1967
1968	if (wait)
1969	sc_ah->ah_info.ah_valid = (cmd == IRDMA_OP_AH_CREATE);
1970
1971	return 0;
1972	}
1973
1974	/**
1975	* irdma_ieq_ah_cb - callback after creation of AH for IEQ
1976	* @cqp_request: pointer to cqp_request of create AH
1977	*/
1978	static void irdma_ieq_ah_cb(struct irdma_cqp_request *cqp_request)
1979	{
1980	struct irdma_sc_qp *qp = cqp_request->param;
1981	struct irdma_sc_ah *sc_ah = qp->pfpdu.ah;
1982	unsigned long flags;
1983
1984	spin_lock_irqsave(&qp->pfpdu.lock, flags);
1985	if (!cqp_request->compl_info.op_ret_val) {
1986	sc_ah->ah_info.ah_valid = true;
1987	irdma_ieq_process_fpdus(qp, ieq: qp->vsi->ieq);
1988	} else {
1989	sc_ah->ah_info.ah_valid = false;
1990	irdma_ieq_cleanup_qp(ieq: qp->vsi->ieq, qp);
1991	}
1992	spin_unlock_irqrestore(lock: &qp->pfpdu.lock, flags);
1993	}
1994
1995	/**
1996	* irdma_ilq_ah_cb - callback after creation of AH for ILQ
1997	* @cqp_request: pointer to cqp_request of create AH
1998	*/
1999	static void irdma_ilq_ah_cb(struct irdma_cqp_request *cqp_request)
2000	{
2001	struct irdma_cm_node *cm_node = cqp_request->param;
2002	struct irdma_sc_ah *sc_ah = cm_node->ah;
2003
2004	sc_ah->ah_info.ah_valid = !cqp_request->compl_info.op_ret_val;
2005	irdma_add_conn_est_qh(cm_node);
2006	}
2007
2008	/**
2009	* irdma_puda_create_ah - create AH for ILQ/IEQ qp's
2010	* @dev: device pointer
2011	* @ah_info: Address handle info
2012	* @wait: When true will wait for operation to complete
2013	* @type: ILQ/IEQ
2014	* @cb_param: Callback param when not waiting
2015	* @ah_ret: Returned pointer to address handle if created
2016	*
2017	*/
2018	int irdma_puda_create_ah(struct irdma_sc_dev *dev,
2019	struct irdma_ah_info *ah_info, bool wait,
2020	enum puda_rsrc_type type, void *cb_param,
2021	struct irdma_sc_ah **ah_ret)
2022	{
2023	struct irdma_sc_ah *ah;
2024	struct irdma_pci_f *rf = dev_to_rf(dev);
2025	int err;
2026
2027	ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
2028	*ah_ret = ah;
2029	if (!ah)
2030	return -ENOMEM;
2031
2032	err = irdma_alloc_rsrc(rf, rsrc_array: rf->allocated_ahs, max_rsrc: rf->max_ah,
2033	req_rsrc_num: &ah_info->ah_idx, next: &rf->next_ah);
2034	if (err)
2035	goto err_free;
2036
2037	ah->dev = dev;
2038	ah->ah_info = *ah_info;
2039
2040	if (type == IRDMA_PUDA_RSRC_TYPE_ILQ)
2041	err = irdma_ah_cqp_op(rf, sc_ah: ah, cmd: IRDMA_OP_AH_CREATE, wait,
2042	callback_fcn: irdma_ilq_ah_cb, cb_param);
2043	else
2044	err = irdma_ah_cqp_op(rf, sc_ah: ah, cmd: IRDMA_OP_AH_CREATE, wait,
2045	callback_fcn: irdma_ieq_ah_cb, cb_param);
2046
2047	if (err)
2048	goto error;
2049	return 0;
2050
2051	error:
2052	irdma_free_rsrc(rf, rsrc_array: rf->allocated_ahs, rsrc_num: ah->ah_info.ah_idx);
2053	err_free:
2054	kfree(objp: ah);
2055	*ah_ret = NULL;
2056	return -ENOMEM;
2057	}
2058
2059	/**
2060	* irdma_puda_free_ah - free a puda address handle
2061	* @dev: device pointer
2062	* @ah: The address handle to free
2063	*/
2064	void irdma_puda_free_ah(struct irdma_sc_dev *dev, struct irdma_sc_ah *ah)
2065	{
2066	struct irdma_pci_f *rf = dev_to_rf(dev);
2067
2068	if (!ah)
2069	return;
2070
2071	if (ah->ah_info.ah_valid) {
2072	irdma_ah_cqp_op(rf, sc_ah: ah, cmd: IRDMA_OP_AH_DESTROY, wait: false, NULL, NULL);
2073	irdma_free_rsrc(rf, rsrc_array: rf->allocated_ahs, rsrc_num: ah->ah_info.ah_idx);
2074	}
2075
2076	kfree(objp: ah);
2077	}
2078
2079	/**
2080	* irdma_gsi_ud_qp_ah_cb - callback after creation of AH for GSI/ID QP
2081	* @cqp_request: pointer to cqp_request of create AH
2082	*/
2083	void irdma_gsi_ud_qp_ah_cb(struct irdma_cqp_request *cqp_request)
2084	{
2085	struct irdma_sc_ah *sc_ah = cqp_request->param;
2086
2087	if (!cqp_request->compl_info.op_ret_val)
2088	sc_ah->ah_info.ah_valid = true;
2089	else
2090	sc_ah->ah_info.ah_valid = false;
2091	}
2092
2093	/**
2094	* irdma_prm_add_pble_mem - add moemory to pble resources
2095	* @pprm: pble resource manager
2096	* @pchunk: chunk of memory to add
2097	*/
2098	int irdma_prm_add_pble_mem(struct irdma_pble_prm *pprm,
2099	struct irdma_chunk *pchunk)
2100	{
2101	u64 sizeofbitmap;
2102
2103	if (pchunk->size & 0xfff)
2104	return -EINVAL;
2105
2106	sizeofbitmap = (u64)pchunk->size >> pprm->pble_shift;
2107
2108	pchunk->bitmapbuf = bitmap_zalloc(nbits: sizeofbitmap, GFP_KERNEL);
2109	if (!pchunk->bitmapbuf)
2110	return -ENOMEM;
2111
2112	pchunk->sizeofbitmap = sizeofbitmap;
2113	/* each pble is 8 bytes hence shift by 3 */
2114	pprm->total_pble_alloc += pchunk->size >> 3;
2115	pprm->free_pble_cnt += pchunk->size >> 3;
2116
2117	return 0;
2118	}
2119
2120	/**
2121	* irdma_prm_get_pbles - get pble's from prm
2122	* @pprm: pble resource manager
2123	* @chunkinfo: nformation about chunk where pble's were acquired
2124	* @mem_size: size of pble memory needed
2125	* @vaddr: returns virtual address of pble memory
2126	* @fpm_addr: returns fpm address of pble memory
2127	*/
2128	int irdma_prm_get_pbles(struct irdma_pble_prm *pprm,
2129	struct irdma_pble_chunkinfo *chunkinfo, u64 mem_size,
2130	u64 **vaddr, u64 *fpm_addr)
2131	{
2132	u64 bits_needed;
2133	u64 bit_idx = PBLE_INVALID_IDX;
2134	struct irdma_chunk *pchunk = NULL;
2135	struct list_head *chunk_entry = pprm->clist.next;
2136	u32 offset;
2137	unsigned long flags;
2138	*vaddr = NULL;
2139	*fpm_addr = 0;
2140
2141	bits_needed = DIV_ROUND_UP_ULL(mem_size, BIT_ULL(pprm->pble_shift));
2142
2143	spin_lock_irqsave(&pprm->prm_lock, flags);
2144	while (chunk_entry != &pprm->clist) {
2145	pchunk = (struct irdma_chunk *)chunk_entry;
2146	bit_idx = bitmap_find_next_zero_area(map: pchunk->bitmapbuf,
2147	size: pchunk->sizeofbitmap, start: 0,
2148	nr: bits_needed, align_mask: 0);
2149	if (bit_idx < pchunk->sizeofbitmap)
2150	break;
2151
2152	/* list.next used macro */
2153	chunk_entry = pchunk->list.next;
2154	}
2155
2156	if (!pchunk || bit_idx >= pchunk->sizeofbitmap) {
2157	spin_unlock_irqrestore(lock: &pprm->prm_lock, flags);
2158	return -ENOMEM;
2159	}
2160
2161	bitmap_set(map: pchunk->bitmapbuf, start: bit_idx, nbits: bits_needed);
2162	offset = bit_idx << pprm->pble_shift;
2163	*vaddr = pchunk->vaddr + offset;
2164	*fpm_addr = pchunk->fpm_addr + offset;
2165
2166	chunkinfo->pchunk = pchunk;
2167	chunkinfo->bit_idx = bit_idx;
2168	chunkinfo->bits_used = bits_needed;
2169	/* 3 is sizeof pble divide */
2170	pprm->free_pble_cnt -= chunkinfo->bits_used << (pprm->pble_shift - 3);
2171	spin_unlock_irqrestore(lock: &pprm->prm_lock, flags);
2172
2173	return 0;
2174	}
2175
2176	/**
2177	* irdma_prm_return_pbles - return pbles back to prm
2178	* @pprm: pble resource manager
2179	* @chunkinfo: chunk where pble's were acquired and to be freed
2180	*/
2181	void irdma_prm_return_pbles(struct irdma_pble_prm *pprm,
2182	struct irdma_pble_chunkinfo *chunkinfo)
2183	{
2184	unsigned long flags;
2185
2186	spin_lock_irqsave(&pprm->prm_lock, flags);
2187	pprm->free_pble_cnt += chunkinfo->bits_used << (pprm->pble_shift - 3);
2188	bitmap_clear(map: chunkinfo->pchunk->bitmapbuf, start: chunkinfo->bit_idx,
2189	nbits: chunkinfo->bits_used);
2190	spin_unlock_irqrestore(lock: &pprm->prm_lock, flags);
2191	}
2192
2193	int irdma_map_vm_page_list(struct irdma_hw *hw, void *va, dma_addr_t *pg_dma,
2194	u32 pg_cnt)
2195	{
2196	struct page *vm_page;
2197	int i;
2198	u8 *addr;
2199
2200	addr = (u8 *)(uintptr_t)va;
2201	for (i = 0; i < pg_cnt; i++) {
2202	vm_page = vmalloc_to_page(addr);
2203	if (!vm_page)
2204	goto err;
2205
2206	pg_dma[i] = dma_map_page(hw->device, vm_page, 0, PAGE_SIZE,
2207	DMA_BIDIRECTIONAL);
2208	if (dma_mapping_error(dev: hw->device, dma_addr: pg_dma[i]))
2209	goto err;
2210
2211	addr += PAGE_SIZE;
2212	}
2213
2214	return 0;
2215
2216	err:
2217	irdma_unmap_vm_page_list(hw, pg_dma, pg_cnt: i);
2218	return -ENOMEM;
2219	}
2220
2221	void irdma_unmap_vm_page_list(struct irdma_hw *hw, dma_addr_t *pg_dma, u32 pg_cnt)
2222	{
2223	int i;
2224
2225	for (i = 0; i < pg_cnt; i++)
2226	dma_unmap_page(hw->device, pg_dma[i], PAGE_SIZE, DMA_BIDIRECTIONAL);
2227	}
2228
2229	/**
2230	* irdma_pble_free_paged_mem - free virtual paged memory
2231	* @chunk: chunk to free with paged memory
2232	*/
2233	void irdma_pble_free_paged_mem(struct irdma_chunk *chunk)
2234	{
2235	if (!chunk->pg_cnt)
2236	goto done;
2237
2238	irdma_unmap_vm_page_list(hw: chunk->dev->hw, pg_dma: chunk->dmainfo.dmaaddrs,
2239	pg_cnt: chunk->pg_cnt);
2240
2241	done:
2242	kfree(objp: chunk->dmainfo.dmaaddrs);
2243	chunk->dmainfo.dmaaddrs = NULL;
2244	vfree(addr: chunk->vaddr);
2245	chunk->vaddr = NULL;
2246	chunk->type = 0;
2247	}
2248
2249	/**
2250	* irdma_pble_get_paged_mem -allocate paged memory for pbles
2251	* @chunk: chunk to add for paged memory
2252	* @pg_cnt: number of pages needed
2253	*/
2254	int irdma_pble_get_paged_mem(struct irdma_chunk *chunk, u32 pg_cnt)
2255	{
2256	u32 size;
2257	void *va;
2258
2259	chunk->dmainfo.dmaaddrs = kzalloc(pg_cnt << 3, GFP_KERNEL);
2260	if (!chunk->dmainfo.dmaaddrs)
2261	return -ENOMEM;
2262
2263	size = PAGE_SIZE * pg_cnt;
2264	va = vmalloc(size);
2265	if (!va)
2266	goto err;
2267
2268	if (irdma_map_vm_page_list(hw: chunk->dev->hw, va, pg_dma: chunk->dmainfo.dmaaddrs,
2269	pg_cnt)) {
2270	vfree(addr: va);
2271	goto err;
2272	}
2273	chunk->vaddr = va;
2274	chunk->size = size;
2275	chunk->pg_cnt = pg_cnt;
2276	chunk->type = PBLE_SD_PAGED;
2277
2278	return 0;
2279	err:
2280	kfree(objp: chunk->dmainfo.dmaaddrs);
2281	chunk->dmainfo.dmaaddrs = NULL;
2282
2283	return -ENOMEM;
2284	}
2285
2286	/**
2287	* irdma_alloc_ws_node_id - Allocate a tx scheduler node ID
2288	* @dev: device pointer
2289	*/
2290	u16 irdma_alloc_ws_node_id(struct irdma_sc_dev *dev)
2291	{
2292	struct irdma_pci_f *rf = dev_to_rf(dev);
2293	u32 next = 1;
2294	u32 node_id;
2295
2296	if (irdma_alloc_rsrc(rf, rsrc_array: rf->allocated_ws_nodes, max_rsrc: rf->max_ws_node_id,
2297	req_rsrc_num: &node_id, next: &next))
2298	return IRDMA_WS_NODE_INVALID;
2299
2300	return (u16)node_id;
2301	}
2302
2303	/**
2304	* irdma_free_ws_node_id - Free a tx scheduler node ID
2305	* @dev: device pointer
2306	* @node_id: Work scheduler node ID
2307	*/
2308	void irdma_free_ws_node_id(struct irdma_sc_dev *dev, u16 node_id)
2309	{
2310	struct irdma_pci_f *rf = dev_to_rf(dev);
2311
2312	irdma_free_rsrc(rf, rsrc_array: rf->allocated_ws_nodes, rsrc_num: (u32)node_id);
2313	}
2314
2315	/**
2316	* irdma_modify_qp_to_err - Modify a QP to error
2317	* @sc_qp: qp structure
2318	*/
2319	void irdma_modify_qp_to_err(struct irdma_sc_qp *sc_qp)
2320	{
2321	struct irdma_qp *qp = sc_qp->qp_uk.back_qp;
2322	struct ib_qp_attr attr;
2323
2324	if (qp->iwdev->rf->reset)
2325	return;
2326	attr.qp_state = IB_QPS_ERR;
2327
2328	if (rdma_protocol_roce(device: qp->ibqp.device, port_num: 1))
2329	irdma_modify_qp_roce(ibqp: &qp->ibqp, attr: &attr, attr_mask: IB_QP_STATE, NULL);
2330	else
2331	irdma_modify_qp(ibqp: &qp->ibqp, attr: &attr, attr_mask: IB_QP_STATE, NULL);
2332	}
2333
2334	void irdma_ib_qp_event(struct irdma_qp *iwqp, enum irdma_qp_event_type event)
2335	{
2336	struct ib_event ibevent;
2337
2338	if (!iwqp->ibqp.event_handler)
2339	return;
2340
2341	switch (event) {
2342	case IRDMA_QP_EVENT_CATASTROPHIC:
2343	ibevent.event = IB_EVENT_QP_FATAL;
2344	break;
2345	case IRDMA_QP_EVENT_ACCESS_ERR:
2346	ibevent.event = IB_EVENT_QP_ACCESS_ERR;
2347	break;
2348	case IRDMA_QP_EVENT_REQ_ERR:
2349	ibevent.event = IB_EVENT_QP_REQ_ERR;
2350	break;
2351	}
2352	ibevent.device = iwqp->ibqp.device;
2353	ibevent.element.qp = &iwqp->ibqp;
2354	iwqp->ibqp.event_handler(&ibevent, iwqp->ibqp.qp_context);
2355	}
2356
2357	void irdma_remove_cmpls_list(struct irdma_cq *iwcq)
2358	{
2359	struct irdma_cmpl_gen *cmpl_node;
2360	struct list_head *tmp_node, *list_node;
2361
2362	list_for_each_safe (list_node, tmp_node, &iwcq->cmpl_generated) {
2363	cmpl_node = list_entry(list_node, struct irdma_cmpl_gen, list);
2364	list_del(entry: &cmpl_node->list);
2365	kfree(objp: cmpl_node);
2366	}
2367	}
2368
2369	int irdma_generated_cmpls(struct irdma_cq *iwcq, struct irdma_cq_poll_info *cq_poll_info)
2370	{
2371	struct irdma_cmpl_gen *cmpl;
2372
2373	if (list_empty(head: &iwcq->cmpl_generated))
2374	return -ENOENT;
2375	cmpl = list_first_entry_or_null(&iwcq->cmpl_generated, struct irdma_cmpl_gen, list);
2376	list_del(entry: &cmpl->list);
2377	memcpy(cq_poll_info, &cmpl->cpi, sizeof(*cq_poll_info));
2378	kfree(objp: cmpl);
2379
2380	ibdev_dbg(iwcq->ibcq.device,
2381	"VERBS: %s: Poll artificially generated completion for QP 0x%X, op %u, wr_id=0x%llx\n",
2382	__func__, cq_poll_info->qp_id, cq_poll_info->op_type,
2383	cq_poll_info->wr_id);
2384
2385	return 0;
2386	}
2387
2388	/**
2389	* irdma_set_cpi_common_values - fill in values for polling info struct
2390	* @cpi: resulting structure of cq_poll_info type
2391	* @qp: QPair
2392	* @qp_num: id of the QP
2393	*/
2394	static void irdma_set_cpi_common_values(struct irdma_cq_poll_info *cpi,
2395	struct irdma_qp_uk *qp, u32 qp_num)
2396	{
2397	cpi->comp_status = IRDMA_COMPL_STATUS_FLUSHED;
2398	cpi->error = true;
2399	cpi->major_err = IRDMA_FLUSH_MAJOR_ERR;
2400	cpi->minor_err = FLUSH_GENERAL_ERR;
2401	cpi->qp_handle = (irdma_qp_handle)(uintptr_t)qp;
2402	cpi->qp_id = qp_num;
2403	}
2404
2405	static inline void irdma_comp_handler(struct irdma_cq *cq)
2406	{
2407	if (!cq->ibcq.comp_handler)
2408	return;
2409	if (atomic_cmpxchg(v: &cq->armed, old: 1, new: 0))
2410	cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
2411	}
2412
2413	void irdma_generate_flush_completions(struct irdma_qp *iwqp)
2414	{
2415	struct irdma_qp_uk *qp = &iwqp->sc_qp.qp_uk;
2416	struct irdma_ring *sq_ring = &qp->sq_ring;
2417	struct irdma_ring *rq_ring = &qp->rq_ring;
2418	struct irdma_cq *iwscq = iwqp->iwscq;
2419	struct irdma_cq *iwrcq = iwqp->iwrcq;
2420	struct irdma_cmpl_gen *cmpl;
2421	__le64 *sw_wqe;
2422	u64 wqe_qword;
2423	u32 wqe_idx;
2424	bool compl_generated = false;
2425	unsigned long flags1;
2426
2427	spin_lock_irqsave(&iwscq->lock, flags1);
2428	if (irdma_uk_cq_empty(cq: &iwscq->sc_cq.cq_uk)) {
2429	unsigned long flags2;
2430
2431	spin_lock_irqsave(&iwqp->lock, flags2);
2432	while (IRDMA_RING_MORE_WORK(*sq_ring)) {
2433	cmpl = kzalloc(sizeof(*cmpl), GFP_ATOMIC);
2434	if (!cmpl) {
2435	spin_unlock_irqrestore(lock: &iwqp->lock, flags: flags2);
2436	spin_unlock_irqrestore(lock: &iwscq->lock, flags: flags1);
2437	return;
2438	}
2439
2440	wqe_idx = sq_ring->tail;
2441	irdma_set_cpi_common_values(cpi: &cmpl->cpi, qp, qp_num: qp->qp_id);
2442
2443	cmpl->cpi.wr_id = qp->sq_wrtrk_array[wqe_idx].wrid;
2444	sw_wqe = qp->sq_base[wqe_idx].elem;
2445	get_64bit_val(wqe_words: sw_wqe, byte_index: 24, val: &wqe_qword);
2446	cmpl->cpi.op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, IRDMAQPSQ_OPCODE);
2447	cmpl->cpi.q_type = IRDMA_CQE_QTYPE_SQ;
2448	/* remove the SQ WR by moving SQ tail*/
2449	IRDMA_RING_SET_TAIL(*sq_ring,
2450	sq_ring->tail + qp->sq_wrtrk_array[sq_ring->tail].quanta);
2451	if (cmpl->cpi.op_type == IRDMAQP_OP_NOP) {
2452	kfree(objp: cmpl);
2453	continue;
2454	}
2455	ibdev_dbg(iwscq->ibcq.device,
2456	"DEV: %s: adding wr_id = 0x%llx SQ Completion to list qp_id=%d\n",
2457	__func__, cmpl->cpi.wr_id, qp->qp_id);
2458	list_add_tail(new: &cmpl->list, head: &iwscq->cmpl_generated);
2459	compl_generated = true;
2460	}
2461	spin_unlock_irqrestore(lock: &iwqp->lock, flags: flags2);
2462	spin_unlock_irqrestore(lock: &iwscq->lock, flags: flags1);
2463	if (compl_generated)
2464	irdma_comp_handler(cq: iwscq);
2465	} else {
2466	spin_unlock_irqrestore(lock: &iwscq->lock, flags: flags1);
2467	mod_delayed_work(wq: iwqp->iwdev->cleanup_wq, dwork: &iwqp->dwork_flush,
2468	delay: msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS));
2469	}
2470
2471	spin_lock_irqsave(&iwrcq->lock, flags1);
2472	if (irdma_uk_cq_empty(cq: &iwrcq->sc_cq.cq_uk)) {
2473	unsigned long flags2;
2474
2475	spin_lock_irqsave(&iwqp->lock, flags2);
2476	while (IRDMA_RING_MORE_WORK(*rq_ring)) {
2477	cmpl = kzalloc(sizeof(*cmpl), GFP_ATOMIC);
2478	if (!cmpl) {
2479	spin_unlock_irqrestore(lock: &iwqp->lock, flags: flags2);
2480	spin_unlock_irqrestore(lock: &iwrcq->lock, flags: flags1);
2481	return;
2482	}
2483
2484	wqe_idx = rq_ring->tail;
2485	irdma_set_cpi_common_values(cpi: &cmpl->cpi, qp, qp_num: qp->qp_id);
2486
2487	cmpl->cpi.wr_id = qp->rq_wrid_array[wqe_idx];
2488	cmpl->cpi.op_type = IRDMA_OP_TYPE_REC;
2489	cmpl->cpi.q_type = IRDMA_CQE_QTYPE_RQ;
2490	/* remove the RQ WR by moving RQ tail */
2491	IRDMA_RING_SET_TAIL(*rq_ring, rq_ring->tail + 1);
2492	ibdev_dbg(iwrcq->ibcq.device,
2493	"DEV: %s: adding wr_id = 0x%llx RQ Completion to list qp_id=%d, wqe_idx=%d\n",
2494	__func__, cmpl->cpi.wr_id, qp->qp_id,
2495	wqe_idx);
2496	list_add_tail(new: &cmpl->list, head: &iwrcq->cmpl_generated);
2497
2498	compl_generated = true;
2499	}
2500	spin_unlock_irqrestore(lock: &iwqp->lock, flags: flags2);
2501	spin_unlock_irqrestore(lock: &iwrcq->lock, flags: flags1);
2502	if (compl_generated)
2503	irdma_comp_handler(cq: iwrcq);
2504	} else {
2505	spin_unlock_irqrestore(lock: &iwrcq->lock, flags: flags1);
2506	mod_delayed_work(wq: iwqp->iwdev->cleanup_wq, dwork: &iwqp->dwork_flush,
2507	delay: msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS));
2508	}
2509	}
2510