1	/*
2	* Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved.
3	* Copyright (c) 2004 Infinicon Corporation. All rights reserved.
4	* Copyright (c) 2004 Intel Corporation. All rights reserved.
5	* Copyright (c) 2004 Topspin Corporation. All rights reserved.
6	* Copyright (c) 2004 Voltaire Corporation. All rights reserved.
7	* Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
8	* Copyright (c) 2005, 2006 Cisco Systems. All rights reserved.
9	*
10	* This software is available to you under a choice of one of two
11	* licenses. You may choose to be licensed under the terms of the GNU
12	* General Public License (GPL) Version 2, available from the file
13	* COPYING in the main directory of this source tree, or the
14	* OpenIB.org BSD license below:
15	*
16	* Redistribution and use in source and binary forms, with or
17	* without modification, are permitted provided that the following
18	* conditions are met:
19	*
20	* - Redistributions of source code must retain the above
21	* copyright notice, this list of conditions and the following
22	* disclaimer.
23	*
24	* - Redistributions in binary form must reproduce the above
25	* copyright notice, this list of conditions and the following
26	* disclaimer in the documentation and/or other materials
27	* provided with the distribution.
28	*
29	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
30	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
31	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
32	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
33	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
34	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
35	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
36	* SOFTWARE.
37	*/
38
39	#include <linux/errno.h>
40	#include <linux/err.h>
41	#include <linux/export.h>
42	#include <linux/string.h>
43	#include <linux/slab.h>
44	#include <linux/in.h>
45	#include <linux/in6.h>
46	#include <net/addrconf.h>
47	#include <linux/security.h>
48
49	#include <rdma/ib_verbs.h>
50	#include <rdma/ib_cache.h>
51	#include <rdma/ib_addr.h>
52	#include <rdma/rw.h>
53	#include <rdma/lag.h>
54
55	#include "core_priv.h"
56	#include <trace/events/rdma_core.h>
57
58	static int ib_resolve_eth_dmac(struct ib_device *device,
59	struct rdma_ah_attr *ah_attr);
60
61	static const char * const ib_events[] = {
62	[IB_EVENT_CQ_ERR] = "CQ error",
63	[IB_EVENT_QP_FATAL] = "QP fatal error",
64	[IB_EVENT_QP_REQ_ERR] = "QP request error",
65	[IB_EVENT_QP_ACCESS_ERR] = "QP access error",
66	[IB_EVENT_COMM_EST] = "communication established",
67	[IB_EVENT_SQ_DRAINED] = "send queue drained",
68	[IB_EVENT_PATH_MIG] = "path migration successful",
69	[IB_EVENT_PATH_MIG_ERR] = "path migration error",
70	[IB_EVENT_DEVICE_FATAL] = "device fatal error",
71	[IB_EVENT_PORT_ACTIVE] = "port active",
72	[IB_EVENT_PORT_ERR] = "port error",
73	[IB_EVENT_LID_CHANGE] = "LID change",
74	[IB_EVENT_PKEY_CHANGE] = "P_key change",
75	[IB_EVENT_SM_CHANGE] = "SM change",
76	[IB_EVENT_SRQ_ERR] = "SRQ error",
77	[IB_EVENT_SRQ_LIMIT_REACHED] = "SRQ limit reached",
78	[IB_EVENT_QP_LAST_WQE_REACHED] = "last WQE reached",
79	[IB_EVENT_CLIENT_REREGISTER] = "client reregister",
80	[IB_EVENT_GID_CHANGE] = "GID changed",
81	};
82
83	const char *__attribute_const__ ib_event_msg(enum ib_event_type event)
84	{
85	size_t index = event;
86
87	return (index < ARRAY_SIZE(ib_events) && ib_events[index]) ?
88	ib_events[index] : "unrecognized event";
89	}
90	EXPORT_SYMBOL(ib_event_msg);
91
92	static const char * const wc_statuses[] = {
93	[IB_WC_SUCCESS] = "success",
94	[IB_WC_LOC_LEN_ERR] = "local length error",
95	[IB_WC_LOC_QP_OP_ERR] = "local QP operation error",
96	[IB_WC_LOC_EEC_OP_ERR] = "local EE context operation error",
97	[IB_WC_LOC_PROT_ERR] = "local protection error",
98	[IB_WC_WR_FLUSH_ERR] = "WR flushed",
99	[IB_WC_MW_BIND_ERR] = "memory bind operation error",
100	[IB_WC_BAD_RESP_ERR] = "bad response error",
101	[IB_WC_LOC_ACCESS_ERR] = "local access error",
102	[IB_WC_REM_INV_REQ_ERR] = "remote invalid request error",
103	[IB_WC_REM_ACCESS_ERR] = "remote access error",
104	[IB_WC_REM_OP_ERR] = "remote operation error",
105	[IB_WC_RETRY_EXC_ERR] = "transport retry counter exceeded",
106	[IB_WC_RNR_RETRY_EXC_ERR] = "RNR retry counter exceeded",
107	[IB_WC_LOC_RDD_VIOL_ERR] = "local RDD violation error",
108	[IB_WC_REM_INV_RD_REQ_ERR] = "remote invalid RD request",
109	[IB_WC_REM_ABORT_ERR] = "operation aborted",
110	[IB_WC_INV_EECN_ERR] = "invalid EE context number",
111	[IB_WC_INV_EEC_STATE_ERR] = "invalid EE context state",
112	[IB_WC_FATAL_ERR] = "fatal error",
113	[IB_WC_RESP_TIMEOUT_ERR] = "response timeout error",
114	[IB_WC_GENERAL_ERR] = "general error",
115	};
116
117	const char *__attribute_const__ ib_wc_status_msg(enum ib_wc_status status)
118	{
119	size_t index = status;
120
121	return (index < ARRAY_SIZE(wc_statuses) && wc_statuses[index]) ?
122	wc_statuses[index] : "unrecognized status";
123	}
124	EXPORT_SYMBOL(ib_wc_status_msg);
125
126	__attribute_const__ int ib_rate_to_mult(enum ib_rate rate)
127	{
128	switch (rate) {
129	case IB_RATE_2_5_GBPS: return 1;
130	case IB_RATE_5_GBPS: return 2;
131	case IB_RATE_10_GBPS: return 4;
132	case IB_RATE_20_GBPS: return 8;
133	case IB_RATE_30_GBPS: return 12;
134	case IB_RATE_40_GBPS: return 16;
135	case IB_RATE_60_GBPS: return 24;
136	case IB_RATE_80_GBPS: return 32;
137	case IB_RATE_120_GBPS: return 48;
138	case IB_RATE_14_GBPS: return 6;
139	case IB_RATE_56_GBPS: return 22;
140	case IB_RATE_112_GBPS: return 45;
141	case IB_RATE_168_GBPS: return 67;
142	case IB_RATE_25_GBPS: return 10;
143	case IB_RATE_100_GBPS: return 40;
144	case IB_RATE_200_GBPS: return 80;
145	case IB_RATE_300_GBPS: return 120;
146	case IB_RATE_28_GBPS: return 11;
147	case IB_RATE_50_GBPS: return 20;
148	case IB_RATE_400_GBPS: return 160;
149	case IB_RATE_600_GBPS: return 240;
150	case IB_RATE_800_GBPS: return 320;
151	case IB_RATE_1600_GBPS: return 640;
152	default: return -1;
153	}
154	}
155	EXPORT_SYMBOL(ib_rate_to_mult);
156
157	__attribute_const__ enum ib_rate mult_to_ib_rate(int mult)
158	{
159	switch (mult) {
160	case 1: return IB_RATE_2_5_GBPS;
161	case 2: return IB_RATE_5_GBPS;
162	case 4: return IB_RATE_10_GBPS;
163	case 8: return IB_RATE_20_GBPS;
164	case 12: return IB_RATE_30_GBPS;
165	case 16: return IB_RATE_40_GBPS;
166	case 24: return IB_RATE_60_GBPS;
167	case 32: return IB_RATE_80_GBPS;
168	case 48: return IB_RATE_120_GBPS;
169	case 6: return IB_RATE_14_GBPS;
170	case 22: return IB_RATE_56_GBPS;
171	case 45: return IB_RATE_112_GBPS;
172	case 67: return IB_RATE_168_GBPS;
173	case 10: return IB_RATE_25_GBPS;
174	case 40: return IB_RATE_100_GBPS;
175	case 80: return IB_RATE_200_GBPS;
176	case 120: return IB_RATE_300_GBPS;
177	case 11: return IB_RATE_28_GBPS;
178	case 20: return IB_RATE_50_GBPS;
179	case 160: return IB_RATE_400_GBPS;
180	case 240: return IB_RATE_600_GBPS;
181	case 320: return IB_RATE_800_GBPS;
182	case 640: return IB_RATE_1600_GBPS;
183	default: return IB_RATE_PORT_CURRENT;
184	}
185	}
186	EXPORT_SYMBOL(mult_to_ib_rate);
187
188	__attribute_const__ int ib_rate_to_mbps(enum ib_rate rate)
189	{
190	switch (rate) {
191	case IB_RATE_2_5_GBPS: return 2500;
192	case IB_RATE_5_GBPS: return 5000;
193	case IB_RATE_10_GBPS: return 10000;
194	case IB_RATE_20_GBPS: return 20000;
195	case IB_RATE_30_GBPS: return 30000;
196	case IB_RATE_40_GBPS: return 40000;
197	case IB_RATE_60_GBPS: return 60000;
198	case IB_RATE_80_GBPS: return 80000;
199	case IB_RATE_120_GBPS: return 120000;
200	case IB_RATE_14_GBPS: return 14062;
201	case IB_RATE_56_GBPS: return 56250;
202	case IB_RATE_112_GBPS: return 112500;
203	case IB_RATE_168_GBPS: return 168750;
204	case IB_RATE_25_GBPS: return 25781;
205	case IB_RATE_100_GBPS: return 103125;
206	case IB_RATE_200_GBPS: return 206250;
207	case IB_RATE_300_GBPS: return 309375;
208	case IB_RATE_28_GBPS: return 28125;
209	case IB_RATE_50_GBPS: return 53125;
210	case IB_RATE_400_GBPS: return 425000;
211	case IB_RATE_600_GBPS: return 637500;
212	case IB_RATE_800_GBPS: return 850000;
213	case IB_RATE_1600_GBPS: return 1700000;
214	default: return -1;
215	}
216	}
217	EXPORT_SYMBOL(ib_rate_to_mbps);
218
219	__attribute_const__ enum rdma_transport_type
220	rdma_node_get_transport(unsigned int node_type)
221	{
222
223	if (node_type == RDMA_NODE_USNIC)
224	return RDMA_TRANSPORT_USNIC;
225	if (node_type == RDMA_NODE_USNIC_UDP)
226	return RDMA_TRANSPORT_USNIC_UDP;
227	if (node_type == RDMA_NODE_RNIC)
228	return RDMA_TRANSPORT_IWARP;
229	if (node_type == RDMA_NODE_UNSPECIFIED)
230	return RDMA_TRANSPORT_UNSPECIFIED;
231
232	return RDMA_TRANSPORT_IB;
233	}
234	EXPORT_SYMBOL(rdma_node_get_transport);
235
236	enum rdma_link_layer rdma_port_get_link_layer(struct ib_device *device,
237	u32 port_num)
238	{
239	enum rdma_transport_type lt;
240	if (device->ops.get_link_layer)
241	return device->ops.get_link_layer(device, port_num);
242
243	lt = rdma_node_get_transport(device->node_type);
244	if (lt == RDMA_TRANSPORT_IB)
245	return IB_LINK_LAYER_INFINIBAND;
246
247	return IB_LINK_LAYER_ETHERNET;
248	}
249	EXPORT_SYMBOL(rdma_port_get_link_layer);
250
251	/* Protection domains */
252
253	/**
254	* __ib_alloc_pd - Allocates an unused protection domain.
255	* @device: The device on which to allocate the protection domain.
256	* @flags: protection domain flags
257	* @caller: caller's build-time module name
258	*
259	* A protection domain object provides an association between QPs, shared
260	* receive queues, address handles, memory regions, and memory windows.
261	*
262	* Every PD has a local_dma_lkey which can be used as the lkey value for local
263	* memory operations.
264	*/
265	struct ib_pd *__ib_alloc_pd(struct ib_device *device, unsigned int flags,
266	const char *caller)
267	{
268	struct ib_pd *pd;
269	int mr_access_flags = 0;
270	int ret;
271
272	pd = rdma_zalloc_drv_obj(device, ib_pd);
273	if (!pd)
274	return ERR_PTR(error: -ENOMEM);
275
276	pd->device = device;
277	pd->flags = flags;
278
279	rdma_restrack_new(res: &pd->res, type: RDMA_RESTRACK_PD);
280	rdma_restrack_set_name(res: &pd->res, caller);
281
282	ret = device->ops.alloc_pd(pd, NULL);
283	if (ret) {
284	rdma_restrack_put(res: &pd->res);
285	kfree(objp: pd);
286	return ERR_PTR(error: ret);
287	}
288	rdma_restrack_add(res: &pd->res);
289
290	if (device->attrs.kernel_cap_flags & IBK_LOCAL_DMA_LKEY)
291	pd->local_dma_lkey = device->local_dma_lkey;
292	else
293	mr_access_flags |= IB_ACCESS_LOCAL_WRITE;
294
295	if (flags & IB_PD_UNSAFE_GLOBAL_RKEY) {
296	pr_warn("%s: enabling unsafe global rkey\n", caller);
297	mr_access_flags |= IB_ACCESS_REMOTE_READ | IB_ACCESS_REMOTE_WRITE;
298	}
299
300	if (mr_access_flags) {
301	struct ib_mr *mr;
302
303	mr = pd->device->ops.get_dma_mr(pd, mr_access_flags);
304	if (IS_ERR(ptr: mr)) {
305	ib_dealloc_pd(pd);
306	return ERR_CAST(ptr: mr);
307	}
308
309	mr->device = pd->device;
310	mr->pd = pd;
311	mr->type = IB_MR_TYPE_DMA;
312	mr->uobject = NULL;
313	mr->need_inval = false;
314
315	pd->__internal_mr = mr;
316
317	if (!(device->attrs.kernel_cap_flags & IBK_LOCAL_DMA_LKEY))
318	pd->local_dma_lkey = pd->__internal_mr->lkey;
319
320	if (flags & IB_PD_UNSAFE_GLOBAL_RKEY)
321	pd->unsafe_global_rkey = pd->__internal_mr->rkey;
322	}
323
324	return pd;
325	}
326	EXPORT_SYMBOL(__ib_alloc_pd);
327
328	/**
329	* ib_dealloc_pd_user - Deallocates a protection domain.
330	* @pd: The protection domain to deallocate.
331	* @udata: Valid user data or NULL for kernel object
332	*
333	* It is an error to call this function while any resources in the pd still
334	* exist. The caller is responsible to synchronously destroy them and
335	* guarantee no new allocations will happen.
336	*/
337	int ib_dealloc_pd_user(struct ib_pd *pd, struct ib_udata *udata)
338	{
339	int ret;
340
341	if (pd->__internal_mr) {
342	ret = pd->device->ops.dereg_mr(pd->__internal_mr, NULL);
343	WARN_ON(ret);
344	pd->__internal_mr = NULL;
345	}
346
347	ret = pd->device->ops.dealloc_pd(pd, udata);
348	if (ret)
349	return ret;
350
351	rdma_restrack_del(res: &pd->res);
352	kfree(objp: pd);
353	return ret;
354	}
355	EXPORT_SYMBOL(ib_dealloc_pd_user);
356
357	/* Address handles */
358
359	/**
360	* rdma_copy_ah_attr - Copy rdma ah attribute from source to destination.
361	* @dest: Pointer to destination ah_attr. Contents of the destination
362	* pointer is assumed to be invalid and attribute are overwritten.
363	* @src: Pointer to source ah_attr.
364	*/
365	void rdma_copy_ah_attr(struct rdma_ah_attr *dest,
366	const struct rdma_ah_attr *src)
367	{
368	*dest = *src;
369	if (dest->grh.sgid_attr)
370	rdma_hold_gid_attr(attr: dest->grh.sgid_attr);
371	}
372	EXPORT_SYMBOL(rdma_copy_ah_attr);
373
374	/**
375	* rdma_replace_ah_attr - Replace valid ah_attr with new one.
376	* @old: Pointer to existing ah_attr which needs to be replaced.
377	* old is assumed to be valid or zero'd
378	* @new: Pointer to the new ah_attr.
379	*
380	* rdma_replace_ah_attr() first releases any reference in the old ah_attr if
381	* old the ah_attr is valid; after that it copies the new attribute and holds
382	* the reference to the replaced ah_attr.
383	*/
384	void rdma_replace_ah_attr(struct rdma_ah_attr *old,
385	const struct rdma_ah_attr *new)
386	{
387	rdma_destroy_ah_attr(ah_attr: old);
388	*old = *new;
389	if (old->grh.sgid_attr)
390	rdma_hold_gid_attr(attr: old->grh.sgid_attr);
391	}
392	EXPORT_SYMBOL(rdma_replace_ah_attr);
393
394	/**
395	* rdma_move_ah_attr - Move ah_attr pointed by source to destination.
396	* @dest: Pointer to destination ah_attr to copy to.
397	* dest is assumed to be valid or zero'd
398	* @src: Pointer to the new ah_attr.
399	*
400	* rdma_move_ah_attr() first releases any reference in the destination ah_attr
401	* if it is valid. This also transfers ownership of internal references from
402	* src to dest, making src invalid in the process. No new reference of the src
403	* ah_attr is taken.
404	*/
405	void rdma_move_ah_attr(struct rdma_ah_attr *dest, struct rdma_ah_attr *src)
406	{
407	rdma_destroy_ah_attr(ah_attr: dest);
408	*dest = *src;
409	src->grh.sgid_attr = NULL;
410	}
411	EXPORT_SYMBOL(rdma_move_ah_attr);
412
413	/*
414	* Validate that the rdma_ah_attr is valid for the device before passing it
415	* off to the driver.
416	*/
417	static int rdma_check_ah_attr(struct ib_device *device,
418	struct rdma_ah_attr *ah_attr)
419	{
420	if (!rdma_is_port_valid(device, port: ah_attr->port_num))
421	return -EINVAL;
422
423	if ((rdma_is_grh_required(device, port_num: ah_attr->port_num) ||
424	ah_attr->type == RDMA_AH_ATTR_TYPE_ROCE) &&
425	!(ah_attr->ah_flags & IB_AH_GRH))
426	return -EINVAL;
427
428	if (ah_attr->grh.sgid_attr) {
429	/*
430	* Make sure the passed sgid_attr is consistent with the
431	* parameters
432	*/
433	if (ah_attr->grh.sgid_attr->index != ah_attr->grh.sgid_index ||
434	ah_attr->grh.sgid_attr->port_num != ah_attr->port_num)
435	return -EINVAL;
436	}
437	return 0;
438	}
439
440	/*
441	* If the ah requires a GRH then ensure that sgid_attr pointer is filled in.
442	* On success the caller is responsible to call rdma_unfill_sgid_attr().
443	*/
444	static int rdma_fill_sgid_attr(struct ib_device *device,
445	struct rdma_ah_attr *ah_attr,
446	const struct ib_gid_attr **old_sgid_attr)
447	{
448	const struct ib_gid_attr *sgid_attr;
449	struct ib_global_route *grh;
450	int ret;
451
452	*old_sgid_attr = ah_attr->grh.sgid_attr;
453
454	ret = rdma_check_ah_attr(device, ah_attr);
455	if (ret)
456	return ret;
457
458	if (!(ah_attr->ah_flags & IB_AH_GRH))
459	return 0;
460
461	grh = rdma_ah_retrieve_grh(attr: ah_attr);
462	if (grh->sgid_attr)
463	return 0;
464
465	sgid_attr =
466	rdma_get_gid_attr(device, port_num: ah_attr->port_num, index: grh->sgid_index);
467	if (IS_ERR(ptr: sgid_attr))
468	return PTR_ERR(ptr: sgid_attr);
469
470	/* Move ownerhip of the kref into the ah_attr */
471	grh->sgid_attr = sgid_attr;
472	return 0;
473	}
474
475	static void rdma_unfill_sgid_attr(struct rdma_ah_attr *ah_attr,
476	const struct ib_gid_attr *old_sgid_attr)
477	{
478	/*
479	* Fill didn't change anything, the caller retains ownership of
480	* whatever it passed
481	*/
482	if (ah_attr->grh.sgid_attr == old_sgid_attr)
483	return;
484
485	/*
486	* Otherwise, we need to undo what rdma_fill_sgid_attr so the caller
487	* doesn't see any change in the rdma_ah_attr. If we get here
488	* old_sgid_attr is NULL.
489	*/
490	rdma_destroy_ah_attr(ah_attr);
491	}
492
493	static const struct ib_gid_attr *
494	rdma_update_sgid_attr(struct rdma_ah_attr *ah_attr,
495	const struct ib_gid_attr *old_attr)
496	{
497	if (old_attr)
498	rdma_put_gid_attr(attr: old_attr);
499	if (ah_attr->ah_flags & IB_AH_GRH) {
500	rdma_hold_gid_attr(attr: ah_attr->grh.sgid_attr);
501	return ah_attr->grh.sgid_attr;
502	}
503	return NULL;
504	}
505
506	static struct ib_ah *_rdma_create_ah(struct ib_pd *pd,
507	struct rdma_ah_attr *ah_attr,
508	u32 flags,
509	struct ib_udata *udata,
510	struct net_device *xmit_slave)
511	{
512	struct rdma_ah_init_attr init_attr = {};
513	struct ib_device *device = pd->device;
514	struct ib_ah *ah;
515	int ret;
516
517	might_sleep_if(flags & RDMA_CREATE_AH_SLEEPABLE);
518
519	if (!udata && !device->ops.create_ah)
520	return ERR_PTR(error: -EOPNOTSUPP);
521
522	ah = rdma_zalloc_drv_obj_gfp(
523	device, ib_ah,
524	(flags & RDMA_CREATE_AH_SLEEPABLE) ? GFP_KERNEL : GFP_ATOMIC);
525	if (!ah)
526	return ERR_PTR(error: -ENOMEM);
527
528	ah->device = device;
529	ah->pd = pd;
530	ah->type = ah_attr->type;
531	ah->sgid_attr = rdma_update_sgid_attr(ah_attr, NULL);
532	init_attr.ah_attr = ah_attr;
533	init_attr.flags = flags;
534	init_attr.xmit_slave = xmit_slave;
535
536	if (udata)
537	ret = device->ops.create_user_ah(ah, &init_attr, udata);
538	else
539	ret = device->ops.create_ah(ah, &init_attr, NULL);
540	if (ret) {
541	if (ah->sgid_attr)
542	rdma_put_gid_attr(attr: ah->sgid_attr);
543	kfree(objp: ah);
544	return ERR_PTR(error: ret);
545	}
546
547	atomic_inc(v: &pd->usecnt);
548	return ah;
549	}
550
551	/**
552	* rdma_create_ah - Creates an address handle for the
553	* given address vector.
554	* @pd: The protection domain associated with the address handle.
555	* @ah_attr: The attributes of the address vector.
556	* @flags: Create address handle flags (see enum rdma_create_ah_flags).
557	*
558	* It returns 0 on success and returns appropriate error code on error.
559	* The address handle is used to reference a local or global destination
560	* in all UD QP post sends.
561	*/
562	struct ib_ah *rdma_create_ah(struct ib_pd *pd, struct rdma_ah_attr *ah_attr,
563	u32 flags)
564	{
565	const struct ib_gid_attr *old_sgid_attr;
566	struct net_device *slave;
567	struct ib_ah *ah;
568	int ret;
569
570	ret = rdma_fill_sgid_attr(device: pd->device, ah_attr, old_sgid_attr: &old_sgid_attr);
571	if (ret)
572	return ERR_PTR(error: ret);
573	slave = rdma_lag_get_ah_roce_slave(device: pd->device, ah_attr,
574	flags: (flags & RDMA_CREATE_AH_SLEEPABLE) ?
575	GFP_KERNEL : GFP_ATOMIC);
576	if (IS_ERR(ptr: slave)) {
577	rdma_unfill_sgid_attr(ah_attr, old_sgid_attr);
578	return ERR_CAST(ptr: slave);
579	}
580	ah = _rdma_create_ah(pd, ah_attr, flags, NULL, xmit_slave: slave);
581	rdma_lag_put_ah_roce_slave(xmit_slave: slave);
582	rdma_unfill_sgid_attr(ah_attr, old_sgid_attr);
583	return ah;
584	}
585	EXPORT_SYMBOL(rdma_create_ah);
586
587	/**
588	* rdma_create_user_ah - Creates an address handle for the
589	* given address vector.
590	* It resolves destination mac address for ah attribute of RoCE type.
591	* @pd: The protection domain associated with the address handle.
592	* @ah_attr: The attributes of the address vector.
593	* @udata: pointer to user's input output buffer information need by
594	* provider driver.
595	*
596	* It returns 0 on success and returns appropriate error code on error.
597	* The address handle is used to reference a local or global destination
598	* in all UD QP post sends.
599	*/
600	struct ib_ah *rdma_create_user_ah(struct ib_pd *pd,
601	struct rdma_ah_attr *ah_attr,
602	struct ib_udata *udata)
603	{
604	const struct ib_gid_attr *old_sgid_attr;
605	struct ib_ah *ah;
606	int err;
607
608	err = rdma_fill_sgid_attr(device: pd->device, ah_attr, old_sgid_attr: &old_sgid_attr);
609	if (err)
610	return ERR_PTR(error: err);
611
612	if (ah_attr->type == RDMA_AH_ATTR_TYPE_ROCE) {
613	err = ib_resolve_eth_dmac(device: pd->device, ah_attr);
614	if (err) {
615	ah = ERR_PTR(error: err);
616	goto out;
617	}
618	}
619
620	ah = _rdma_create_ah(pd, ah_attr, flags: RDMA_CREATE_AH_SLEEPABLE,
621	udata, NULL);
622
623	out:
624	rdma_unfill_sgid_attr(ah_attr, old_sgid_attr);
625	return ah;
626	}
627	EXPORT_SYMBOL(rdma_create_user_ah);
628
629	int ib_get_rdma_header_version(const union rdma_network_hdr *hdr)
630	{
631	const struct iphdr *ip4h = (struct iphdr *)&hdr->roce4grh;
632	struct iphdr ip4h_checked;
633	const struct ipv6hdr *ip6h = (struct ipv6hdr *)&hdr->ibgrh;
634
635	/* If it's IPv6, the version must be 6, otherwise, the first
636	* 20 bytes (before the IPv4 header) are garbled.
637	*/
638	if (ip6h->version != 6)
639	return (ip4h->version == 4) ? 4 : 0;
640	/* version may be 6 or 4 because the first 20 bytes could be garbled */
641
642	/* RoCE v2 requires no options, thus header length
643	* must be 5 words
644	*/
645	if (ip4h->ihl != 5)
646	return 6;
647
648	/* Verify checksum.
649	* We can't write on scattered buffers so we need to copy to
650	* temp buffer.
651	*/
652	memcpy(&ip4h_checked, ip4h, sizeof(ip4h_checked));
653	ip4h_checked.check = 0;
654	ip4h_checked.check = ip_fast_csum(iph: (u8 *)&ip4h_checked, ihl: 5);
655	/* if IPv4 header checksum is OK, believe it */
656	if (ip4h->check == ip4h_checked.check)
657	return 4;
658	return 6;
659	}
660	EXPORT_SYMBOL(ib_get_rdma_header_version);
661
662	static enum rdma_network_type ib_get_net_type_by_grh(struct ib_device *device,
663	u32 port_num,
664	const struct ib_grh *grh)
665	{
666	int grh_version;
667
668	if (rdma_protocol_ib(device, port_num))
669	return RDMA_NETWORK_IB;
670
671	grh_version = ib_get_rdma_header_version((union rdma_network_hdr *)grh);
672
673	if (grh_version == 4)
674	return RDMA_NETWORK_IPV4;
675
676	if (grh->next_hdr == IPPROTO_UDP)
677	return RDMA_NETWORK_IPV6;
678
679	return RDMA_NETWORK_ROCE_V1;
680	}
681
682	struct find_gid_index_context {
683	u16 vlan_id;
684	enum ib_gid_type gid_type;
685	};
686
687	static bool find_gid_index(const union ib_gid *gid,
688	const struct ib_gid_attr *gid_attr,
689	void *context)
690	{
691	struct find_gid_index_context *ctx = context;
692	u16 vlan_id = 0xffff;
693	int ret;
694
695	if (ctx->gid_type != gid_attr->gid_type)
696	return false;
697
698	ret = rdma_read_gid_l2_fields(attr: gid_attr, vlan_id: &vlan_id, NULL);
699	if (ret)
700	return false;
701
702	return ctx->vlan_id == vlan_id;
703	}
704
705	static const struct ib_gid_attr *
706	get_sgid_attr_from_eth(struct ib_device *device, u32 port_num,
707	u16 vlan_id, const union ib_gid *sgid,
708	enum ib_gid_type gid_type)
709	{
710	struct find_gid_index_context context = {.vlan_id = vlan_id,
711	.gid_type = gid_type};
712
713	return rdma_find_gid_by_filter(device, gid: sgid, port_num, filter: find_gid_index,
714	context: &context);
715	}
716
717	int ib_get_gids_from_rdma_hdr(const union rdma_network_hdr *hdr,
718	enum rdma_network_type net_type,
719	union ib_gid *sgid, union ib_gid *dgid)
720	{
721	struct sockaddr_in src_in;
722	struct sockaddr_in dst_in;
723	__be32 src_saddr, dst_saddr;
724
725	if (!sgid || !dgid)
726	return -EINVAL;
727
728	if (net_type == RDMA_NETWORK_IPV4) {
729	memcpy(&src_in.sin_addr.s_addr,
730	&hdr->roce4grh.saddr, 4);
731	memcpy(&dst_in.sin_addr.s_addr,
732	&hdr->roce4grh.daddr, 4);
733	src_saddr = src_in.sin_addr.s_addr;
734	dst_saddr = dst_in.sin_addr.s_addr;
735	ipv6_addr_set_v4mapped(addr: src_saddr,
736	v4mapped: (struct in6_addr *)sgid);
737	ipv6_addr_set_v4mapped(addr: dst_saddr,
738	v4mapped: (struct in6_addr *)dgid);
739	return 0;
740	} else if (net_type == RDMA_NETWORK_IPV6 ||
741	net_type == RDMA_NETWORK_IB || net_type == RDMA_NETWORK_ROCE_V1) {
742	*dgid = hdr->ibgrh.dgid;
743	*sgid = hdr->ibgrh.sgid;
744	return 0;
745	} else {
746	return -EINVAL;
747	}
748	}
749	EXPORT_SYMBOL(ib_get_gids_from_rdma_hdr);
750
751	/* Resolve destination mac address and hop limit for unicast destination
752	* GID entry, considering the source GID entry as well.
753	* ah_attribute must have valid port_num, sgid_index.
754	*/
755	static int ib_resolve_unicast_gid_dmac(struct ib_device *device,
756	struct rdma_ah_attr *ah_attr)
757	{
758	struct ib_global_route *grh = rdma_ah_retrieve_grh(attr: ah_attr);
759	const struct ib_gid_attr *sgid_attr = grh->sgid_attr;
760	int hop_limit = 0xff;
761	int ret = 0;
762
763	/* If destination is link local and source GID is RoCEv1,
764	* IP stack is not used.
765	*/
766	if (rdma_link_local_addr(addr: (struct in6_addr *)grh->dgid.raw) &&
767	sgid_attr->gid_type == IB_GID_TYPE_ROCE) {
768	rdma_get_ll_mac(addr: (struct in6_addr *)grh->dgid.raw,
769	mac: ah_attr->roce.dmac);
770	return ret;
771	}
772
773	ret = rdma_addr_find_l2_eth_by_grh(sgid: &sgid_attr->gid, dgid: &grh->dgid,
774	dmac: ah_attr->roce.dmac,
775	sgid_attr, hoplimit: &hop_limit);
776
777	grh->hop_limit = hop_limit;
778	return ret;
779	}
780
781	/*
782	* This function initializes address handle attributes from the incoming packet.
783	* Incoming packet has dgid of the receiver node on which this code is
784	* getting executed and, sgid contains the GID of the sender.
785	*
786	* When resolving mac address of destination, the arrived dgid is used
787	* as sgid and, sgid is used as dgid because sgid contains destinations
788	* GID whom to respond to.
789	*
790	* On success the caller is responsible to call rdma_destroy_ah_attr on the
791	* attr.
792	*/
793	int ib_init_ah_attr_from_wc(struct ib_device *device, u32 port_num,
794	const struct ib_wc *wc, const struct ib_grh *grh,
795	struct rdma_ah_attr *ah_attr)
796	{
797	u32 flow_class;
798	int ret;
799	enum rdma_network_type net_type = RDMA_NETWORK_IB;
800	enum ib_gid_type gid_type = IB_GID_TYPE_IB;
801	const struct ib_gid_attr *sgid_attr;
802	int hoplimit = 0xff;
803	union ib_gid dgid;
804	union ib_gid sgid;
805
806	might_sleep();
807
808	memset(ah_attr, 0, sizeof *ah_attr);
809	ah_attr->type = rdma_ah_find_type(dev: device, port_num);
810	if (rdma_cap_eth_ah(device, port_num)) {
811	if (wc->wc_flags & IB_WC_WITH_NETWORK_HDR_TYPE)
812	net_type = wc->network_hdr_type;
813	else
814	net_type = ib_get_net_type_by_grh(device, port_num, grh);
815	gid_type = ib_network_to_gid_type(network_type: net_type);
816	}
817	ret = ib_get_gids_from_rdma_hdr((union rdma_network_hdr *)grh, net_type,
818	&sgid, &dgid);
819	if (ret)
820	return ret;
821
822	rdma_ah_set_sl(attr: ah_attr, sl: wc->sl);
823	rdma_ah_set_port_num(attr: ah_attr, port_num);
824
825	if (rdma_protocol_roce(device, port_num)) {
826	u16 vlan_id = wc->wc_flags & IB_WC_WITH_VLAN ?
827	wc->vlan_id : 0xffff;
828
829	if (!(wc->wc_flags & IB_WC_GRH))
830	return -EPROTOTYPE;
831
832	sgid_attr = get_sgid_attr_from_eth(device, port_num,
833	vlan_id, sgid: &dgid,
834	gid_type);
835	if (IS_ERR(ptr: sgid_attr))
836	return PTR_ERR(ptr: sgid_attr);
837
838	flow_class = be32_to_cpu(grh->version_tclass_flow);
839	rdma_move_grh_sgid_attr(attr: ah_attr,
840	dgid: &sgid,
841	flow_label: flow_class & 0xFFFFF,
842	hop_limit: hoplimit,
843	traffic_class: (flow_class >> 20) & 0xFF,
844	sgid_attr);
845
846	ret = ib_resolve_unicast_gid_dmac(device, ah_attr);
847	if (ret)
848	rdma_destroy_ah_attr(ah_attr);
849
850	return ret;
851	} else {
852	rdma_ah_set_dlid(attr: ah_attr, dlid: wc->slid);
853	rdma_ah_set_path_bits(attr: ah_attr, src_path_bits: wc->dlid_path_bits);
854
855	if ((wc->wc_flags & IB_WC_GRH) == 0)
856	return 0;
857
858	if (dgid.global.interface_id !=
859	cpu_to_be64(IB_SA_WELL_KNOWN_GUID)) {
860	sgid_attr = rdma_find_gid_by_port(
861	ib_dev: device, gid: &dgid, gid_type: IB_GID_TYPE_IB, port: port_num, NULL);
862	} else
863	sgid_attr = rdma_get_gid_attr(device, port_num, index: 0);
864
865	if (IS_ERR(ptr: sgid_attr))
866	return PTR_ERR(ptr: sgid_attr);
867	flow_class = be32_to_cpu(grh->version_tclass_flow);
868	rdma_move_grh_sgid_attr(attr: ah_attr,
869	dgid: &sgid,
870	flow_label: flow_class & 0xFFFFF,
871	hop_limit: hoplimit,
872	traffic_class: (flow_class >> 20) & 0xFF,
873	sgid_attr);
874
875	return 0;
876	}
877	}
878	EXPORT_SYMBOL(ib_init_ah_attr_from_wc);
879
880	/**
881	* rdma_move_grh_sgid_attr - Sets the sgid attribute of GRH, taking ownership
882	* of the reference
883	*
884	* @attr: Pointer to AH attribute structure
885	* @dgid: Destination GID
886	* @flow_label: Flow label
887	* @hop_limit: Hop limit
888	* @traffic_class: traffic class
889	* @sgid_attr: Pointer to SGID attribute
890	*
891	* This takes ownership of the sgid_attr reference. The caller must ensure
892	* rdma_destroy_ah_attr() is called before destroying the rdma_ah_attr after
893	* calling this function.
894	*/
895	void rdma_move_grh_sgid_attr(struct rdma_ah_attr *attr, union ib_gid *dgid,
896	u32 flow_label, u8 hop_limit, u8 traffic_class,
897	const struct ib_gid_attr *sgid_attr)
898	{
899	rdma_ah_set_grh(attr, dgid, flow_label, sgid_index: sgid_attr->index, hop_limit,
900	traffic_class);
901	attr->grh.sgid_attr = sgid_attr;
902	}
903	EXPORT_SYMBOL(rdma_move_grh_sgid_attr);
904
905	/**
906	* rdma_destroy_ah_attr - Release reference to SGID attribute of
907	* ah attribute.
908	* @ah_attr: Pointer to ah attribute
909	*
910	* Release reference to the SGID attribute of the ah attribute if it is
911	* non NULL. It is safe to call this multiple times, and safe to call it on
912	* a zero initialized ah_attr.
913	*/
914	void rdma_destroy_ah_attr(struct rdma_ah_attr *ah_attr)
915	{
916	if (ah_attr->grh.sgid_attr) {
917	rdma_put_gid_attr(attr: ah_attr->grh.sgid_attr);
918	ah_attr->grh.sgid_attr = NULL;
919	}
920	}
921	EXPORT_SYMBOL(rdma_destroy_ah_attr);
922
923	struct ib_ah *ib_create_ah_from_wc(struct ib_pd *pd, const struct ib_wc *wc,
924	const struct ib_grh *grh, u32 port_num)
925	{
926	struct rdma_ah_attr ah_attr;
927	struct ib_ah *ah;
928	int ret;
929
930	ret = ib_init_ah_attr_from_wc(pd->device, port_num, wc, grh, &ah_attr);
931	if (ret)
932	return ERR_PTR(error: ret);
933
934	ah = rdma_create_ah(pd, &ah_attr, RDMA_CREATE_AH_SLEEPABLE);
935
936	rdma_destroy_ah_attr(&ah_attr);
937	return ah;
938	}
939	EXPORT_SYMBOL(ib_create_ah_from_wc);
940
941	int rdma_modify_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr)
942	{
943	const struct ib_gid_attr *old_sgid_attr;
944	int ret;
945
946	if (ah->type != ah_attr->type)
947	return -EINVAL;
948
949	ret = rdma_fill_sgid_attr(device: ah->device, ah_attr, old_sgid_attr: &old_sgid_attr);
950	if (ret)
951	return ret;
952
953	ret = ah->device->ops.modify_ah ?
954	ah->device->ops.modify_ah(ah, ah_attr) :
955	-EOPNOTSUPP;
956
957	ah->sgid_attr = rdma_update_sgid_attr(ah_attr, old_attr: ah->sgid_attr);
958	rdma_unfill_sgid_attr(ah_attr, old_sgid_attr);
959	return ret;
960	}
961	EXPORT_SYMBOL(rdma_modify_ah);
962
963	int rdma_query_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr)
964	{
965	ah_attr->grh.sgid_attr = NULL;
966
967	return ah->device->ops.query_ah ?
968	ah->device->ops.query_ah(ah, ah_attr) :
969	-EOPNOTSUPP;
970	}
971	EXPORT_SYMBOL(rdma_query_ah);
972
973	int rdma_destroy_ah_user(struct ib_ah *ah, u32 flags, struct ib_udata *udata)
974	{
975	const struct ib_gid_attr *sgid_attr = ah->sgid_attr;
976	struct ib_pd *pd;
977	int ret;
978
979	might_sleep_if(flags & RDMA_DESTROY_AH_SLEEPABLE);
980
981	pd = ah->pd;
982
983	ret = ah->device->ops.destroy_ah(ah, flags);
984	if (ret)
985	return ret;
986
987	atomic_dec(v: &pd->usecnt);
988	if (sgid_attr)
989	rdma_put_gid_attr(attr: sgid_attr);
990
991	kfree(objp: ah);
992	return ret;
993	}
994	EXPORT_SYMBOL(rdma_destroy_ah_user);
995
996	/* Shared receive queues */
997
998	/**
999	* ib_create_srq_user - Creates a SRQ associated with the specified protection
1000	* domain.
1001	* @pd: The protection domain associated with the SRQ.
1002	* @srq_init_attr: A list of initial attributes required to create the
1003	* SRQ. If SRQ creation succeeds, then the attributes are updated to
1004	* the actual capabilities of the created SRQ.
1005	* @uobject: uobject pointer if this is not a kernel SRQ
1006	* @udata: udata pointer if this is not a kernel SRQ
1007	*
1008	* srq_attr->max_wr and srq_attr->max_sge are read the determine the
1009	* requested size of the SRQ, and set to the actual values allocated
1010	* on return. If ib_create_srq() succeeds, then max_wr and max_sge
1011	* will always be at least as large as the requested values.
1012	*/
1013	struct ib_srq *ib_create_srq_user(struct ib_pd *pd,
1014	struct ib_srq_init_attr *srq_init_attr,
1015	struct ib_usrq_object *uobject,
1016	struct ib_udata *udata)
1017	{
1018	struct ib_srq *srq;
1019	int ret;
1020
1021	srq = rdma_zalloc_drv_obj(pd->device, ib_srq);
1022	if (!srq)
1023	return ERR_PTR(error: -ENOMEM);
1024
1025	srq->device = pd->device;
1026	srq->pd = pd;
1027	srq->event_handler = srq_init_attr->event_handler;
1028	srq->srq_context = srq_init_attr->srq_context;
1029	srq->srq_type = srq_init_attr->srq_type;
1030	srq->uobject = uobject;
1031
1032	if (ib_srq_has_cq(srq_type: srq->srq_type)) {
1033	srq->ext.cq = srq_init_attr->ext.cq;
1034	atomic_inc(v: &srq->ext.cq->usecnt);
1035	}
1036	if (srq->srq_type == IB_SRQT_XRC) {
1037	srq->ext.xrc.xrcd = srq_init_attr->ext.xrc.xrcd;
1038	if (srq->ext.xrc.xrcd)
1039	atomic_inc(v: &srq->ext.xrc.xrcd->usecnt);
1040	}
1041	atomic_inc(v: &pd->usecnt);
1042
1043	rdma_restrack_new(res: &srq->res, type: RDMA_RESTRACK_SRQ);
1044	rdma_restrack_parent_name(dst: &srq->res, parent: &pd->res);
1045
1046	ret = pd->device->ops.create_srq(srq, srq_init_attr, udata);
1047	if (ret) {
1048	rdma_restrack_put(res: &srq->res);
1049	atomic_dec(v: &pd->usecnt);
1050	if (srq->srq_type == IB_SRQT_XRC && srq->ext.xrc.xrcd)
1051	atomic_dec(v: &srq->ext.xrc.xrcd->usecnt);
1052	if (ib_srq_has_cq(srq_type: srq->srq_type))
1053	atomic_dec(v: &srq->ext.cq->usecnt);
1054	kfree(objp: srq);
1055	return ERR_PTR(error: ret);
1056	}
1057
1058	rdma_restrack_add(res: &srq->res);
1059
1060	return srq;
1061	}
1062	EXPORT_SYMBOL(ib_create_srq_user);
1063
1064	int ib_modify_srq(struct ib_srq *srq,
1065	struct ib_srq_attr *srq_attr,
1066	enum ib_srq_attr_mask srq_attr_mask)
1067	{
1068	return srq->device->ops.modify_srq ?
1069	srq->device->ops.modify_srq(srq, srq_attr, srq_attr_mask,
1070	NULL) : -EOPNOTSUPP;
1071	}
1072	EXPORT_SYMBOL(ib_modify_srq);
1073
1074	int ib_query_srq(struct ib_srq *srq,
1075	struct ib_srq_attr *srq_attr)
1076	{
1077	return srq->device->ops.query_srq ?
1078	srq->device->ops.query_srq(srq, srq_attr) : -EOPNOTSUPP;
1079	}
1080	EXPORT_SYMBOL(ib_query_srq);
1081
1082	int ib_destroy_srq_user(struct ib_srq *srq, struct ib_udata *udata)
1083	{
1084	int ret;
1085
1086	if (atomic_read(v: &srq->usecnt))
1087	return -EBUSY;
1088
1089	ret = srq->device->ops.destroy_srq(srq, udata);
1090	if (ret)
1091	return ret;
1092
1093	atomic_dec(v: &srq->pd->usecnt);
1094	if (srq->srq_type == IB_SRQT_XRC && srq->ext.xrc.xrcd)
1095	atomic_dec(v: &srq->ext.xrc.xrcd->usecnt);
1096	if (ib_srq_has_cq(srq_type: srq->srq_type))
1097	atomic_dec(v: &srq->ext.cq->usecnt);
1098	rdma_restrack_del(res: &srq->res);
1099	kfree(objp: srq);
1100
1101	return ret;
1102	}
1103	EXPORT_SYMBOL(ib_destroy_srq_user);
1104
1105	/* Queue pairs */
1106
1107	static void __ib_qp_event_handler(struct ib_event *event, void *context)
1108	{
1109	struct ib_qp *qp = event->element.qp;
1110
1111	if (event->event == IB_EVENT_QP_LAST_WQE_REACHED)
1112	complete(&qp->srq_completion);
1113	if (qp->registered_event_handler)
1114	qp->registered_event_handler(event, qp->qp_context);
1115	}
1116
1117	static void __ib_shared_qp_event_handler(struct ib_event *event, void *context)
1118	{
1119	struct ib_qp *qp = context;
1120	unsigned long flags;
1121
1122	spin_lock_irqsave(&qp->device->qp_open_list_lock, flags);
1123	list_for_each_entry(event->element.qp, &qp->open_list, open_list)
1124	if (event->element.qp->event_handler)
1125	event->element.qp->event_handler(event, event->element.qp->qp_context);
1126	spin_unlock_irqrestore(lock: &qp->device->qp_open_list_lock, flags);
1127	}
1128
1129	static struct ib_qp *__ib_open_qp(struct ib_qp *real_qp,
1130	void (*event_handler)(struct ib_event *, void *),
1131	void *qp_context)
1132	{
1133	struct ib_qp *qp;
1134	unsigned long flags;
1135	int err;
1136
1137	qp = kzalloc(sizeof *qp, GFP_KERNEL);
1138	if (!qp)
1139	return ERR_PTR(error: -ENOMEM);
1140
1141	qp->real_qp = real_qp;
1142	err = ib_open_shared_qp_security(qp, dev: real_qp->device);
1143	if (err) {
1144	kfree(objp: qp);
1145	return ERR_PTR(error: err);
1146	}
1147
1148	qp->real_qp = real_qp;
1149	atomic_inc(v: &real_qp->usecnt);
1150	qp->device = real_qp->device;
1151	qp->event_handler = event_handler;
1152	qp->qp_context = qp_context;
1153	qp->qp_num = real_qp->qp_num;
1154	qp->qp_type = real_qp->qp_type;
1155
1156	spin_lock_irqsave(&real_qp->device->qp_open_list_lock, flags);
1157	list_add(new: &qp->open_list, head: &real_qp->open_list);
1158	spin_unlock_irqrestore(lock: &real_qp->device->qp_open_list_lock, flags);
1159
1160	return qp;
1161	}
1162
1163	struct ib_qp *ib_open_qp(struct ib_xrcd *xrcd,
1164	struct ib_qp_open_attr *qp_open_attr)
1165	{
1166	struct ib_qp *qp, *real_qp;
1167
1168	if (qp_open_attr->qp_type != IB_QPT_XRC_TGT)
1169	return ERR_PTR(error: -EINVAL);
1170
1171	down_read(sem: &xrcd->tgt_qps_rwsem);
1172	real_qp = xa_load(&xrcd->tgt_qps, index: qp_open_attr->qp_num);
1173	if (!real_qp) {
1174	up_read(sem: &xrcd->tgt_qps_rwsem);
1175	return ERR_PTR(error: -EINVAL);
1176	}
1177	qp = __ib_open_qp(real_qp, event_handler: qp_open_attr->event_handler,
1178	qp_context: qp_open_attr->qp_context);
1179	up_read(sem: &xrcd->tgt_qps_rwsem);
1180	return qp;
1181	}
1182	EXPORT_SYMBOL(ib_open_qp);
1183
1184	static struct ib_qp *create_xrc_qp_user(struct ib_qp *qp,
1185	struct ib_qp_init_attr *qp_init_attr)
1186	{
1187	struct ib_qp *real_qp = qp;
1188	int err;
1189
1190	qp->event_handler = __ib_shared_qp_event_handler;
1191	qp->qp_context = qp;
1192	qp->pd = NULL;
1193	qp->send_cq = qp->recv_cq = NULL;
1194	qp->srq = NULL;
1195	qp->xrcd = qp_init_attr->xrcd;
1196	atomic_inc(v: &qp_init_attr->xrcd->usecnt);
1197	INIT_LIST_HEAD(list: &qp->open_list);
1198
1199	qp = __ib_open_qp(real_qp, event_handler: qp_init_attr->event_handler,
1200	qp_context: qp_init_attr->qp_context);
1201	if (IS_ERR(ptr: qp))
1202	return qp;
1203
1204	err = xa_err(entry: xa_store(&qp_init_attr->xrcd->tgt_qps, index: real_qp->qp_num,
1205	entry: real_qp, GFP_KERNEL));
1206	if (err) {
1207	ib_close_qp(qp);
1208	return ERR_PTR(error: err);
1209	}
1210	return qp;
1211	}
1212
1213	static struct ib_qp *create_qp(struct ib_device *dev, struct ib_pd *pd,
1214	struct ib_qp_init_attr *attr,
1215	struct ib_udata *udata,
1216	struct ib_uqp_object *uobj, const char *caller)
1217	{
1218	struct ib_udata dummy = {};
1219	struct ib_qp *qp;
1220	int ret;
1221
1222	if (!dev->ops.create_qp)
1223	return ERR_PTR(error: -EOPNOTSUPP);
1224
1225	qp = rdma_zalloc_drv_obj_numa(dev, ib_qp);
1226	if (!qp)
1227	return ERR_PTR(error: -ENOMEM);
1228
1229	qp->device = dev;
1230	qp->pd = pd;
1231	qp->uobject = uobj;
1232	qp->real_qp = qp;
1233
1234	qp->qp_type = attr->qp_type;
1235	qp->rwq_ind_tbl = attr->rwq_ind_tbl;
1236	qp->srq = attr->srq;
1237	qp->event_handler = __ib_qp_event_handler;
1238	qp->registered_event_handler = attr->event_handler;
1239	qp->port = attr->port_num;
1240	qp->qp_context = attr->qp_context;
1241
1242	spin_lock_init(&qp->mr_lock);
1243	INIT_LIST_HEAD(list: &qp->rdma_mrs);
1244	INIT_LIST_HEAD(list: &qp->sig_mrs);
1245	init_completion(x: &qp->srq_completion);
1246
1247	qp->send_cq = attr->send_cq;
1248	qp->recv_cq = attr->recv_cq;
1249
1250	rdma_restrack_new(res: &qp->res, type: RDMA_RESTRACK_QP);
1251	WARN_ONCE(!udata && !caller, "Missing kernel QP owner");
1252	rdma_restrack_set_name(res: &qp->res, caller: udata ? NULL : caller);
1253	ret = dev->ops.create_qp(qp, attr, udata);
1254	if (ret)
1255	goto err_create;
1256
1257	/*
1258	* TODO: The mlx4 internally overwrites send_cq and recv_cq.
1259	* Unfortunately, it is not an easy task to fix that driver.
1260	*/
1261	qp->send_cq = attr->send_cq;
1262	qp->recv_cq = attr->recv_cq;
1263
1264	ret = ib_create_qp_security(qp, dev);
1265	if (ret)
1266	goto err_security;
1267
1268	rdma_restrack_add(res: &qp->res);
1269	return qp;
1270
1271	err_security:
1272	qp->device->ops.destroy_qp(qp, udata ? &dummy : NULL);
1273	err_create:
1274	rdma_restrack_put(res: &qp->res);
1275	kfree(objp: qp);
1276	return ERR_PTR(error: ret);
1277
1278	}
1279
1280	/**
1281	* ib_create_qp_user - Creates a QP associated with the specified protection
1282	* domain.
1283	* @dev: IB device
1284	* @pd: The protection domain associated with the QP.
1285	* @attr: A list of initial attributes required to create the
1286	* QP. If QP creation succeeds, then the attributes are updated to
1287	* the actual capabilities of the created QP.
1288	* @udata: User data
1289	* @uobj: uverbs obect
1290	* @caller: caller's build-time module name
1291	*/
1292	struct ib_qp *ib_create_qp_user(struct ib_device *dev, struct ib_pd *pd,
1293	struct ib_qp_init_attr *attr,
1294	struct ib_udata *udata,
1295	struct ib_uqp_object *uobj, const char *caller)
1296	{
1297	struct ib_qp *qp, *xrc_qp;
1298
1299	if (attr->qp_type == IB_QPT_XRC_TGT)
1300	qp = create_qp(dev, pd, attr, NULL, NULL, caller);
1301	else
1302	qp = create_qp(dev, pd, attr, udata, uobj, NULL);
1303	if (attr->qp_type != IB_QPT_XRC_TGT || IS_ERR(ptr: qp))
1304	return qp;
1305
1306	xrc_qp = create_xrc_qp_user(qp, qp_init_attr: attr);
1307	if (IS_ERR(ptr: xrc_qp)) {
1308	ib_destroy_qp(qp);
1309	return xrc_qp;
1310	}
1311
1312	xrc_qp->uobject = uobj;
1313	return xrc_qp;
1314	}
1315	EXPORT_SYMBOL(ib_create_qp_user);
1316
1317	void ib_qp_usecnt_inc(struct ib_qp *qp)
1318	{
1319	if (qp->pd)
1320	atomic_inc(v: &qp->pd->usecnt);
1321	if (qp->send_cq)
1322	atomic_inc(v: &qp->send_cq->usecnt);
1323	if (qp->recv_cq)
1324	atomic_inc(v: &qp->recv_cq->usecnt);
1325	if (qp->srq)
1326	atomic_inc(v: &qp->srq->usecnt);
1327	if (qp->rwq_ind_tbl)
1328	atomic_inc(v: &qp->rwq_ind_tbl->usecnt);
1329	}
1330	EXPORT_SYMBOL(ib_qp_usecnt_inc);
1331
1332	void ib_qp_usecnt_dec(struct ib_qp *qp)
1333	{
1334	if (qp->rwq_ind_tbl)
1335	atomic_dec(v: &qp->rwq_ind_tbl->usecnt);
1336	if (qp->srq)
1337	atomic_dec(v: &qp->srq->usecnt);
1338	if (qp->recv_cq)
1339	atomic_dec(v: &qp->recv_cq->usecnt);
1340	if (qp->send_cq)
1341	atomic_dec(v: &qp->send_cq->usecnt);
1342	if (qp->pd)
1343	atomic_dec(v: &qp->pd->usecnt);
1344	}
1345	EXPORT_SYMBOL(ib_qp_usecnt_dec);
1346
1347	struct ib_qp *ib_create_qp_kernel(struct ib_pd *pd,
1348	struct ib_qp_init_attr *qp_init_attr,
1349	const char *caller)
1350	{
1351	struct ib_device *device = pd->device;
1352	struct ib_qp *qp;
1353	int ret;
1354
1355	/*
1356	* If the callers is using the RDMA API calculate the resources
1357	* needed for the RDMA READ/WRITE operations.
1358	*
1359	* Note that these callers need to pass in a port number.
1360	*/
1361	if (qp_init_attr->cap.max_rdma_ctxs)
1362	rdma_rw_init_qp(dev: device, attr: qp_init_attr);
1363
1364	qp = create_qp(dev: device, pd, attr: qp_init_attr, NULL, NULL, caller);
1365	if (IS_ERR(ptr: qp))
1366	return qp;
1367
1368	ib_qp_usecnt_inc(qp);
1369
1370	if (qp_init_attr->cap.max_rdma_ctxs) {
1371	ret = rdma_rw_init_mrs(qp, attr: qp_init_attr);
1372	if (ret)
1373	goto err;
1374	}
1375
1376	/*
1377	* Note: all hw drivers guarantee that max_send_sge is lower than
1378	* the device RDMA WRITE SGE limit but not all hw drivers ensure that
1379	* max_send_sge <= max_sge_rd.
1380	*/
1381	qp->max_write_sge = qp_init_attr->cap.max_send_sge;
1382	qp->max_read_sge = min_t(u32, qp_init_attr->cap.max_send_sge,
1383	device->attrs.max_sge_rd);
1384	if (qp_init_attr->create_flags & IB_QP_CREATE_INTEGRITY_EN)
1385	qp->integrity_en = true;
1386
1387	return qp;
1388
1389	err:
1390	ib_destroy_qp(qp);
1391	return ERR_PTR(error: ret);
1392
1393	}
1394	EXPORT_SYMBOL(ib_create_qp_kernel);
1395
1396	static const struct {
1397	int valid;
1398	enum ib_qp_attr_mask req_param[IB_QPT_MAX];
1399	enum ib_qp_attr_mask opt_param[IB_QPT_MAX];
1400	} qp_state_table[IB_QPS_ERR + 1][IB_QPS_ERR + 1] = {
1401	[IB_QPS_RESET] = {
1402	[IB_QPS_RESET] = { .valid = 1 },
1403	[IB_QPS_INIT] = {
1404	.valid = 1,
1405	.req_param = {
1406	[IB_QPT_UD] = (IB_QP_PKEY_INDEX |
1407	IB_QP_PORT |
1408	IB_QP_QKEY),
1409	[IB_QPT_RAW_PACKET] = IB_QP_PORT,
1410	[IB_QPT_UC] = (IB_QP_PKEY_INDEX |
1411	IB_QP_PORT |
1412	IB_QP_ACCESS_FLAGS),
1413	[IB_QPT_RC] = (IB_QP_PKEY_INDEX |
1414	IB_QP_PORT |
1415	IB_QP_ACCESS_FLAGS),
1416	[IB_QPT_XRC_INI] = (IB_QP_PKEY_INDEX |
1417	IB_QP_PORT |
1418	IB_QP_ACCESS_FLAGS),
1419	[IB_QPT_XRC_TGT] = (IB_QP_PKEY_INDEX |
1420	IB_QP_PORT |
1421	IB_QP_ACCESS_FLAGS),
1422	[IB_QPT_SMI] = (IB_QP_PKEY_INDEX |
1423	IB_QP_QKEY),
1424	[IB_QPT_GSI] = (IB_QP_PKEY_INDEX |
1425	IB_QP_QKEY),
1426	}
1427	},
1428	},
1429	[IB_QPS_INIT] = {
1430	[IB_QPS_RESET] = { .valid = 1 },
1431	[IB_QPS_ERR] = { .valid = 1 },
1432	[IB_QPS_INIT] = {
1433	.valid = 1,
1434	.opt_param = {
1435	[IB_QPT_UD] = (IB_QP_PKEY_INDEX |
1436	IB_QP_PORT |
1437	IB_QP_QKEY),
1438	[IB_QPT_UC] = (IB_QP_PKEY_INDEX |
1439	IB_QP_PORT |
1440	IB_QP_ACCESS_FLAGS),
1441	[IB_QPT_RC] = (IB_QP_PKEY_INDEX |
1442	IB_QP_PORT |
1443	IB_QP_ACCESS_FLAGS),
1444	[IB_QPT_XRC_INI] = (IB_QP_PKEY_INDEX |
1445	IB_QP_PORT |
1446	IB_QP_ACCESS_FLAGS),
1447	[IB_QPT_XRC_TGT] = (IB_QP_PKEY_INDEX |
1448	IB_QP_PORT |
1449	IB_QP_ACCESS_FLAGS),
1450	[IB_QPT_SMI] = (IB_QP_PKEY_INDEX |
1451	IB_QP_QKEY),
1452	[IB_QPT_GSI] = (IB_QP_PKEY_INDEX |
1453	IB_QP_QKEY),
1454	}
1455	},
1456	[IB_QPS_RTR] = {
1457	.valid = 1,
1458	.req_param = {
1459	[IB_QPT_UC] = (IB_QP_AV |
1460	IB_QP_PATH_MTU |
1461	IB_QP_DEST_QPN |
1462	IB_QP_RQ_PSN),
1463	[IB_QPT_RC] = (IB_QP_AV |
1464	IB_QP_PATH_MTU |
1465	IB_QP_DEST_QPN |
1466	IB_QP_RQ_PSN |
1467	IB_QP_MAX_DEST_RD_ATOMIC |
1468	IB_QP_MIN_RNR_TIMER),
1469	[IB_QPT_XRC_INI] = (IB_QP_AV |
1470	IB_QP_PATH_MTU |
1471	IB_QP_DEST_QPN |
1472	IB_QP_RQ_PSN),
1473	[IB_QPT_XRC_TGT] = (IB_QP_AV |
1474	IB_QP_PATH_MTU |
1475	IB_QP_DEST_QPN |
1476	IB_QP_RQ_PSN |
1477	IB_QP_MAX_DEST_RD_ATOMIC |
1478	IB_QP_MIN_RNR_TIMER),
1479	},
1480	.opt_param = {
1481	[IB_QPT_UD] = (IB_QP_PKEY_INDEX |
1482	IB_QP_QKEY),
1483	[IB_QPT_UC] = (IB_QP_ALT_PATH |
1484	IB_QP_ACCESS_FLAGS |
1485	IB_QP_PKEY_INDEX),
1486	[IB_QPT_RC] = (IB_QP_ALT_PATH |
1487	IB_QP_ACCESS_FLAGS |
1488	IB_QP_PKEY_INDEX),
1489	[IB_QPT_XRC_INI] = (IB_QP_ALT_PATH |
1490	IB_QP_ACCESS_FLAGS |
1491	IB_QP_PKEY_INDEX),
1492	[IB_QPT_XRC_TGT] = (IB_QP_ALT_PATH |
1493	IB_QP_ACCESS_FLAGS |
1494	IB_QP_PKEY_INDEX),
1495	[IB_QPT_SMI] = (IB_QP_PKEY_INDEX |
1496	IB_QP_QKEY),
1497	[IB_QPT_GSI] = (IB_QP_PKEY_INDEX |
1498	IB_QP_QKEY),
1499	},
1500	},
1501	},
1502	[IB_QPS_RTR] = {
1503	[IB_QPS_RESET] = { .valid = 1 },
1504	[IB_QPS_ERR] = { .valid = 1 },
1505	[IB_QPS_RTS] = {
1506	.valid = 1,
1507	.req_param = {
1508	[IB_QPT_UD] = IB_QP_SQ_PSN,
1509	[IB_QPT_UC] = IB_QP_SQ_PSN,
1510	[IB_QPT_RC] = (IB_QP_TIMEOUT |
1511	IB_QP_RETRY_CNT |
1512	IB_QP_RNR_RETRY |
1513	IB_QP_SQ_PSN |
1514	IB_QP_MAX_QP_RD_ATOMIC),
1515	[IB_QPT_XRC_INI] = (IB_QP_TIMEOUT |
1516	IB_QP_RETRY_CNT |
1517	IB_QP_RNR_RETRY |
1518	IB_QP_SQ_PSN |
1519	IB_QP_MAX_QP_RD_ATOMIC),
1520	[IB_QPT_XRC_TGT] = (IB_QP_TIMEOUT |
1521	IB_QP_SQ_PSN),
1522	[IB_QPT_SMI] = IB_QP_SQ_PSN,
1523	[IB_QPT_GSI] = IB_QP_SQ_PSN,
1524	},
1525	.opt_param = {
1526	[IB_QPT_UD] = (IB_QP_CUR_STATE |
1527	IB_QP_QKEY),
1528	[IB_QPT_UC] = (IB_QP_CUR_STATE |
1529	IB_QP_ALT_PATH |
1530	IB_QP_ACCESS_FLAGS |
1531	IB_QP_PATH_MIG_STATE),
1532	[IB_QPT_RC] = (IB_QP_CUR_STATE |
1533	IB_QP_ALT_PATH |
1534	IB_QP_ACCESS_FLAGS |
1535	IB_QP_MIN_RNR_TIMER |
1536	IB_QP_PATH_MIG_STATE),
1537	[IB_QPT_XRC_INI] = (IB_QP_CUR_STATE |
1538	IB_QP_ALT_PATH |
1539	IB_QP_ACCESS_FLAGS |
1540	IB_QP_PATH_MIG_STATE),
1541	[IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE |
1542	IB_QP_ALT_PATH |
1543	IB_QP_ACCESS_FLAGS |
1544	IB_QP_MIN_RNR_TIMER |
1545	IB_QP_PATH_MIG_STATE),
1546	[IB_QPT_SMI] = (IB_QP_CUR_STATE |
1547	IB_QP_QKEY),
1548	[IB_QPT_GSI] = (IB_QP_CUR_STATE |
1549	IB_QP_QKEY),
1550	[IB_QPT_RAW_PACKET] = IB_QP_RATE_LIMIT,
1551	}
1552	}
1553	},
1554	[IB_QPS_RTS] = {
1555	[IB_QPS_RESET] = { .valid = 1 },
1556	[IB_QPS_ERR] = { .valid = 1 },
1557	[IB_QPS_RTS] = {
1558	.valid = 1,
1559	.opt_param = {
1560	[IB_QPT_UD] = (IB_QP_CUR_STATE |
1561	IB_QP_QKEY),
1562	[IB_QPT_UC] = (IB_QP_CUR_STATE |
1563	IB_QP_ACCESS_FLAGS |
1564	IB_QP_ALT_PATH |
1565	IB_QP_PATH_MIG_STATE),
1566	[IB_QPT_RC] = (IB_QP_CUR_STATE |
1567	IB_QP_ACCESS_FLAGS |
1568	IB_QP_ALT_PATH |
1569	IB_QP_PATH_MIG_STATE |
1570	IB_QP_MIN_RNR_TIMER),
1571	[IB_QPT_XRC_INI] = (IB_QP_CUR_STATE |
1572	IB_QP_ACCESS_FLAGS |
1573	IB_QP_ALT_PATH |
1574	IB_QP_PATH_MIG_STATE),
1575	[IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE |
1576	IB_QP_ACCESS_FLAGS |
1577	IB_QP_ALT_PATH |
1578	IB_QP_PATH_MIG_STATE |
1579	IB_QP_MIN_RNR_TIMER),
1580	[IB_QPT_SMI] = (IB_QP_CUR_STATE |
1581	IB_QP_QKEY),
1582	[IB_QPT_GSI] = (IB_QP_CUR_STATE |
1583	IB_QP_QKEY),
1584	[IB_QPT_RAW_PACKET] = IB_QP_RATE_LIMIT,
1585	}
1586	},
1587	[IB_QPS_SQD] = {
1588	.valid = 1,
1589	.opt_param = {
1590	[IB_QPT_UD] = IB_QP_EN_SQD_ASYNC_NOTIFY,
1591	[IB_QPT_UC] = IB_QP_EN_SQD_ASYNC_NOTIFY,
1592	[IB_QPT_RC] = IB_QP_EN_SQD_ASYNC_NOTIFY,
1593	[IB_QPT_XRC_INI] = IB_QP_EN_SQD_ASYNC_NOTIFY,
1594	[IB_QPT_XRC_TGT] = IB_QP_EN_SQD_ASYNC_NOTIFY, /* ??? */
1595	[IB_QPT_SMI] = IB_QP_EN_SQD_ASYNC_NOTIFY,
1596	[IB_QPT_GSI] = IB_QP_EN_SQD_ASYNC_NOTIFY
1597	}
1598	},
1599	},
1600	[IB_QPS_SQD] = {
1601	[IB_QPS_RESET] = { .valid = 1 },
1602	[IB_QPS_ERR] = { .valid = 1 },
1603	[IB_QPS_RTS] = {
1604	.valid = 1,
1605	.opt_param = {
1606	[IB_QPT_UD] = (IB_QP_CUR_STATE |
1607	IB_QP_QKEY),
1608	[IB_QPT_UC] = (IB_QP_CUR_STATE |
1609	IB_QP_ALT_PATH |
1610	IB_QP_ACCESS_FLAGS |
1611	IB_QP_PATH_MIG_STATE),
1612	[IB_QPT_RC] = (IB_QP_CUR_STATE |
1613	IB_QP_ALT_PATH |
1614	IB_QP_ACCESS_FLAGS |
1615	IB_QP_MIN_RNR_TIMER |
1616	IB_QP_PATH_MIG_STATE),
1617	[IB_QPT_XRC_INI] = (IB_QP_CUR_STATE |
1618	IB_QP_ALT_PATH |
1619	IB_QP_ACCESS_FLAGS |
1620	IB_QP_PATH_MIG_STATE),
1621	[IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE |
1622	IB_QP_ALT_PATH |
1623	IB_QP_ACCESS_FLAGS |
1624	IB_QP_MIN_RNR_TIMER |
1625	IB_QP_PATH_MIG_STATE),
1626	[IB_QPT_SMI] = (IB_QP_CUR_STATE |
1627	IB_QP_QKEY),
1628	[IB_QPT_GSI] = (IB_QP_CUR_STATE |
1629	IB_QP_QKEY),
1630	}
1631	},
1632	[IB_QPS_SQD] = {
1633	.valid = 1,
1634	.opt_param = {
1635	[IB_QPT_UD] = (IB_QP_PKEY_INDEX |
1636	IB_QP_QKEY),
1637	[IB_QPT_UC] = (IB_QP_AV |
1638	IB_QP_ALT_PATH |
1639	IB_QP_ACCESS_FLAGS |
1640	IB_QP_PKEY_INDEX |
1641	IB_QP_PATH_MIG_STATE),
1642	[IB_QPT_RC] = (IB_QP_PORT |
1643	IB_QP_AV |
1644	IB_QP_TIMEOUT |
1645	IB_QP_RETRY_CNT |
1646	IB_QP_RNR_RETRY |
1647	IB_QP_MAX_QP_RD_ATOMIC |
1648	IB_QP_MAX_DEST_RD_ATOMIC |
1649	IB_QP_ALT_PATH |
1650	IB_QP_ACCESS_FLAGS |
1651	IB_QP_PKEY_INDEX |
1652	IB_QP_MIN_RNR_TIMER |
1653	IB_QP_PATH_MIG_STATE),
1654	[IB_QPT_XRC_INI] = (IB_QP_PORT |
1655	IB_QP_AV |
1656	IB_QP_TIMEOUT |
1657	IB_QP_RETRY_CNT |
1658	IB_QP_RNR_RETRY |
1659	IB_QP_MAX_QP_RD_ATOMIC |
1660	IB_QP_ALT_PATH |
1661	IB_QP_ACCESS_FLAGS |
1662	IB_QP_PKEY_INDEX |
1663	IB_QP_PATH_MIG_STATE),
1664	[IB_QPT_XRC_TGT] = (IB_QP_PORT |
1665	IB_QP_AV |
1666	IB_QP_TIMEOUT |
1667	IB_QP_MAX_DEST_RD_ATOMIC |
1668	IB_QP_ALT_PATH |
1669	IB_QP_ACCESS_FLAGS |
1670	IB_QP_PKEY_INDEX |
1671	IB_QP_MIN_RNR_TIMER |
1672	IB_QP_PATH_MIG_STATE),
1673	[IB_QPT_SMI] = (IB_QP_PKEY_INDEX |
1674	IB_QP_QKEY),
1675	[IB_QPT_GSI] = (IB_QP_PKEY_INDEX |
1676	IB_QP_QKEY),
1677	}
1678	}
1679	},
1680	[IB_QPS_SQE] = {
1681	[IB_QPS_RESET] = { .valid = 1 },
1682	[IB_QPS_ERR] = { .valid = 1 },
1683	[IB_QPS_RTS] = {
1684	.valid = 1,
1685	.opt_param = {
1686	[IB_QPT_UD] = (IB_QP_CUR_STATE |
1687	IB_QP_QKEY),
1688	[IB_QPT_UC] = (IB_QP_CUR_STATE |
1689	IB_QP_ACCESS_FLAGS),
1690	[IB_QPT_SMI] = (IB_QP_CUR_STATE |
1691	IB_QP_QKEY),
1692	[IB_QPT_GSI] = (IB_QP_CUR_STATE |
1693	IB_QP_QKEY),
1694	}
1695	}
1696	},
1697	[IB_QPS_ERR] = {
1698	[IB_QPS_RESET] = { .valid = 1 },
1699	[IB_QPS_ERR] = { .valid = 1 }
1700	}
1701	};
1702
1703	bool ib_modify_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state,
1704	enum ib_qp_type type, enum ib_qp_attr_mask mask)
1705	{
1706	enum ib_qp_attr_mask req_param, opt_param;
1707
1708	if (mask & IB_QP_CUR_STATE &&
1709	cur_state != IB_QPS_RTR && cur_state != IB_QPS_RTS &&
1710	cur_state != IB_QPS_SQD && cur_state != IB_QPS_SQE)
1711	return false;
1712
1713	if (!qp_state_table[cur_state][next_state].valid)
1714	return false;
1715
1716	req_param = qp_state_table[cur_state][next_state].req_param[type];
1717	opt_param = qp_state_table[cur_state][next_state].opt_param[type];
1718
1719	if ((mask & req_param) != req_param)
1720	return false;
1721
1722	if (mask & ~(req_param | opt_param | IB_QP_STATE))
1723	return false;
1724
1725	return true;
1726	}
1727	EXPORT_SYMBOL(ib_modify_qp_is_ok);
1728
1729	/**
1730	* ib_resolve_eth_dmac - Resolve destination mac address
1731	* @device: Device to consider
1732	* @ah_attr: address handle attribute which describes the
1733	* source and destination parameters
1734	* ib_resolve_eth_dmac() resolves destination mac address and L3 hop limit It
1735	* returns 0 on success or appropriate error code. It initializes the
1736	* necessary ah_attr fields when call is successful.
1737	*/
1738	static int ib_resolve_eth_dmac(struct ib_device *device,
1739	struct rdma_ah_attr *ah_attr)
1740	{
1741	int ret = 0;
1742
1743	if (rdma_is_multicast_addr(addr: (struct in6_addr *)ah_attr->grh.dgid.raw)) {
1744	if (ipv6_addr_v4mapped(a: (struct in6_addr *)ah_attr->grh.dgid.raw)) {
1745	__be32 addr = 0;
1746
1747	memcpy(&addr, ah_attr->grh.dgid.raw + 12, 4);
1748	ip_eth_mc_map(naddr: addr, buf: (char *)ah_attr->roce.dmac);
1749	} else {
1750	ipv6_eth_mc_map(addr: (struct in6_addr *)ah_attr->grh.dgid.raw,
1751	buf: (char *)ah_attr->roce.dmac);
1752	}
1753	} else {
1754	ret = ib_resolve_unicast_gid_dmac(device, ah_attr);
1755	}
1756	return ret;
1757	}
1758
1759	static bool is_qp_type_connected(const struct ib_qp *qp)
1760	{
1761	return (qp->qp_type == IB_QPT_UC ||
1762	qp->qp_type == IB_QPT_RC ||
1763	qp->qp_type == IB_QPT_XRC_INI ||
1764	qp->qp_type == IB_QPT_XRC_TGT);
1765	}
1766
1767	/*
1768	* IB core internal function to perform QP attributes modification.
1769	*/
1770	static int _ib_modify_qp(struct ib_qp *qp, struct ib_qp_attr *attr,
1771	int attr_mask, struct ib_udata *udata)
1772	{
1773	u32 port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
1774	const struct ib_gid_attr *old_sgid_attr_av;
1775	const struct ib_gid_attr *old_sgid_attr_alt_av;
1776	int ret;
1777
1778	attr->xmit_slave = NULL;
1779	if (attr_mask & IB_QP_AV) {
1780	ret = rdma_fill_sgid_attr(device: qp->device, ah_attr: &attr->ah_attr,
1781	old_sgid_attr: &old_sgid_attr_av);
1782	if (ret)
1783	return ret;
1784
1785	if (attr->ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE &&
1786	is_qp_type_connected(qp)) {
1787	struct net_device *slave;
1788
1789	/*
1790	* If the user provided the qp_attr then we have to
1791	* resolve it. Kerne users have to provide already
1792	* resolved rdma_ah_attr's.
1793	*/
1794	if (udata) {
1795	ret = ib_resolve_eth_dmac(device: qp->device,
1796	ah_attr: &attr->ah_attr);
1797	if (ret)
1798	goto out_av;
1799	}
1800	slave = rdma_lag_get_ah_roce_slave(device: qp->device,
1801	ah_attr: &attr->ah_attr,
1802	GFP_KERNEL);
1803	if (IS_ERR(ptr: slave)) {
1804	ret = PTR_ERR(ptr: slave);
1805	goto out_av;
1806	}
1807	attr->xmit_slave = slave;
1808	}
1809	}
1810	if (attr_mask & IB_QP_ALT_PATH) {
1811	/*
1812	* FIXME: This does not track the migration state, so if the
1813	* user loads a new alternate path after the HW has migrated
1814	* from primary->alternate we will keep the wrong
1815	* references. This is OK for IB because the reference
1816	* counting does not serve any functional purpose.
1817	*/
1818	ret = rdma_fill_sgid_attr(device: qp->device, ah_attr: &attr->alt_ah_attr,
1819	old_sgid_attr: &old_sgid_attr_alt_av);
1820	if (ret)
1821	goto out_av;
1822
1823	/*
1824	* Today the core code can only handle alternate paths and APM
1825	* for IB. Ban them in roce mode.
1826	*/
1827	if (!(rdma_protocol_ib(device: qp->device,
1828	port_num: attr->alt_ah_attr.port_num) &&
1829	rdma_protocol_ib(device: qp->device, port_num: port))) {
1830	ret = -EINVAL;
1831	goto out;
1832	}
1833	}
1834
1835	if (rdma_ib_or_roce(device: qp->device, port_num: port)) {
1836	if (attr_mask & IB_QP_RQ_PSN && attr->rq_psn & ~0xffffff) {
1837	dev_warn(&qp->device->dev,
1838	"%s rq_psn overflow, masking to 24 bits\n",
1839	__func__);
1840	attr->rq_psn &= 0xffffff;
1841	}
1842
1843	if (attr_mask & IB_QP_SQ_PSN && attr->sq_psn & ~0xffffff) {
1844	dev_warn(&qp->device->dev,
1845	" %s sq_psn overflow, masking to 24 bits\n",
1846	__func__);
1847	attr->sq_psn &= 0xffffff;
1848	}
1849	}
1850
1851	/*
1852	* Bind this qp to a counter automatically based on the rdma counter
1853	* rules. This only set in RST2INIT with port specified
1854	*/
1855	if (!qp->counter && (attr_mask & IB_QP_PORT) &&
1856	((attr_mask & IB_QP_STATE) && attr->qp_state == IB_QPS_INIT))
1857	rdma_counter_bind_qp_auto(qp, port: attr->port_num);
1858
1859	ret = ib_security_modify_qp(qp, qp_attr: attr, qp_attr_mask: attr_mask, udata);
1860	if (ret)
1861	goto out;
1862
1863	if (attr_mask & IB_QP_PORT)
1864	qp->port = attr->port_num;
1865	if (attr_mask & IB_QP_AV)
1866	qp->av_sgid_attr =
1867	rdma_update_sgid_attr(ah_attr: &attr->ah_attr, old_attr: qp->av_sgid_attr);
1868	if (attr_mask & IB_QP_ALT_PATH)
1869	qp->alt_path_sgid_attr = rdma_update_sgid_attr(
1870	ah_attr: &attr->alt_ah_attr, old_attr: qp->alt_path_sgid_attr);
1871
1872	out:
1873	if (attr_mask & IB_QP_ALT_PATH)
1874	rdma_unfill_sgid_attr(ah_attr: &attr->alt_ah_attr, old_sgid_attr: old_sgid_attr_alt_av);
1875	out_av:
1876	if (attr_mask & IB_QP_AV) {
1877	rdma_lag_put_ah_roce_slave(xmit_slave: attr->xmit_slave);
1878	rdma_unfill_sgid_attr(ah_attr: &attr->ah_attr, old_sgid_attr: old_sgid_attr_av);
1879	}
1880	return ret;
1881	}
1882
1883	/**
1884	* ib_modify_qp_with_udata - Modifies the attributes for the specified QP.
1885	* @ib_qp: The QP to modify.
1886	* @attr: On input, specifies the QP attributes to modify. On output,
1887	* the current values of selected QP attributes are returned.
1888	* @attr_mask: A bit-mask used to specify which attributes of the QP
1889	* are being modified.
1890	* @udata: pointer to user's input output buffer information
1891	* are being modified.
1892	* It returns 0 on success and returns appropriate error code on error.
1893	*/
1894	int ib_modify_qp_with_udata(struct ib_qp *ib_qp, struct ib_qp_attr *attr,
1895	int attr_mask, struct ib_udata *udata)
1896	{
1897	return _ib_modify_qp(qp: ib_qp->real_qp, attr, attr_mask, udata);
1898	}
1899	EXPORT_SYMBOL(ib_modify_qp_with_udata);
1900
1901	static void ib_get_width_and_speed(u32 netdev_speed, u32 lanes,
1902	u16 *speed, u8 *width)
1903	{
1904	if (!lanes) {
1905	if (netdev_speed <= SPEED_1000) {
1906	*width = IB_WIDTH_1X;
1907	*speed = IB_SPEED_SDR;
1908	} else if (netdev_speed <= SPEED_10000) {
1909	*width = IB_WIDTH_1X;
1910	*speed = IB_SPEED_FDR10;
1911	} else if (netdev_speed <= SPEED_20000) {
1912	*width = IB_WIDTH_4X;
1913	*speed = IB_SPEED_DDR;
1914	} else if (netdev_speed <= SPEED_25000) {
1915	*width = IB_WIDTH_1X;
1916	*speed = IB_SPEED_EDR;
1917	} else if (netdev_speed <= SPEED_40000) {
1918	*width = IB_WIDTH_4X;
1919	*speed = IB_SPEED_FDR10;
1920	} else if (netdev_speed <= SPEED_50000) {
1921	*width = IB_WIDTH_2X;
1922	*speed = IB_SPEED_EDR;
1923	} else if (netdev_speed <= SPEED_100000) {
1924	*width = IB_WIDTH_4X;
1925	*speed = IB_SPEED_EDR;
1926	} else if (netdev_speed <= SPEED_200000) {
1927	*width = IB_WIDTH_4X;
1928	*speed = IB_SPEED_HDR;
1929	} else {
1930	*width = IB_WIDTH_4X;
1931	*speed = IB_SPEED_NDR;
1932	}
1933
1934	return;
1935	}
1936
1937	switch (lanes) {
1938	case 1:
1939	*width = IB_WIDTH_1X;
1940	break;
1941	case 2:
1942	*width = IB_WIDTH_2X;
1943	break;
1944	case 4:
1945	*width = IB_WIDTH_4X;
1946	break;
1947	case 8:
1948	*width = IB_WIDTH_8X;
1949	break;
1950	case 12:
1951	*width = IB_WIDTH_12X;
1952	break;
1953	default:
1954	*width = IB_WIDTH_1X;
1955	}
1956
1957	switch (netdev_speed / lanes) {
1958	case SPEED_2500:
1959	*speed = IB_SPEED_SDR;
1960	break;
1961	case SPEED_5000:
1962	*speed = IB_SPEED_DDR;
1963	break;
1964	case SPEED_10000:
1965	*speed = IB_SPEED_FDR10;
1966	break;
1967	case SPEED_14000:
1968	*speed = IB_SPEED_FDR;
1969	break;
1970	case SPEED_25000:
1971	*speed = IB_SPEED_EDR;
1972	break;
1973	case SPEED_50000:
1974	*speed = IB_SPEED_HDR;
1975	break;
1976	case SPEED_100000:
1977	*speed = IB_SPEED_NDR;
1978	break;
1979	default:
1980	*speed = IB_SPEED_SDR;
1981	}
1982	}
1983
1984	int ib_get_eth_speed(struct ib_device *dev, u32 port_num, u16 *speed, u8 *width)
1985	{
1986	int rc;
1987	u32 netdev_speed;
1988	struct net_device *netdev;
1989	struct ethtool_link_ksettings lksettings = {};
1990
1991	if (rdma_port_get_link_layer(dev, port_num) != IB_LINK_LAYER_ETHERNET)
1992	return -EINVAL;
1993
1994	netdev = ib_device_get_netdev(ib_dev: dev, port: port_num);
1995	if (!netdev)
1996	return -ENODEV;
1997
1998	rtnl_lock();
1999	rc = __ethtool_get_link_ksettings(dev: netdev, link_ksettings: &lksettings);
2000	rtnl_unlock();
2001
2002	dev_put(dev: netdev);
2003
2004	if (!rc && lksettings.base.speed != (u32)SPEED_UNKNOWN) {
2005	netdev_speed = lksettings.base.speed;
2006	} else {
2007	netdev_speed = SPEED_1000;
2008	if (rc)
2009	pr_warn("%s speed is unknown, defaulting to %u\n",
2010	netdev->name, netdev_speed);
2011	}
2012
2013	ib_get_width_and_speed(netdev_speed, lanes: lksettings.lanes,
2014	speed, width);
2015
2016	return 0;
2017	}
2018	EXPORT_SYMBOL(ib_get_eth_speed);
2019
2020	int ib_modify_qp(struct ib_qp *qp,
2021	struct ib_qp_attr *qp_attr,
2022	int qp_attr_mask)
2023	{
2024	return _ib_modify_qp(qp: qp->real_qp, attr: qp_attr, attr_mask: qp_attr_mask, NULL);
2025	}
2026	EXPORT_SYMBOL(ib_modify_qp);
2027
2028	int ib_query_qp(struct ib_qp *qp,
2029	struct ib_qp_attr *qp_attr,
2030	int qp_attr_mask,
2031	struct ib_qp_init_attr *qp_init_attr)
2032	{
2033	qp_attr->ah_attr.grh.sgid_attr = NULL;
2034	qp_attr->alt_ah_attr.grh.sgid_attr = NULL;
2035
2036	return qp->device->ops.query_qp ?
2037	qp->device->ops.query_qp(qp->real_qp, qp_attr, qp_attr_mask,
2038	qp_init_attr) : -EOPNOTSUPP;
2039	}
2040	EXPORT_SYMBOL(ib_query_qp);
2041
2042	int ib_close_qp(struct ib_qp *qp)
2043	{
2044	struct ib_qp *real_qp;
2045	unsigned long flags;
2046
2047	real_qp = qp->real_qp;
2048	if (real_qp == qp)
2049	return -EINVAL;
2050
2051	spin_lock_irqsave(&real_qp->device->qp_open_list_lock, flags);
2052	list_del(entry: &qp->open_list);
2053	spin_unlock_irqrestore(lock: &real_qp->device->qp_open_list_lock, flags);
2054
2055	atomic_dec(v: &real_qp->usecnt);
2056	if (qp->qp_sec)
2057	ib_close_shared_qp_security(sec: qp->qp_sec);
2058	kfree(objp: qp);
2059
2060	return 0;
2061	}
2062	EXPORT_SYMBOL(ib_close_qp);
2063
2064	static int __ib_destroy_shared_qp(struct ib_qp *qp)
2065	{
2066	struct ib_xrcd *xrcd;
2067	struct ib_qp *real_qp;
2068	int ret;
2069
2070	real_qp = qp->real_qp;
2071	xrcd = real_qp->xrcd;
2072	down_write(sem: &xrcd->tgt_qps_rwsem);
2073	ib_close_qp(qp);
2074	if (atomic_read(v: &real_qp->usecnt) == 0)
2075	xa_erase(&xrcd->tgt_qps, index: real_qp->qp_num);
2076	else
2077	real_qp = NULL;
2078	up_write(sem: &xrcd->tgt_qps_rwsem);
2079
2080	if (real_qp) {
2081	ret = ib_destroy_qp(qp: real_qp);
2082	if (!ret)
2083	atomic_dec(v: &xrcd->usecnt);
2084	}
2085
2086	return 0;
2087	}
2088
2089	int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata)
2090	{
2091	const struct ib_gid_attr *alt_path_sgid_attr = qp->alt_path_sgid_attr;
2092	const struct ib_gid_attr *av_sgid_attr = qp->av_sgid_attr;
2093	struct ib_qp_security *sec;
2094	int ret;
2095
2096	WARN_ON_ONCE(qp->mrs_used > 0);
2097
2098	if (atomic_read(v: &qp->usecnt))
2099	return -EBUSY;
2100
2101	if (qp->real_qp != qp)
2102	return __ib_destroy_shared_qp(qp);
2103
2104	sec = qp->qp_sec;
2105	if (sec)
2106	ib_destroy_qp_security_begin(sec);
2107
2108	if (!qp->uobject)
2109	rdma_rw_cleanup_mrs(qp);
2110
2111	rdma_counter_unbind_qp(qp, port: qp->port, force: true);
2112	ret = qp->device->ops.destroy_qp(qp, udata);
2113	if (ret) {
2114	if (sec)
2115	ib_destroy_qp_security_abort(sec);
2116	return ret;
2117	}
2118
2119	if (alt_path_sgid_attr)
2120	rdma_put_gid_attr(attr: alt_path_sgid_attr);
2121	if (av_sgid_attr)
2122	rdma_put_gid_attr(attr: av_sgid_attr);
2123
2124	ib_qp_usecnt_dec(qp);
2125	if (sec)
2126	ib_destroy_qp_security_end(sec);
2127
2128	rdma_restrack_del(res: &qp->res);
2129	kfree(objp: qp);
2130	return ret;
2131	}
2132	EXPORT_SYMBOL(ib_destroy_qp_user);
2133
2134	/* Completion queues */
2135
2136	struct ib_cq *__ib_create_cq(struct ib_device *device,
2137	ib_comp_handler comp_handler,
2138	void (*event_handler)(struct ib_event *, void *),
2139	void *cq_context,
2140	const struct ib_cq_init_attr *cq_attr,
2141	const char *caller)
2142	{
2143	struct ib_cq *cq;
2144	int ret;
2145
2146	cq = rdma_zalloc_drv_obj(device, ib_cq);
2147	if (!cq)
2148	return ERR_PTR(error: -ENOMEM);
2149
2150	cq->device = device;
2151	cq->uobject = NULL;
2152	cq->comp_handler = comp_handler;
2153	cq->event_handler = event_handler;
2154	cq->cq_context = cq_context;
2155	atomic_set(v: &cq->usecnt, i: 0);
2156
2157	rdma_restrack_new(res: &cq->res, type: RDMA_RESTRACK_CQ);
2158	rdma_restrack_set_name(res: &cq->res, caller);
2159
2160	ret = device->ops.create_cq(cq, cq_attr, NULL);
2161	if (ret) {
2162	rdma_restrack_put(res: &cq->res);
2163	kfree(objp: cq);
2164	return ERR_PTR(error: ret);
2165	}
2166
2167	rdma_restrack_add(res: &cq->res);
2168	return cq;
2169	}
2170	EXPORT_SYMBOL(__ib_create_cq);
2171
2172	int rdma_set_cq_moderation(struct ib_cq *cq, u16 cq_count, u16 cq_period)
2173	{
2174	if (cq->shared)
2175	return -EOPNOTSUPP;
2176
2177	return cq->device->ops.modify_cq ?
2178	cq->device->ops.modify_cq(cq, cq_count,
2179	cq_period) : -EOPNOTSUPP;
2180	}
2181	EXPORT_SYMBOL(rdma_set_cq_moderation);
2182
2183	int ib_destroy_cq_user(struct ib_cq *cq, struct ib_udata *udata)
2184	{
2185	int ret;
2186
2187	if (WARN_ON_ONCE(cq->shared))
2188	return -EOPNOTSUPP;
2189
2190	if (atomic_read(v: &cq->usecnt))
2191	return -EBUSY;
2192
2193	ret = cq->device->ops.destroy_cq(cq, udata);
2194	if (ret)
2195	return ret;
2196
2197	rdma_restrack_del(res: &cq->res);
2198	kfree(objp: cq);
2199	return ret;
2200	}
2201	EXPORT_SYMBOL(ib_destroy_cq_user);
2202
2203	int ib_resize_cq(struct ib_cq *cq, int cqe)
2204	{
2205	if (cq->shared)
2206	return -EOPNOTSUPP;
2207
2208	return cq->device->ops.resize_cq ?
2209	cq->device->ops.resize_cq(cq, cqe, NULL) : -EOPNOTSUPP;
2210	}
2211	EXPORT_SYMBOL(ib_resize_cq);
2212
2213	/* Memory regions */
2214
2215	struct ib_mr *ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
2216	u64 virt_addr, int access_flags)
2217	{
2218	struct ib_mr *mr;
2219
2220	if (access_flags & IB_ACCESS_ON_DEMAND) {
2221	if (!(pd->device->attrs.kernel_cap_flags &
2222	IBK_ON_DEMAND_PAGING)) {
2223	pr_debug("ODP support not available\n");
2224	return ERR_PTR(error: -EINVAL);
2225	}
2226	}
2227
2228	mr = pd->device->ops.reg_user_mr(pd, start, length, virt_addr,
2229	access_flags, NULL, NULL);
2230
2231	if (IS_ERR(ptr: mr))
2232	return mr;
2233
2234	mr->device = pd->device;
2235	mr->type = IB_MR_TYPE_USER;
2236	mr->pd = pd;
2237	mr->dm = NULL;
2238	atomic_inc(v: &pd->usecnt);
2239	mr->iova = virt_addr;
2240	mr->length = length;
2241
2242	rdma_restrack_new(res: &mr->res, type: RDMA_RESTRACK_MR);
2243	rdma_restrack_parent_name(dst: &mr->res, parent: &pd->res);
2244	rdma_restrack_add(res: &mr->res);
2245
2246	return mr;
2247	}
2248	EXPORT_SYMBOL(ib_reg_user_mr);
2249
2250	int ib_advise_mr(struct ib_pd *pd, enum ib_uverbs_advise_mr_advice advice,
2251	u32 flags, struct ib_sge *sg_list, u32 num_sge)
2252	{
2253	if (!pd->device->ops.advise_mr)
2254	return -EOPNOTSUPP;
2255
2256	if (!num_sge)
2257	return 0;
2258
2259	return pd->device->ops.advise_mr(pd, advice, flags, sg_list, num_sge,
2260	NULL);
2261	}
2262	EXPORT_SYMBOL(ib_advise_mr);
2263
2264	int ib_dereg_mr_user(struct ib_mr *mr, struct ib_udata *udata)
2265	{
2266	struct ib_pd *pd = mr->pd;
2267	struct ib_dm *dm = mr->dm;
2268	struct ib_dmah *dmah = mr->dmah;
2269	struct ib_sig_attrs *sig_attrs = mr->sig_attrs;
2270	int ret;
2271
2272	trace_mr_dereg(mr);
2273	rdma_restrack_del(res: &mr->res);
2274	ret = mr->device->ops.dereg_mr(mr, udata);
2275	if (!ret) {
2276	atomic_dec(v: &pd->usecnt);
2277	if (dm)
2278	atomic_dec(v: &dm->usecnt);
2279	if (dmah)
2280	atomic_dec(v: &dmah->usecnt);
2281	kfree(objp: sig_attrs);
2282	}
2283
2284	return ret;
2285	}
2286	EXPORT_SYMBOL(ib_dereg_mr_user);
2287
2288	/**
2289	* ib_alloc_mr() - Allocates a memory region
2290	* @pd: protection domain associated with the region
2291	* @mr_type: memory region type
2292	* @max_num_sg: maximum sg entries available for registration.
2293	*
2294	* Notes:
2295	* Memory registeration page/sg lists must not exceed max_num_sg.
2296	* For mr_type IB_MR_TYPE_MEM_REG, the total length cannot exceed
2297	* max_num_sg * used_page_size.
2298	*
2299	*/
2300	struct ib_mr *ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
2301	u32 max_num_sg)
2302	{
2303	struct ib_mr *mr;
2304
2305	if (!pd->device->ops.alloc_mr) {
2306	mr = ERR_PTR(error: -EOPNOTSUPP);
2307	goto out;
2308	}
2309
2310	if (mr_type == IB_MR_TYPE_INTEGRITY) {
2311	WARN_ON_ONCE(1);
2312	mr = ERR_PTR(error: -EINVAL);
2313	goto out;
2314	}
2315
2316	mr = pd->device->ops.alloc_mr(pd, mr_type, max_num_sg);
2317	if (IS_ERR(ptr: mr))
2318	goto out;
2319
2320	mr->device = pd->device;
2321	mr->pd = pd;
2322	mr->dm = NULL;
2323	mr->uobject = NULL;
2324	atomic_inc(v: &pd->usecnt);
2325	mr->need_inval = false;
2326	mr->type = mr_type;
2327	mr->sig_attrs = NULL;
2328
2329	rdma_restrack_new(res: &mr->res, type: RDMA_RESTRACK_MR);
2330	rdma_restrack_parent_name(dst: &mr->res, parent: &pd->res);
2331	rdma_restrack_add(res: &mr->res);
2332	out:
2333	trace_mr_alloc(pd, mr_type, max_num_sg, mr);
2334	return mr;
2335	}
2336	EXPORT_SYMBOL(ib_alloc_mr);
2337
2338	/**
2339	* ib_alloc_mr_integrity() - Allocates an integrity memory region
2340	* @pd: protection domain associated with the region
2341	* @max_num_data_sg: maximum data sg entries available for registration
2342	* @max_num_meta_sg: maximum metadata sg entries available for
2343	* registration
2344	*
2345	* Notes:
2346	* Memory registration page/sg lists must not exceed max_num_sg,
2347	* also the integrity page/sg lists must not exceed max_num_meta_sg.
2348	*
2349	*/
2350	struct ib_mr *ib_alloc_mr_integrity(struct ib_pd *pd,
2351	u32 max_num_data_sg,
2352	u32 max_num_meta_sg)
2353	{
2354	struct ib_mr *mr;
2355	struct ib_sig_attrs *sig_attrs;
2356
2357	if (!pd->device->ops.alloc_mr_integrity ||
2358	!pd->device->ops.map_mr_sg_pi) {
2359	mr = ERR_PTR(error: -EOPNOTSUPP);
2360	goto out;
2361	}
2362
2363	if (!max_num_meta_sg) {
2364	mr = ERR_PTR(error: -EINVAL);
2365	goto out;
2366	}
2367
2368	sig_attrs = kzalloc(sizeof(struct ib_sig_attrs), GFP_KERNEL);
2369	if (!sig_attrs) {
2370	mr = ERR_PTR(error: -ENOMEM);
2371	goto out;
2372	}
2373
2374	mr = pd->device->ops.alloc_mr_integrity(pd, max_num_data_sg,
2375	max_num_meta_sg);
2376	if (IS_ERR(ptr: mr)) {
2377	kfree(objp: sig_attrs);
2378	goto out;
2379	}
2380
2381	mr->device = pd->device;
2382	mr->pd = pd;
2383	mr->dm = NULL;
2384	mr->uobject = NULL;
2385	atomic_inc(v: &pd->usecnt);
2386	mr->need_inval = false;
2387	mr->type = IB_MR_TYPE_INTEGRITY;
2388	mr->sig_attrs = sig_attrs;
2389
2390	rdma_restrack_new(res: &mr->res, type: RDMA_RESTRACK_MR);
2391	rdma_restrack_parent_name(dst: &mr->res, parent: &pd->res);
2392	rdma_restrack_add(res: &mr->res);
2393	out:
2394	trace_mr_integ_alloc(pd, max_num_data_sg, max_num_meta_sg, mr);
2395	return mr;
2396	}
2397	EXPORT_SYMBOL(ib_alloc_mr_integrity);
2398
2399	/* Multicast groups */
2400
2401	static bool is_valid_mcast_lid(struct ib_qp *qp, u16 lid)
2402	{
2403	struct ib_qp_init_attr init_attr = {};
2404	struct ib_qp_attr attr = {};
2405	int num_eth_ports = 0;
2406	unsigned int port;
2407
2408	/* If QP state >= init, it is assigned to a port and we can check this
2409	* port only.
2410	*/
2411	if (!ib_query_qp(qp, &attr, IB_QP_STATE | IB_QP_PORT, &init_attr)) {
2412	if (attr.qp_state >= IB_QPS_INIT) {
2413	if (rdma_port_get_link_layer(qp->device, attr.port_num) !=
2414	IB_LINK_LAYER_INFINIBAND)
2415	return true;
2416	goto lid_check;
2417	}
2418	}
2419
2420	/* Can't get a quick answer, iterate over all ports */
2421	rdma_for_each_port(qp->device, port)
2422	if (rdma_port_get_link_layer(qp->device, port) !=
2423	IB_LINK_LAYER_INFINIBAND)
2424	num_eth_ports++;
2425
2426	/* If we have at lease one Ethernet port, RoCE annex declares that
2427	* multicast LID should be ignored. We can't tell at this step if the
2428	* QP belongs to an IB or Ethernet port.
2429	*/
2430	if (num_eth_ports)
2431	return true;
2432
2433	/* If all the ports are IB, we can check according to IB spec. */
2434	lid_check:
2435	return !(lid < be16_to_cpu(IB_MULTICAST_LID_BASE) ||
2436	lid == be16_to_cpu(IB_LID_PERMISSIVE));
2437	}
2438
2439	int ib_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
2440	{
2441	int ret;
2442
2443	if (!qp->device->ops.attach_mcast)
2444	return -EOPNOTSUPP;
2445
2446	if (!rdma_is_multicast_addr(addr: (struct in6_addr *)gid->raw) ||
2447	qp->qp_type != IB_QPT_UD || !is_valid_mcast_lid(qp, lid))
2448	return -EINVAL;
2449
2450	ret = qp->device->ops.attach_mcast(qp, gid, lid);
2451	if (!ret)
2452	atomic_inc(v: &qp->usecnt);
2453	return ret;
2454	}
2455	EXPORT_SYMBOL(ib_attach_mcast);
2456
2457	int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
2458	{
2459	int ret;
2460
2461	if (!qp->device->ops.detach_mcast)
2462	return -EOPNOTSUPP;
2463
2464	if (!rdma_is_multicast_addr(addr: (struct in6_addr *)gid->raw) ||
2465	qp->qp_type != IB_QPT_UD || !is_valid_mcast_lid(qp, lid))
2466	return -EINVAL;
2467
2468	ret = qp->device->ops.detach_mcast(qp, gid, lid);
2469	if (!ret)
2470	atomic_dec(v: &qp->usecnt);
2471	return ret;
2472	}
2473	EXPORT_SYMBOL(ib_detach_mcast);
2474
2475	/**
2476	* ib_alloc_xrcd_user - Allocates an XRC domain.
2477	* @device: The device on which to allocate the XRC domain.
2478	* @inode: inode to connect XRCD
2479	* @udata: Valid user data or NULL for kernel object
2480	*/
2481	struct ib_xrcd *ib_alloc_xrcd_user(struct ib_device *device,
2482	struct inode *inode, struct ib_udata *udata)
2483	{
2484	struct ib_xrcd *xrcd;
2485	int ret;
2486
2487	if (!device->ops.alloc_xrcd)
2488	return ERR_PTR(error: -EOPNOTSUPP);
2489
2490	xrcd = rdma_zalloc_drv_obj(device, ib_xrcd);
2491	if (!xrcd)
2492	return ERR_PTR(error: -ENOMEM);
2493
2494	xrcd->device = device;
2495	xrcd->inode = inode;
2496	atomic_set(v: &xrcd->usecnt, i: 0);
2497	init_rwsem(&xrcd->tgt_qps_rwsem);
2498	xa_init(xa: &xrcd->tgt_qps);
2499
2500	ret = device->ops.alloc_xrcd(xrcd, udata);
2501	if (ret)
2502	goto err;
2503	return xrcd;
2504	err:
2505	kfree(objp: xrcd);
2506	return ERR_PTR(error: ret);
2507	}
2508	EXPORT_SYMBOL(ib_alloc_xrcd_user);
2509
2510	/**
2511	* ib_dealloc_xrcd_user - Deallocates an XRC domain.
2512	* @xrcd: The XRC domain to deallocate.
2513	* @udata: Valid user data or NULL for kernel object
2514	*/
2515	int ib_dealloc_xrcd_user(struct ib_xrcd *xrcd, struct ib_udata *udata)
2516	{
2517	int ret;
2518
2519	if (atomic_read(v: &xrcd->usecnt))
2520	return -EBUSY;
2521
2522	WARN_ON(!xa_empty(&xrcd->tgt_qps));
2523	ret = xrcd->device->ops.dealloc_xrcd(xrcd, udata);
2524	if (ret)
2525	return ret;
2526	kfree(objp: xrcd);
2527	return ret;
2528	}
2529	EXPORT_SYMBOL(ib_dealloc_xrcd_user);
2530
2531	/**
2532	* ib_create_wq - Creates a WQ associated with the specified protection
2533	* domain.
2534	* @pd: The protection domain associated with the WQ.
2535	* @wq_attr: A list of initial attributes required to create the
2536	* WQ. If WQ creation succeeds, then the attributes are updated to
2537	* the actual capabilities of the created WQ.
2538	*
2539	* wq_attr->max_wr and wq_attr->max_sge determine
2540	* the requested size of the WQ, and set to the actual values allocated
2541	* on return.
2542	* If ib_create_wq() succeeds, then max_wr and max_sge will always be
2543	* at least as large as the requested values.
2544	*/
2545	struct ib_wq *ib_create_wq(struct ib_pd *pd,
2546	struct ib_wq_init_attr *wq_attr)
2547	{
2548	struct ib_wq *wq;
2549
2550	if (!pd->device->ops.create_wq)
2551	return ERR_PTR(error: -EOPNOTSUPP);
2552
2553	wq = pd->device->ops.create_wq(pd, wq_attr, NULL);
2554	if (!IS_ERR(ptr: wq)) {
2555	wq->event_handler = wq_attr->event_handler;
2556	wq->wq_context = wq_attr->wq_context;
2557	wq->wq_type = wq_attr->wq_type;
2558	wq->cq = wq_attr->cq;
2559	wq->device = pd->device;
2560	wq->pd = pd;
2561	wq->uobject = NULL;
2562	atomic_inc(v: &pd->usecnt);
2563	atomic_inc(v: &wq_attr->cq->usecnt);
2564	atomic_set(v: &wq->usecnt, i: 0);
2565	}
2566	return wq;
2567	}
2568	EXPORT_SYMBOL(ib_create_wq);
2569
2570	/**
2571	* ib_destroy_wq_user - Destroys the specified user WQ.
2572	* @wq: The WQ to destroy.
2573	* @udata: Valid user data
2574	*/
2575	int ib_destroy_wq_user(struct ib_wq *wq, struct ib_udata *udata)
2576	{
2577	struct ib_cq *cq = wq->cq;
2578	struct ib_pd *pd = wq->pd;
2579	int ret;
2580
2581	if (atomic_read(v: &wq->usecnt))
2582	return -EBUSY;
2583
2584	ret = wq->device->ops.destroy_wq(wq, udata);
2585	if (ret)
2586	return ret;
2587
2588	atomic_dec(v: &pd->usecnt);
2589	atomic_dec(v: &cq->usecnt);
2590	return ret;
2591	}
2592	EXPORT_SYMBOL(ib_destroy_wq_user);
2593
2594	int ib_check_mr_status(struct ib_mr *mr, u32 check_mask,
2595	struct ib_mr_status *mr_status)
2596	{
2597	if (!mr->device->ops.check_mr_status)
2598	return -EOPNOTSUPP;
2599
2600	return mr->device->ops.check_mr_status(mr, check_mask, mr_status);
2601	}
2602	EXPORT_SYMBOL(ib_check_mr_status);
2603
2604	int ib_set_vf_link_state(struct ib_device *device, int vf, u32 port,
2605	int state)
2606	{
2607	if (!device->ops.set_vf_link_state)
2608	return -EOPNOTSUPP;
2609
2610	return device->ops.set_vf_link_state(device, vf, port, state);
2611	}
2612	EXPORT_SYMBOL(ib_set_vf_link_state);
2613
2614	int ib_get_vf_config(struct ib_device *device, int vf, u32 port,
2615	struct ifla_vf_info *info)
2616	{
2617	if (!device->ops.get_vf_config)
2618	return -EOPNOTSUPP;
2619
2620	return device->ops.get_vf_config(device, vf, port, info);
2621	}
2622	EXPORT_SYMBOL(ib_get_vf_config);
2623
2624	int ib_get_vf_stats(struct ib_device *device, int vf, u32 port,
2625	struct ifla_vf_stats *stats)
2626	{
2627	if (!device->ops.get_vf_stats)
2628	return -EOPNOTSUPP;
2629
2630	return device->ops.get_vf_stats(device, vf, port, stats);
2631	}
2632	EXPORT_SYMBOL(ib_get_vf_stats);
2633
2634	int ib_set_vf_guid(struct ib_device *device, int vf, u32 port, u64 guid,
2635	int type)
2636	{
2637	if (!device->ops.set_vf_guid)
2638	return -EOPNOTSUPP;
2639
2640	return device->ops.set_vf_guid(device, vf, port, guid, type);
2641	}
2642	EXPORT_SYMBOL(ib_set_vf_guid);
2643
2644	int ib_get_vf_guid(struct ib_device *device, int vf, u32 port,
2645	struct ifla_vf_guid *node_guid,
2646	struct ifla_vf_guid *port_guid)
2647	{
2648	if (!device->ops.get_vf_guid)
2649	return -EOPNOTSUPP;
2650
2651	return device->ops.get_vf_guid(device, vf, port, node_guid, port_guid);
2652	}
2653	EXPORT_SYMBOL(ib_get_vf_guid);
2654	/**
2655	* ib_map_mr_sg_pi() - Map the dma mapped SG lists for PI (protection
2656	* information) and set an appropriate memory region for registration.
2657	* @mr: memory region
2658	* @data_sg: dma mapped scatterlist for data
2659	* @data_sg_nents: number of entries in data_sg
2660	* @data_sg_offset: offset in bytes into data_sg
2661	* @meta_sg: dma mapped scatterlist for metadata
2662	* @meta_sg_nents: number of entries in meta_sg
2663	* @meta_sg_offset: offset in bytes into meta_sg
2664	* @page_size: page vector desired page size
2665	*
2666	* Constraints:
2667	* - The MR must be allocated with type IB_MR_TYPE_INTEGRITY.
2668	*
2669	* Return: 0 on success.
2670	*
2671	* After this completes successfully, the memory region
2672	* is ready for registration.
2673	*/
2674	int ib_map_mr_sg_pi(struct ib_mr *mr, struct scatterlist *data_sg,
2675	int data_sg_nents, unsigned int *data_sg_offset,
2676	struct scatterlist *meta_sg, int meta_sg_nents,
2677	unsigned int *meta_sg_offset, unsigned int page_size)
2678	{
2679	if (unlikely(!mr->device->ops.map_mr_sg_pi ||
2680	WARN_ON_ONCE(mr->type != IB_MR_TYPE_INTEGRITY)))
2681	return -EOPNOTSUPP;
2682
2683	mr->page_size = page_size;
2684
2685	return mr->device->ops.map_mr_sg_pi(mr, data_sg, data_sg_nents,
2686	data_sg_offset, meta_sg,
2687	meta_sg_nents, meta_sg_offset);
2688	}
2689	EXPORT_SYMBOL(ib_map_mr_sg_pi);
2690
2691	/**
2692	* ib_map_mr_sg() - Map the largest prefix of a dma mapped SG list
2693	* and set it the memory region.
2694	* @mr: memory region
2695	* @sg: dma mapped scatterlist
2696	* @sg_nents: number of entries in sg
2697	* @sg_offset: offset in bytes into sg
2698	* @page_size: page vector desired page size
2699	*
2700	* Constraints:
2701	*
2702	* - The first sg element is allowed to have an offset.
2703	* - Each sg element must either be aligned to page_size or virtually
2704	* contiguous to the previous element. In case an sg element has a
2705	* non-contiguous offset, the mapping prefix will not include it.
2706	* - The last sg element is allowed to have length less than page_size.
2707	* - If sg_nents total byte length exceeds the mr max_num_sge * page_size
2708	* then only max_num_sg entries will be mapped.
2709	* - If the MR was allocated with type IB_MR_TYPE_SG_GAPS, none of these
2710	* constraints holds and the page_size argument is ignored.
2711	*
2712	* Returns the number of sg elements that were mapped to the memory region.
2713	*
2714	* After this completes successfully, the memory region
2715	* is ready for registration.
2716	*/
2717	int ib_map_mr_sg(struct ib_mr *mr, struct scatterlist *sg, int sg_nents,
2718	unsigned int *sg_offset, unsigned int page_size)
2719	{
2720	if (unlikely(!mr->device->ops.map_mr_sg))
2721	return -EOPNOTSUPP;
2722
2723	mr->page_size = page_size;
2724
2725	return mr->device->ops.map_mr_sg(mr, sg, sg_nents, sg_offset);
2726	}
2727	EXPORT_SYMBOL(ib_map_mr_sg);
2728
2729	/**
2730	* ib_sg_to_pages() - Convert the largest prefix of a sg list
2731	* to a page vector
2732	* @mr: memory region
2733	* @sgl: dma mapped scatterlist
2734	* @sg_nents: number of entries in sg
2735	* @sg_offset_p: ==== =======================================================
2736	* IN start offset in bytes into sg
2737	* OUT offset in bytes for element n of the sg of the first
2738	* byte that has not been processed where n is the return
2739	* value of this function.
2740	* ==== =======================================================
2741	* @set_page: driver page assignment function pointer
2742	*
2743	* Core service helper for drivers to convert the largest
2744	* prefix of given sg list to a page vector. The sg list
2745	* prefix converted is the prefix that meet the requirements
2746	* of ib_map_mr_sg.
2747	*
2748	* Returns the number of sg elements that were assigned to
2749	* a page vector.
2750	*/
2751	int ib_sg_to_pages(struct ib_mr *mr, struct scatterlist *sgl, int sg_nents,
2752	unsigned int *sg_offset_p, int (*set_page)(struct ib_mr *, u64))
2753	{
2754	struct scatterlist *sg;
2755	u64 last_end_dma_addr = 0;
2756	unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0;
2757	unsigned int last_page_off = 0;
2758	u64 page_mask = ~((u64)mr->page_size - 1);
2759	int i, ret;
2760
2761	if (unlikely(sg_nents <= 0 || sg_offset > sg_dma_len(&sgl[0])))
2762	return -EINVAL;
2763
2764	mr->iova = sg_dma_address(&sgl[0]) + sg_offset;
2765	mr->length = 0;
2766
2767	for_each_sg(sgl, sg, sg_nents, i) {
2768	u64 dma_addr = sg_dma_address(sg) + sg_offset;
2769	u64 prev_addr = dma_addr;
2770	unsigned int dma_len = sg_dma_len(sg) - sg_offset;
2771	u64 end_dma_addr = dma_addr + dma_len;
2772	u64 page_addr = dma_addr & page_mask;
2773
2774	/*
2775	* For the second and later elements, check whether either the
2776	* end of element i-1 or the start of element i is not aligned
2777	* on a page boundary.
2778	*/
2779	if (i && (last_page_off != 0 || page_addr != dma_addr)) {
2780	/* Stop mapping if there is a gap. */
2781	if (last_end_dma_addr != dma_addr)
2782	break;
2783
2784	/*
2785	* Coalesce this element with the last. If it is small
2786	* enough just update mr->length. Otherwise start
2787	* mapping from the next page.
2788	*/
2789	goto next_page;
2790	}
2791
2792	do {
2793	ret = set_page(mr, page_addr);
2794	if (unlikely(ret < 0)) {
2795	sg_offset = prev_addr - sg_dma_address(sg);
2796	mr->length += prev_addr - dma_addr;
2797	if (sg_offset_p)
2798	*sg_offset_p = sg_offset;
2799	return i || sg_offset ? i : ret;
2800	}
2801	prev_addr = page_addr;
2802	next_page:
2803	page_addr += mr->page_size;
2804	} while (page_addr < end_dma_addr);
2805
2806	mr->length += dma_len;
2807	last_end_dma_addr = end_dma_addr;
2808	last_page_off = end_dma_addr & ~page_mask;
2809
2810	sg_offset = 0;
2811	}
2812
2813	if (sg_offset_p)
2814	*sg_offset_p = 0;
2815	return i;
2816	}
2817	EXPORT_SYMBOL(ib_sg_to_pages);
2818
2819	struct ib_drain_cqe {
2820	struct ib_cqe cqe;
2821	struct completion done;
2822	};
2823
2824	static void ib_drain_qp_done(struct ib_cq *cq, struct ib_wc *wc)
2825	{
2826	struct ib_drain_cqe *cqe = container_of(wc->wr_cqe, struct ib_drain_cqe,
2827	cqe);
2828
2829	complete(&cqe->done);
2830	}
2831
2832	/*
2833	* Post a WR and block until its completion is reaped for the SQ.
2834	*/
2835	static void __ib_drain_sq(struct ib_qp *qp)
2836	{
2837	struct ib_cq *cq = qp->send_cq;
2838	struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
2839	struct ib_drain_cqe sdrain;
2840	struct ib_rdma_wr swr = {
2841	.wr = {
2842	.next = NULL,
2843	{ .wr_cqe = &sdrain.cqe, },
2844	.opcode = IB_WR_RDMA_WRITE,
2845	},
2846	};
2847	int ret;
2848
2849	ret = ib_modify_qp(qp, &attr, IB_QP_STATE);
2850	if (ret) {
2851	WARN_ONCE(ret, "failed to drain send queue: %d\n", ret);
2852	return;
2853	}
2854
2855	sdrain.cqe.done = ib_drain_qp_done;
2856	init_completion(x: &sdrain.done);
2857
2858	ret = ib_post_send(qp, send_wr: &swr.wr, NULL);
2859	if (ret) {
2860	WARN_ONCE(ret, "failed to drain send queue: %d\n", ret);
2861	return;
2862	}
2863
2864	if (cq->poll_ctx == IB_POLL_DIRECT)
2865	while (wait_for_completion_timeout(x: &sdrain.done, HZ / 10) <= 0)
2866	ib_process_cq_direct(cq, budget: -1);
2867	else
2868	wait_for_completion(&sdrain.done);
2869	}
2870
2871	/*
2872	* Post a WR and block until its completion is reaped for the RQ.
2873	*/
2874	static void __ib_drain_rq(struct ib_qp *qp)
2875	{
2876	struct ib_cq *cq = qp->recv_cq;
2877	struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
2878	struct ib_drain_cqe rdrain;
2879	struct ib_recv_wr rwr = {};
2880	int ret;
2881
2882	ret = ib_modify_qp(qp, &attr, IB_QP_STATE);
2883	if (ret) {
2884	WARN_ONCE(ret, "failed to drain recv queue: %d\n", ret);
2885	return;
2886	}
2887
2888	rwr.wr_cqe = &rdrain.cqe;
2889	rdrain.cqe.done = ib_drain_qp_done;
2890	init_completion(x: &rdrain.done);
2891
2892	ret = ib_post_recv(qp, recv_wr: &rwr, NULL);
2893	if (ret) {
2894	WARN_ONCE(ret, "failed to drain recv queue: %d\n", ret);
2895	return;
2896	}
2897
2898	if (cq->poll_ctx == IB_POLL_DIRECT)
2899	while (wait_for_completion_timeout(x: &rdrain.done, HZ / 10) <= 0)
2900	ib_process_cq_direct(cq, budget: -1);
2901	else
2902	wait_for_completion(&rdrain.done);
2903	}
2904
2905	/*
2906	* __ib_drain_srq() - Block until Last WQE Reached event arrives, or timeout
2907	* expires.
2908	* @qp: queue pair associated with SRQ to drain
2909	*
2910	* Quoting 10.3.1 Queue Pair and EE Context States:
2911	*
2912	* Note, for QPs that are associated with an SRQ, the Consumer should take the
2913	* QP through the Error State before invoking a Destroy QP or a Modify QP to the
2914	* Reset State. The Consumer may invoke the Destroy QP without first performing
2915	* a Modify QP to the Error State and waiting for the Affiliated Asynchronous
2916	* Last WQE Reached Event. However, if the Consumer does not wait for the
2917	* Affiliated Asynchronous Last WQE Reached Event, then WQE and Data Segment
2918	* leakage may occur. Therefore, it is good programming practice to tear down a
2919	* QP that is associated with an SRQ by using the following process:
2920	*
2921	* - Put the QP in the Error State
2922	* - Wait for the Affiliated Asynchronous Last WQE Reached Event;
2923	* - either:
2924	* drain the CQ by invoking the Poll CQ verb and either wait for CQ
2925	* to be empty or the number of Poll CQ operations has exceeded
2926	* CQ capacity size;
2927	* - or
2928	* post another WR that completes on the same CQ and wait for this
2929	* WR to return as a WC;
2930	* - and then invoke a Destroy QP or Reset QP.
2931	*
2932	* We use the first option.
2933	*/
2934	static void __ib_drain_srq(struct ib_qp *qp)
2935	{
2936	struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
2937	struct ib_cq *cq;
2938	int n, polled = 0;
2939	int ret;
2940
2941	if (!qp->srq) {
2942	WARN_ONCE(1, "QP 0x%p is not associated with SRQ\n", qp);
2943	return;
2944	}
2945
2946	ret = ib_modify_qp(qp, &attr, IB_QP_STATE);
2947	if (ret) {
2948	WARN_ONCE(ret, "failed to drain shared recv queue: %d\n", ret);
2949	return;
2950	}
2951
2952	if (ib_srq_has_cq(srq_type: qp->srq->srq_type)) {
2953	cq = qp->srq->ext.cq;
2954	} else if (qp->recv_cq) {
2955	cq = qp->recv_cq;
2956	} else {
2957	WARN_ONCE(1, "QP 0x%p has no CQ associated with SRQ\n", qp);
2958	return;
2959	}
2960
2961	if (wait_for_completion_timeout(x: &qp->srq_completion, timeout: 60 * HZ) > 0) {
2962	while (polled != cq->cqe) {
2963	n = ib_process_cq_direct(cq, budget: cq->cqe - polled);
2964	if (!n)
2965	return;
2966	polled += n;
2967	}
2968	}
2969	}
2970
2971	/**
2972	* ib_drain_sq() - Block until all SQ CQEs have been consumed by the
2973	* application.
2974	* @qp: queue pair to drain
2975	*
2976	* If the device has a provider-specific drain function, then
2977	* call that. Otherwise call the generic drain function
2978	* __ib_drain_sq().
2979	*
2980	* The caller must:
2981	*
2982	* ensure there is room in the CQ and SQ for the drain work request and
2983	* completion.
2984	*
2985	* allocate the CQ using ib_alloc_cq().
2986	*
2987	* ensure that there are no other contexts that are posting WRs concurrently.
2988	* Otherwise the drain is not guaranteed.
2989	*/
2990	void ib_drain_sq(struct ib_qp *qp)
2991	{
2992	if (qp->device->ops.drain_sq)
2993	qp->device->ops.drain_sq(qp);
2994	else
2995	__ib_drain_sq(qp);
2996	trace_cq_drain_complete(cq: qp->send_cq);
2997	}
2998	EXPORT_SYMBOL(ib_drain_sq);
2999
3000	/**
3001	* ib_drain_rq() - Block until all RQ CQEs have been consumed by the
3002	* application.
3003	* @qp: queue pair to drain
3004	*
3005	* If the device has a provider-specific drain function, then
3006	* call that. Otherwise call the generic drain function
3007	* __ib_drain_rq().
3008	*
3009	* The caller must:
3010	*
3011	* ensure there is room in the CQ and RQ for the drain work request and
3012	* completion.
3013	*
3014	* allocate the CQ using ib_alloc_cq().
3015	*
3016	* ensure that there are no other contexts that are posting WRs concurrently.
3017	* Otherwise the drain is not guaranteed.
3018	*/
3019	void ib_drain_rq(struct ib_qp *qp)
3020	{
3021	if (qp->device->ops.drain_rq)
3022	qp->device->ops.drain_rq(qp);
3023	else
3024	__ib_drain_rq(qp);
3025	trace_cq_drain_complete(cq: qp->recv_cq);
3026	}
3027	EXPORT_SYMBOL(ib_drain_rq);
3028
3029	/**
3030	* ib_drain_qp() - Block until all CQEs have been consumed by the
3031	* application on both the RQ and SQ.
3032	* @qp: queue pair to drain
3033	*
3034	* The caller must:
3035	*
3036	* ensure there is room in the CQ(s), SQ, and RQ for drain work requests
3037	* and completions.
3038	*
3039	* allocate the CQs using ib_alloc_cq().
3040	*
3041	* ensure that there are no other contexts that are posting WRs concurrently.
3042	* Otherwise the drain is not guaranteed.
3043	*/
3044	void ib_drain_qp(struct ib_qp *qp)
3045	{
3046	ib_drain_sq(qp);
3047	if (!qp->srq)
3048	ib_drain_rq(qp);
3049	else
3050	__ib_drain_srq(qp);
3051	}
3052	EXPORT_SYMBOL(ib_drain_qp);
3053
3054	struct net_device *rdma_alloc_netdev(struct ib_device *device, u32 port_num,
3055	enum rdma_netdev_t type, const char *name,
3056	unsigned char name_assign_type,
3057	void (*setup)(struct net_device *))
3058	{
3059	struct rdma_netdev_alloc_params params;
3060	struct net_device *netdev;
3061	int rc;
3062
3063	if (!device->ops.rdma_netdev_get_params)
3064	return ERR_PTR(error: -EOPNOTSUPP);
3065
3066	rc = device->ops.rdma_netdev_get_params(device, port_num, type,
3067	&params);
3068	if (rc)
3069	return ERR_PTR(error: rc);
3070
3071	netdev = alloc_netdev_mqs(sizeof_priv: params.sizeof_priv, name, name_assign_type,
3072	setup, txqs: params.txqs, rxqs: params.rxqs);
3073	if (!netdev)
3074	return ERR_PTR(error: -ENOMEM);
3075
3076	return netdev;
3077	}
3078	EXPORT_SYMBOL(rdma_alloc_netdev);
3079
3080	int rdma_init_netdev(struct ib_device *device, u32 port_num,
3081	enum rdma_netdev_t type, const char *name,
3082	unsigned char name_assign_type,
3083	void (*setup)(struct net_device *),
3084	struct net_device *netdev)
3085	{
3086	struct rdma_netdev_alloc_params params;
3087	int rc;
3088
3089	if (!device->ops.rdma_netdev_get_params)
3090	return -EOPNOTSUPP;
3091
3092	rc = device->ops.rdma_netdev_get_params(device, port_num, type,
3093	&params);
3094	if (rc)
3095	return rc;
3096
3097	return params.initialize_rdma_netdev(device, port_num,
3098	netdev, params.param);
3099	}
3100	EXPORT_SYMBOL(rdma_init_netdev);
3101
3102	void __rdma_block_iter_start(struct ib_block_iter *biter,
3103	struct scatterlist *sglist, unsigned int nents,
3104	unsigned long pgsz)
3105	{
3106	memset(biter, 0, sizeof(struct ib_block_iter));
3107	biter->__sg = sglist;
3108	biter->__sg_nents = nents;
3109
3110	/* Driver provides best block size to use */
3111	biter->__pg_bit = __fls(word: pgsz);
3112	}
3113	EXPORT_SYMBOL(__rdma_block_iter_start);
3114
3115	bool __rdma_block_iter_next(struct ib_block_iter *biter)
3116	{
3117	unsigned int block_offset;
3118	unsigned int delta;
3119
3120	if (!biter->__sg_nents || !biter->__sg)
3121	return false;
3122
3123	biter->__dma_addr = sg_dma_address(biter->__sg) + biter->__sg_advance;
3124	block_offset = biter->__dma_addr & (BIT_ULL(biter->__pg_bit) - 1);
3125	delta = BIT_ULL(biter->__pg_bit) - block_offset;
3126
3127	while (biter->__sg_nents && biter->__sg &&
3128	sg_dma_len(biter->__sg) - biter->__sg_advance <= delta) {
3129	delta -= sg_dma_len(biter->__sg) - biter->__sg_advance;
3130	biter->__sg_advance = 0;
3131	biter->__sg = sg_next(sg: biter->__sg);
3132	biter->__sg_nents--;
3133	}
3134	biter->__sg_advance += delta;
3135
3136	return true;
3137	}
3138	EXPORT_SYMBOL(__rdma_block_iter_next);
3139
3140	/**
3141	* rdma_alloc_hw_stats_struct - Helper function to allocate dynamic struct
3142	* for the drivers.
3143	* @descs: array of static descriptors
3144	* @num_counters: number of elements in array
3145	* @lifespan: milliseconds between updates
3146	*/
3147	struct rdma_hw_stats *rdma_alloc_hw_stats_struct(
3148	const struct rdma_stat_desc *descs, int num_counters,
3149	unsigned long lifespan)
3150	{
3151	struct rdma_hw_stats *stats;
3152
3153	stats = kzalloc(struct_size(stats, value, num_counters), GFP_KERNEL);
3154	if (!stats)
3155	return NULL;
3156
3157	stats->is_disabled = kcalloc(BITS_TO_LONGS(num_counters),
3158	sizeof(*stats->is_disabled), GFP_KERNEL);
3159	if (!stats->is_disabled)
3160	goto err;
3161
3162	stats->descs = descs;
3163	stats->num_counters = num_counters;
3164	stats->lifespan = msecs_to_jiffies(m: lifespan);
3165	mutex_init(&stats->lock);
3166
3167	return stats;
3168
3169	err:
3170	kfree(objp: stats);
3171	return NULL;
3172	}
3173	EXPORT_SYMBOL(rdma_alloc_hw_stats_struct);
3174
3175	/**
3176	* rdma_free_hw_stats_struct - Helper function to release rdma_hw_stats
3177	* @stats: statistics to release
3178	*/
3179	void rdma_free_hw_stats_struct(struct rdma_hw_stats *stats)
3180	{
3181	if (!stats)
3182	return;
3183
3184	kfree(objp: stats->is_disabled);
3185	kfree(objp: stats);
3186	}
3187	EXPORT_SYMBOL(rdma_free_hw_stats_struct);
3188