1	/*
2	* Copyright (c) 2006 - 2009 Mellanox Technology Inc. All rights reserved.
3	* Copyright (C) 2008 - 2011 Bart Van Assche <bvanassche@acm.org>.
4	*
5	* This software is available to you under a choice of one of two
6	* licenses. You may choose to be licensed under the terms of the GNU
7	* General Public License (GPL) Version 2, available from the file
8	* COPYING in the main directory of this source tree, or the
9	* OpenIB.org BSD license below:
10	*
11	* Redistribution and use in source and binary forms, with or
12	* without modification, are permitted provided that the following
13	* conditions are met:
14	*
15	* - Redistributions of source code must retain the above
16	* copyright notice, this list of conditions and the following
17	* disclaimer.
18	*
19	* - Redistributions in binary form must reproduce the above
20	* copyright notice, this list of conditions and the following
21	* disclaimer in the documentation and/or other materials
22	* provided with the distribution.
23	*
24	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31	* SOFTWARE.
32	*
33	*/
34
35	#include <linux/module.h>
36	#include <linux/init.h>
37	#include <linux/slab.h>
38	#include <linux/err.h>
39	#include <linux/ctype.h>
40	#include <linux/kthread.h>
41	#include <linux/string.h>
42	#include <linux/delay.h>
43	#include <linux/atomic.h>
44	#include <linux/inet.h>
45	#include <rdma/ib_cache.h>
46	#include <scsi/scsi_proto.h>
47	#include <scsi/scsi_tcq.h>
48	#include <target/target_core_base.h>
49	#include <target/target_core_fabric.h>
50	#include "ib_srpt.h"
51
52	/* Name of this kernel module. */
53	#define DRV_NAME "ib_srpt"
54
55	#define SRPT_ID_STRING "Linux SRP target"
56
57	#undef pr_fmt
58	#define pr_fmt(fmt) DRV_NAME " " fmt
59
60	MODULE_AUTHOR("Vu Pham and Bart Van Assche");
61	MODULE_DESCRIPTION("SCSI RDMA Protocol target driver");
62	MODULE_LICENSE("Dual BSD/GPL");
63
64	/*
65	* Global Variables
66	*/
67
68	static u64 srpt_service_guid;
69	static DEFINE_SPINLOCK(srpt_dev_lock); /* Protects srpt_dev_list. */
70	static LIST_HEAD(srpt_dev_list); /* List of srpt_device structures. */
71	static DEFINE_MUTEX(srpt_mc_mutex); /* Protects srpt_memory_caches. */
72	static DEFINE_XARRAY(srpt_memory_caches); /* See also srpt_memory_cache_entry */
73
74	static unsigned srp_max_req_size = DEFAULT_MAX_REQ_SIZE;
75	module_param(srp_max_req_size, int, 0444);
76	MODULE_PARM_DESC(srp_max_req_size,
77	"Maximum size of SRP request messages in bytes.");
78
79	static int srpt_srq_size = DEFAULT_SRPT_SRQ_SIZE;
80	module_param(srpt_srq_size, int, 0444);
81	MODULE_PARM_DESC(srpt_srq_size,
82	"Shared receive queue (SRQ) size.");
83
84	static int srpt_set_u64_x(const char *buffer, const struct kernel_param *kp)
85	{
86	return kstrtou64(s: buffer, base: 16, res: (u64 *)kp->arg);
87	}
88	static int srpt_get_u64_x(char *buffer, const struct kernel_param *kp)
89	{
90	return sprintf(buf: buffer, fmt: "0x%016llx\n", *(u64 *)kp->arg);
91	}
92	module_param_call(srpt_service_guid, srpt_set_u64_x, srpt_get_u64_x,
93	&srpt_service_guid, 0444);
94	MODULE_PARM_DESC(srpt_service_guid,
95	"Using this value for ioc_guid, id_ext, and cm_listen_id instead of using the node_guid of the first HCA.");
96
97	static struct ib_client srpt_client;
98	/* Protects both rdma_cm_port and rdma_cm_id. */
99	static DEFINE_MUTEX(rdma_cm_mutex);
100	/* Port number RDMA/CM will bind to. */
101	static u16 rdma_cm_port;
102	static struct rdma_cm_id *rdma_cm_id;
103	static void srpt_release_cmd(struct se_cmd *se_cmd);
104	static void srpt_free_ch(struct kref *kref);
105	static int srpt_queue_status(struct se_cmd *cmd);
106	static void srpt_recv_done(struct ib_cq *cq, struct ib_wc *wc);
107	static void srpt_send_done(struct ib_cq *cq, struct ib_wc *wc);
108	static void srpt_process_wait_list(struct srpt_rdma_ch *ch);
109
110	/* Type of the entries in srpt_memory_caches. */
111	struct srpt_memory_cache_entry {
112	refcount_t ref;
113	struct kmem_cache *c;
114	};
115
116	static struct kmem_cache *srpt_cache_get(unsigned int object_size)
117	{
118	struct srpt_memory_cache_entry *e;
119	char name[32];
120	void *res;
121
122	guard(mutex)(T: &srpt_mc_mutex);
123	e = xa_load(&srpt_memory_caches, index: object_size);
124	if (e) {
125	refcount_inc(r: &e->ref);
126	return e->c;
127	}
128	snprintf(buf: name, size: sizeof(name), fmt: "srpt-%u", object_size);
129	e = kmalloc(sizeof(*e), GFP_KERNEL);
130	if (!e)
131	return NULL;
132	refcount_set(r: &e->ref, n: 1);
133	e->c = kmem_cache_create(name, object_size, /*align=*/512, 0, NULL);
134	if (!e->c)
135	goto free_entry;
136	res = xa_store(&srpt_memory_caches, index: object_size, entry: e, GFP_KERNEL);
137	if (xa_is_err(entry: res))
138	goto destroy_cache;
139	return e->c;
140
141	destroy_cache:
142	kmem_cache_destroy(s: e->c);
143
144	free_entry:
145	kfree(objp: e);
146	return NULL;
147	}
148
149	static void srpt_cache_put(struct kmem_cache *c)
150	{
151	struct srpt_memory_cache_entry *e = NULL;
152	unsigned long object_size;
153
154	guard(mutex)(T: &srpt_mc_mutex);
155	xa_for_each(&srpt_memory_caches, object_size, e)
156	if (e->c == c)
157	break;
158	if (WARN_ON_ONCE(!e))
159	return;
160	if (!refcount_dec_and_test(r: &e->ref))
161	return;
162	WARN_ON_ONCE(xa_erase(&srpt_memory_caches, object_size) != e);
163	kmem_cache_destroy(s: e->c);
164	kfree(objp: e);
165	}
166
167	/*
168	* The only allowed channel state changes are those that change the channel
169	* state into a state with a higher numerical value. Hence the new > prev test.
170	*/
171	static bool srpt_set_ch_state(struct srpt_rdma_ch *ch, enum rdma_ch_state new)
172	{
173	unsigned long flags;
174	enum rdma_ch_state prev;
175	bool changed = false;
176
177	spin_lock_irqsave(&ch->spinlock, flags);
178	prev = ch->state;
179	if (new > prev) {
180	ch->state = new;
181	changed = true;
182	}
183	spin_unlock_irqrestore(lock: &ch->spinlock, flags);
184
185	return changed;
186	}
187
188	/**
189	* srpt_event_handler - asynchronous IB event callback function
190	* @handler: IB event handler registered by ib_register_event_handler().
191	* @event: Description of the event that occurred.
192	*
193	* Callback function called by the InfiniBand core when an asynchronous IB
194	* event occurs. This callback may occur in interrupt context. See also
195	* section 11.5.2, Set Asynchronous Event Handler in the InfiniBand
196	* Architecture Specification.
197	*/
198	static void srpt_event_handler(struct ib_event_handler *handler,
199	struct ib_event *event)
200	{
201	struct srpt_device *sdev =
202	container_of(handler, struct srpt_device, event_handler);
203	struct srpt_port *sport;
204	u8 port_num;
205
206	pr_debug("ASYNC event= %d on device= %s\n", event->event,
207	dev_name(&sdev->device->dev));
208
209	switch (event->event) {
210	case IB_EVENT_PORT_ERR:
211	port_num = event->element.port_num - 1;
212	if (port_num < sdev->device->phys_port_cnt) {
213	sport = &sdev->port[port_num];
214	sport->lid = 0;
215	sport->sm_lid = 0;
216	} else {
217	WARN(true, "event %d: port_num %d out of range 1..%d\n",
218	event->event, port_num + 1,
219	sdev->device->phys_port_cnt);
220	}
221	break;
222	case IB_EVENT_PORT_ACTIVE:
223	case IB_EVENT_LID_CHANGE:
224	case IB_EVENT_PKEY_CHANGE:
225	case IB_EVENT_SM_CHANGE:
226	case IB_EVENT_CLIENT_REREGISTER:
227	case IB_EVENT_GID_CHANGE:
228	/* Refresh port data asynchronously. */
229	port_num = event->element.port_num - 1;
230	if (port_num < sdev->device->phys_port_cnt) {
231	sport = &sdev->port[port_num];
232	if (!sport->lid && !sport->sm_lid)
233	schedule_work(work: &sport->work);
234	} else {
235	WARN(true, "event %d: port_num %d out of range 1..%d\n",
236	event->event, port_num + 1,
237	sdev->device->phys_port_cnt);
238	}
239	break;
240	default:
241	pr_err("received unrecognized IB event %d\n", event->event);
242	break;
243	}
244	}
245
246	/**
247	* srpt_srq_event - SRQ event callback function
248	* @event: Description of the event that occurred.
249	* @ctx: Context pointer specified at SRQ creation time.
250	*/
251	static void srpt_srq_event(struct ib_event *event, void *ctx)
252	{
253	pr_debug("SRQ event %d\n", event->event);
254	}
255
256	static const char *get_ch_state_name(enum rdma_ch_state s)
257	{
258	switch (s) {
259	case CH_CONNECTING:
260	return "connecting";
261	case CH_LIVE:
262	return "live";
263	case CH_DISCONNECTING:
264	return "disconnecting";
265	case CH_DRAINING:
266	return "draining";
267	case CH_DISCONNECTED:
268	return "disconnected";
269	}
270	return "???";
271	}
272
273	/**
274	* srpt_qp_event - QP event callback function
275	* @event: Description of the event that occurred.
276	* @ptr: SRPT RDMA channel.
277	*/
278	static void srpt_qp_event(struct ib_event *event, void *ptr)
279	{
280	struct srpt_rdma_ch *ch = ptr;
281
282	pr_debug("QP event %d on ch=%p sess_name=%s-%d state=%s\n",
283	event->event, ch, ch->sess_name, ch->qp->qp_num,
284	get_ch_state_name(ch->state));
285
286	switch (event->event) {
287	case IB_EVENT_COMM_EST:
288	if (ch->using_rdma_cm)
289	rdma_notify(id: ch->rdma_cm.cm_id, event: event->event);
290	else
291	ib_cm_notify(cm_id: ch->ib_cm.cm_id, event: event->event);
292	break;
293	case IB_EVENT_QP_LAST_WQE_REACHED:
294	pr_debug("%s-%d, state %s: received Last WQE event.\n",
295	ch->sess_name, ch->qp->qp_num,
296	get_ch_state_name(ch->state));
297	break;
298	default:
299	pr_err("received unrecognized IB QP event %d\n", event->event);
300	break;
301	}
302	}
303
304	/**
305	* srpt_set_ioc - initialize a IOUnitInfo structure
306	* @c_list: controller list.
307	* @slot: one-based slot number.
308	* @value: four-bit value.
309	*
310	* Copies the lowest four bits of value in element slot of the array of four
311	* bit elements called c_list (controller list). The index slot is one-based.
312	*/
313	static void srpt_set_ioc(u8 *c_list, u32 slot, u8 value)
314	{
315	u16 id;
316	u8 tmp;
317
318	id = (slot - 1) / 2;
319	if (slot & 0x1) {
320	tmp = c_list[id] & 0xf;
321	c_list[id] = (value << 4) | tmp;
322	} else {
323	tmp = c_list[id] & 0xf0;
324	c_list[id] = (value & 0xf) | tmp;
325	}
326	}
327
328	/**
329	* srpt_get_class_port_info - copy ClassPortInfo to a management datagram
330	* @mad: Datagram that will be sent as response to DM_ATTR_CLASS_PORT_INFO.
331	*
332	* See also section 16.3.3.1 ClassPortInfo in the InfiniBand Architecture
333	* Specification.
334	*/
335	static void srpt_get_class_port_info(struct ib_dm_mad *mad)
336	{
337	struct ib_class_port_info *cif;
338
339	cif = (struct ib_class_port_info *)mad->data;
340	memset(cif, 0, sizeof(*cif));
341	cif->base_version = 1;
342	cif->class_version = 1;
343
344	ib_set_cpi_resp_time(cpi: cif, rtime: 20);
345	mad->mad_hdr.status = 0;
346	}
347
348	/**
349	* srpt_get_iou - write IOUnitInfo to a management datagram
350	* @mad: Datagram that will be sent as response to DM_ATTR_IOU_INFO.
351	*
352	* See also section 16.3.3.3 IOUnitInfo in the InfiniBand Architecture
353	* Specification. See also section B.7, table B.6 in the SRP r16a document.
354	*/
355	static void srpt_get_iou(struct ib_dm_mad *mad)
356	{
357	struct ib_dm_iou_info *ioui;
358	u8 slot;
359	int i;
360
361	ioui = (struct ib_dm_iou_info *)mad->data;
362	ioui->change_id = cpu_to_be16(1);
363	ioui->max_controllers = 16;
364
365	/* set present for slot 1 and empty for the rest */
366	srpt_set_ioc(c_list: ioui->controller_list, slot: 1, value: 1);
367	for (i = 1, slot = 2; i < 16; i++, slot++)
368	srpt_set_ioc(c_list: ioui->controller_list, slot, value: 0);
369
370	mad->mad_hdr.status = 0;
371	}
372
373	/**
374	* srpt_get_ioc - write IOControllerprofile to a management datagram
375	* @sport: HCA port through which the MAD has been received.
376	* @slot: Slot number specified in DM_ATTR_IOC_PROFILE query.
377	* @mad: Datagram that will be sent as response to DM_ATTR_IOC_PROFILE.
378	*
379	* See also section 16.3.3.4 IOControllerProfile in the InfiniBand
380	* Architecture Specification. See also section B.7, table B.7 in the SRP
381	* r16a document.
382	*/
383	static void srpt_get_ioc(struct srpt_port *sport, u32 slot,
384	struct ib_dm_mad *mad)
385	{
386	struct srpt_device *sdev = sport->sdev;
387	struct ib_dm_ioc_profile *iocp;
388	int send_queue_depth;
389
390	iocp = (struct ib_dm_ioc_profile *)mad->data;
391
392	if (!slot || slot > 16) {
393	mad->mad_hdr.status
394	= cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
395	return;
396	}
397
398	if (slot > 2) {
399	mad->mad_hdr.status
400	= cpu_to_be16(DM_MAD_STATUS_NO_IOC);
401	return;
402	}
403
404	if (sdev->use_srq)
405	send_queue_depth = sdev->srq_size;
406	else
407	send_queue_depth = min(MAX_SRPT_RQ_SIZE,
408	sdev->device->attrs.max_qp_wr);
409
410	memset(iocp, 0, sizeof(*iocp));
411	strcpy(p: iocp->id_string, SRPT_ID_STRING);
412	iocp->guid = cpu_to_be64(srpt_service_guid);
413	iocp->vendor_id = cpu_to_be32(sdev->device->attrs.vendor_id);
414	iocp->device_id = cpu_to_be32(sdev->device->attrs.vendor_part_id);
415	iocp->device_version = cpu_to_be16(sdev->device->attrs.hw_ver);
416	iocp->subsys_vendor_id = cpu_to_be32(sdev->device->attrs.vendor_id);
417	iocp->subsys_device_id = 0x0;
418	iocp->io_class = cpu_to_be16(SRP_REV16A_IB_IO_CLASS);
419	iocp->io_subclass = cpu_to_be16(SRP_IO_SUBCLASS);
420	iocp->protocol = cpu_to_be16(SRP_PROTOCOL);
421	iocp->protocol_version = cpu_to_be16(SRP_PROTOCOL_VERSION);
422	iocp->send_queue_depth = cpu_to_be16(send_queue_depth);
423	iocp->rdma_read_depth = 4;
424	iocp->send_size = cpu_to_be32(srp_max_req_size);
425	iocp->rdma_size = cpu_to_be32(min(sport->port_attrib.srp_max_rdma_size,
426	1U << 24));
427	iocp->num_svc_entries = 1;
428	iocp->op_cap_mask = SRP_SEND_TO_IOC | SRP_SEND_FROM_IOC |
429	SRP_RDMA_READ_FROM_IOC | SRP_RDMA_WRITE_FROM_IOC;
430
431	mad->mad_hdr.status = 0;
432	}
433
434	/**
435	* srpt_get_svc_entries - write ServiceEntries to a management datagram
436	* @ioc_guid: I/O controller GUID to use in reply.
437	* @slot: I/O controller number.
438	* @hi: End of the range of service entries to be specified in the reply.
439	* @lo: Start of the range of service entries to be specified in the reply..
440	* @mad: Datagram that will be sent as response to DM_ATTR_SVC_ENTRIES.
441	*
442	* See also section 16.3.3.5 ServiceEntries in the InfiniBand Architecture
443	* Specification. See also section B.7, table B.8 in the SRP r16a document.
444	*/
445	static void srpt_get_svc_entries(u64 ioc_guid,
446	u16 slot, u8 hi, u8 lo, struct ib_dm_mad *mad)
447	{
448	struct ib_dm_svc_entries *svc_entries;
449
450	WARN_ON(!ioc_guid);
451
452	if (!slot || slot > 16) {
453	mad->mad_hdr.status
454	= cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
455	return;
456	}
457
458	if (slot > 2 || lo > hi || hi > 1) {
459	mad->mad_hdr.status
460	= cpu_to_be16(DM_MAD_STATUS_NO_IOC);
461	return;
462	}
463
464	svc_entries = (struct ib_dm_svc_entries *)mad->data;
465	memset(svc_entries, 0, sizeof(*svc_entries));
466	svc_entries->service_entries[0].id = cpu_to_be64(ioc_guid);
467	snprintf(buf: svc_entries->service_entries[0].name,
468	size: sizeof(svc_entries->service_entries[0].name),
469	fmt: "%s%016llx",
470	SRP_SERVICE_NAME_PREFIX,
471	ioc_guid);
472
473	mad->mad_hdr.status = 0;
474	}
475
476	/**
477	* srpt_mgmt_method_get - process a received management datagram
478	* @sp: HCA port through which the MAD has been received.
479	* @rq_mad: received MAD.
480	* @rsp_mad: response MAD.
481	*/
482	static void srpt_mgmt_method_get(struct srpt_port *sp, struct ib_mad *rq_mad,
483	struct ib_dm_mad *rsp_mad)
484	{
485	u16 attr_id;
486	u32 slot;
487	u8 hi, lo;
488
489	attr_id = be16_to_cpu(rq_mad->mad_hdr.attr_id);
490	switch (attr_id) {
491	case DM_ATTR_CLASS_PORT_INFO:
492	srpt_get_class_port_info(mad: rsp_mad);
493	break;
494	case DM_ATTR_IOU_INFO:
495	srpt_get_iou(mad: rsp_mad);
496	break;
497	case DM_ATTR_IOC_PROFILE:
498	slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
499	srpt_get_ioc(sport: sp, slot, mad: rsp_mad);
500	break;
501	case DM_ATTR_SVC_ENTRIES:
502	slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
503	hi = (u8) ((slot >> 8) & 0xff);
504	lo = (u8) (slot & 0xff);
505	slot = (u16) ((slot >> 16) & 0xffff);
506	srpt_get_svc_entries(ioc_guid: srpt_service_guid,
507	slot, hi, lo, mad: rsp_mad);
508	break;
509	default:
510	rsp_mad->mad_hdr.status =
511	cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
512	break;
513	}
514	}
515
516	/**
517	* srpt_mad_send_handler - MAD send completion callback
518	* @mad_agent: Return value of ib_register_mad_agent().
519	* @mad_wc: Work completion reporting that the MAD has been sent.
520	*/
521	static void srpt_mad_send_handler(struct ib_mad_agent *mad_agent,
522	struct ib_mad_send_wc *mad_wc)
523	{
524	rdma_destroy_ah(ah: mad_wc->send_buf->ah, flags: RDMA_DESTROY_AH_SLEEPABLE);
525	ib_free_send_mad(send_buf: mad_wc->send_buf);
526	}
527
528	/**
529	* srpt_mad_recv_handler - MAD reception callback function
530	* @mad_agent: Return value of ib_register_mad_agent().
531	* @send_buf: Not used.
532	* @mad_wc: Work completion reporting that a MAD has been received.
533	*/
534	static void srpt_mad_recv_handler(struct ib_mad_agent *mad_agent,
535	struct ib_mad_send_buf *send_buf,
536	struct ib_mad_recv_wc *mad_wc)
537	{
538	struct srpt_port *sport = (struct srpt_port *)mad_agent->context;
539	struct ib_ah *ah;
540	struct ib_mad_send_buf *rsp;
541	struct ib_dm_mad *dm_mad;
542
543	if (!mad_wc || !mad_wc->recv_buf.mad)
544	return;
545
546	ah = ib_create_ah_from_wc(pd: mad_agent->qp->pd, wc: mad_wc->wc,
547	grh: mad_wc->recv_buf.grh, port_num: mad_agent->port_num);
548	if (IS_ERR(ptr: ah))
549	goto err;
550
551	BUILD_BUG_ON(offsetof(struct ib_dm_mad, data) != IB_MGMT_DEVICE_HDR);
552
553	rsp = ib_create_send_mad(mad_agent, remote_qpn: mad_wc->wc->src_qp,
554	pkey_index: mad_wc->wc->pkey_index, rmpp_active: 0,
555	hdr_len: IB_MGMT_DEVICE_HDR, data_len: IB_MGMT_DEVICE_DATA,
556	GFP_KERNEL,
557	IB_MGMT_BASE_VERSION);
558	if (IS_ERR(ptr: rsp))
559	goto err_rsp;
560
561	rsp->ah = ah;
562
563	dm_mad = rsp->mad;
564	memcpy(dm_mad, mad_wc->recv_buf.mad, sizeof(*dm_mad));
565	dm_mad->mad_hdr.method = IB_MGMT_METHOD_GET_RESP;
566	dm_mad->mad_hdr.status = 0;
567
568	switch (mad_wc->recv_buf.mad->mad_hdr.method) {
569	case IB_MGMT_METHOD_GET:
570	srpt_mgmt_method_get(sp: sport, rq_mad: mad_wc->recv_buf.mad, rsp_mad: dm_mad);
571	break;
572	case IB_MGMT_METHOD_SET:
573	dm_mad->mad_hdr.status =
574	cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
575	break;
576	default:
577	dm_mad->mad_hdr.status =
578	cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD);
579	break;
580	}
581
582	if (!ib_post_send_mad(send_buf: rsp, NULL)) {
583	ib_free_recv_mad(mad_recv_wc: mad_wc);
584	/* will destroy_ah & free_send_mad in send completion */
585	return;
586	}
587
588	ib_free_send_mad(send_buf: rsp);
589
590	err_rsp:
591	rdma_destroy_ah(ah, flags: RDMA_DESTROY_AH_SLEEPABLE);
592	err:
593	ib_free_recv_mad(mad_recv_wc: mad_wc);
594	}
595
596	static int srpt_format_guid(char *buf, unsigned int size, const __be64 *guid)
597	{
598	const __be16 *g = (const __be16 *)guid;
599
600	return snprintf(buf, size, fmt: "%04x:%04x:%04x:%04x",
601	be16_to_cpu(g[0]), be16_to_cpu(g[1]),
602	be16_to_cpu(g[2]), be16_to_cpu(g[3]));
603	}
604
605	/**
606	* srpt_refresh_port - configure a HCA port
607	* @sport: SRPT HCA port.
608	*
609	* Enable InfiniBand management datagram processing, update the cached sm_lid,
610	* lid and gid values, and register a callback function for processing MADs
611	* on the specified port.
612	*
613	* Note: It is safe to call this function more than once for the same port.
614	*/
615	static int srpt_refresh_port(struct srpt_port *sport)
616	{
617	struct ib_mad_agent *mad_agent;
618	struct ib_mad_reg_req reg_req;
619	struct ib_port_modify port_modify;
620	struct ib_port_attr port_attr;
621	int ret;
622
623	ret = ib_query_port(device: sport->sdev->device, port_num: sport->port, port_attr: &port_attr);
624	if (ret)
625	return ret;
626
627	sport->sm_lid = port_attr.sm_lid;
628	sport->lid = port_attr.lid;
629
630	ret = rdma_query_gid(device: sport->sdev->device, port_num: sport->port, index: 0, gid: &sport->gid);
631	if (ret)
632	return ret;
633
634	srpt_format_guid(buf: sport->guid_name, ARRAY_SIZE(sport->guid_name),
635	guid: &sport->gid.global.interface_id);
636	snprintf(buf: sport->gid_name, ARRAY_SIZE(sport->gid_name),
637	fmt: "0x%016llx%016llx",
638	be64_to_cpu(sport->gid.global.subnet_prefix),
639	be64_to_cpu(sport->gid.global.interface_id));
640
641	if (rdma_protocol_iwarp(device: sport->sdev->device, port_num: sport->port))
642	return 0;
643
644	memset(&port_modify, 0, sizeof(port_modify));
645	port_modify.set_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
646	port_modify.clr_port_cap_mask = 0;
647
648	ret = ib_modify_port(device: sport->sdev->device, port_num: sport->port, port_modify_mask: 0, port_modify: &port_modify);
649	if (ret) {
650	pr_warn("%s-%d: enabling device management failed (%d). Note: this is expected if SR-IOV is enabled.\n",
651	dev_name(&sport->sdev->device->dev), sport->port, ret);
652	return 0;
653	}
654
655	if (!sport->mad_agent) {
656	memset(&reg_req, 0, sizeof(reg_req));
657	reg_req.mgmt_class = IB_MGMT_CLASS_DEVICE_MGMT;
658	reg_req.mgmt_class_version = IB_MGMT_BASE_VERSION;
659	set_bit(IB_MGMT_METHOD_GET, addr: reg_req.method_mask);
660	set_bit(IB_MGMT_METHOD_SET, addr: reg_req.method_mask);
661
662	mad_agent = ib_register_mad_agent(device: sport->sdev->device,
663	port_num: sport->port,
664	qp_type: IB_QPT_GSI,
665	mad_reg_req: &reg_req, rmpp_version: 0,
666	send_handler: srpt_mad_send_handler,
667	recv_handler: srpt_mad_recv_handler,
668	context: sport, registration_flags: 0);
669	if (IS_ERR(ptr: mad_agent)) {
670	pr_err("%s-%d: MAD agent registration failed (%pe). Note: this is expected if SR-IOV is enabled.\n",
671	dev_name(&sport->sdev->device->dev), sport->port,
672	mad_agent);
673	sport->mad_agent = NULL;
674	memset(&port_modify, 0, sizeof(port_modify));
675	port_modify.clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
676	ib_modify_port(device: sport->sdev->device, port_num: sport->port, port_modify_mask: 0,
677	port_modify: &port_modify);
678	return 0;
679	}
680
681	sport->mad_agent = mad_agent;
682	}
683
684	return 0;
685	}
686
687	/**
688	* srpt_unregister_mad_agent - unregister MAD callback functions
689	* @sdev: SRPT HCA pointer.
690	* @port_cnt: number of ports with registered MAD
691	*
692	* Note: It is safe to call this function more than once for the same device.
693	*/
694	static void srpt_unregister_mad_agent(struct srpt_device *sdev, int port_cnt)
695	{
696	struct ib_port_modify port_modify = {
697	.clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP,
698	};
699	struct srpt_port *sport;
700	int i;
701
702	for (i = 1; i <= port_cnt; i++) {
703	sport = &sdev->port[i - 1];
704	WARN_ON(sport->port != i);
705	if (sport->mad_agent) {
706	ib_modify_port(device: sdev->device, port_num: i, port_modify_mask: 0, port_modify: &port_modify);
707	ib_unregister_mad_agent(mad_agent: sport->mad_agent);
708	sport->mad_agent = NULL;
709	}
710	}
711	}
712
713	/**
714	* srpt_alloc_ioctx - allocate a SRPT I/O context structure
715	* @sdev: SRPT HCA pointer.
716	* @ioctx_size: I/O context size.
717	* @buf_cache: I/O buffer cache.
718	* @dir: DMA data direction.
719	*/
720	static struct srpt_ioctx *srpt_alloc_ioctx(struct srpt_device *sdev,
721	int ioctx_size,
722	struct kmem_cache *buf_cache,
723	enum dma_data_direction dir)
724	{
725	struct srpt_ioctx *ioctx;
726
727	ioctx = kzalloc(ioctx_size, GFP_KERNEL);
728	if (!ioctx)
729	goto err;
730
731	ioctx->buf = kmem_cache_alloc(buf_cache, GFP_KERNEL);
732	if (!ioctx->buf)
733	goto err_free_ioctx;
734
735	ioctx->dma = ib_dma_map_single(dev: sdev->device, cpu_addr: ioctx->buf,
736	size: kmem_cache_size(s: buf_cache), direction: dir);
737	if (ib_dma_mapping_error(dev: sdev->device, dma_addr: ioctx->dma))
738	goto err_free_buf;
739
740	return ioctx;
741
742	err_free_buf:
743	kmem_cache_free(s: buf_cache, objp: ioctx->buf);
744	err_free_ioctx:
745	kfree(objp: ioctx);
746	err:
747	return NULL;
748	}
749
750	/**
751	* srpt_free_ioctx - free a SRPT I/O context structure
752	* @sdev: SRPT HCA pointer.
753	* @ioctx: I/O context pointer.
754	* @buf_cache: I/O buffer cache.
755	* @dir: DMA data direction.
756	*/
757	static void srpt_free_ioctx(struct srpt_device *sdev, struct srpt_ioctx *ioctx,
758	struct kmem_cache *buf_cache,
759	enum dma_data_direction dir)
760	{
761	if (!ioctx)
762	return;
763
764	ib_dma_unmap_single(dev: sdev->device, addr: ioctx->dma,
765	size: kmem_cache_size(s: buf_cache), direction: dir);
766	kmem_cache_free(s: buf_cache, objp: ioctx->buf);
767	kfree(objp: ioctx);
768	}
769
770	/**
771	* srpt_alloc_ioctx_ring - allocate a ring of SRPT I/O context structures
772	* @sdev: Device to allocate the I/O context ring for.
773	* @ring_size: Number of elements in the I/O context ring.
774	* @ioctx_size: I/O context size.
775	* @buf_cache: I/O buffer cache.
776	* @alignment_offset: Offset in each ring buffer at which the SRP information
777	* unit starts.
778	* @dir: DMA data direction.
779	*/
780	static struct srpt_ioctx **srpt_alloc_ioctx_ring(struct srpt_device *sdev,
781	int ring_size, int ioctx_size,
782	struct kmem_cache *buf_cache,
783	int alignment_offset,
784	enum dma_data_direction dir)
785	{
786	struct srpt_ioctx **ring;
787	int i;
788
789	WARN_ON(ioctx_size != sizeof(struct srpt_recv_ioctx) &&
790	ioctx_size != sizeof(struct srpt_send_ioctx));
791
792	ring = kvmalloc_array(ring_size, sizeof(ring[0]), GFP_KERNEL);
793	if (!ring)
794	goto out;
795	for (i = 0; i < ring_size; ++i) {
796	ring[i] = srpt_alloc_ioctx(sdev, ioctx_size, buf_cache, dir);
797	if (!ring[i])
798	goto err;
799	ring[i]->index = i;
800	ring[i]->offset = alignment_offset;
801	}
802	goto out;
803
804	err:
805	while (--i >= 0)
806	srpt_free_ioctx(sdev, ioctx: ring[i], buf_cache, dir);
807	kvfree(addr: ring);
808	ring = NULL;
809	out:
810	return ring;
811	}
812
813	/**
814	* srpt_free_ioctx_ring - free the ring of SRPT I/O context structures
815	* @ioctx_ring: I/O context ring to be freed.
816	* @sdev: SRPT HCA pointer.
817	* @ring_size: Number of ring elements.
818	* @buf_cache: I/O buffer cache.
819	* @dir: DMA data direction.
820	*/
821	static void srpt_free_ioctx_ring(struct srpt_ioctx **ioctx_ring,
822	struct srpt_device *sdev, int ring_size,
823	struct kmem_cache *buf_cache,
824	enum dma_data_direction dir)
825	{
826	int i;
827
828	if (!ioctx_ring)
829	return;
830
831	for (i = 0; i < ring_size; ++i)
832	srpt_free_ioctx(sdev, ioctx: ioctx_ring[i], buf_cache, dir);
833	kvfree(addr: ioctx_ring);
834	}
835
836	/**
837	* srpt_set_cmd_state - set the state of a SCSI command
838	* @ioctx: Send I/O context.
839	* @new: New I/O context state.
840	*
841	* Does not modify the state of aborted commands. Returns the previous command
842	* state.
843	*/
844	static enum srpt_command_state srpt_set_cmd_state(struct srpt_send_ioctx *ioctx,
845	enum srpt_command_state new)
846	{
847	enum srpt_command_state previous;
848
849	previous = ioctx->state;
850	if (previous != SRPT_STATE_DONE)
851	ioctx->state = new;
852
853	return previous;
854	}
855
856	/**
857	* srpt_test_and_set_cmd_state - test and set the state of a command
858	* @ioctx: Send I/O context.
859	* @old: Current I/O context state.
860	* @new: New I/O context state.
861	*
862	* Returns true if and only if the previous command state was equal to 'old'.
863	*/
864	static bool srpt_test_and_set_cmd_state(struct srpt_send_ioctx *ioctx,
865	enum srpt_command_state old,
866	enum srpt_command_state new)
867	{
868	enum srpt_command_state previous;
869
870	WARN_ON(!ioctx);
871	WARN_ON(old == SRPT_STATE_DONE);
872	WARN_ON(new == SRPT_STATE_NEW);
873
874	previous = ioctx->state;
875	if (previous == old)
876	ioctx->state = new;
877
878	return previous == old;
879	}
880
881	/**
882	* srpt_post_recv - post an IB receive request
883	* @sdev: SRPT HCA pointer.
884	* @ch: SRPT RDMA channel.
885	* @ioctx: Receive I/O context pointer.
886	*/
887	static int srpt_post_recv(struct srpt_device *sdev, struct srpt_rdma_ch *ch,
888	struct srpt_recv_ioctx *ioctx)
889	{
890	struct ib_sge list;
891	struct ib_recv_wr wr;
892
893	BUG_ON(!sdev);
894	list.addr = ioctx->ioctx.dma + ioctx->ioctx.offset;
895	list.length = srp_max_req_size;
896	list.lkey = sdev->lkey;
897
898	ioctx->ioctx.cqe.done = srpt_recv_done;
899	wr.wr_cqe = &ioctx->ioctx.cqe;
900	wr.next = NULL;
901	wr.sg_list = &list;
902	wr.num_sge = 1;
903
904	if (sdev->use_srq)
905	return ib_post_srq_recv(srq: sdev->srq, recv_wr: &wr, NULL);
906	else
907	return ib_post_recv(qp: ch->qp, recv_wr: &wr, NULL);
908	}
909
910	/**
911	* srpt_zerolength_write - perform a zero-length RDMA write
912	* @ch: SRPT RDMA channel.
913	*
914	* A quote from the InfiniBand specification: C9-88: For an HCA responder
915	* using Reliable Connection service, for each zero-length RDMA READ or WRITE
916	* request, the R_Key shall not be validated, even if the request includes
917	* Immediate data.
918	*/
919	static int srpt_zerolength_write(struct srpt_rdma_ch *ch)
920	{
921	struct ib_rdma_wr wr = {
922	.wr = {
923	.next = NULL,
924	{ .wr_cqe = &ch->zw_cqe, },
925	.opcode = IB_WR_RDMA_WRITE,
926	.send_flags = IB_SEND_SIGNALED,
927	}
928	};
929
930	pr_debug("%s-%d: queued zerolength write\n", ch->sess_name,
931	ch->qp->qp_num);
932
933	return ib_post_send(qp: ch->qp, send_wr: &wr.wr, NULL);
934	}
935
936	static void srpt_zerolength_write_done(struct ib_cq *cq, struct ib_wc *wc)
937	{
938	struct srpt_rdma_ch *ch = wc->qp->qp_context;
939
940	pr_debug("%s-%d wc->status %d\n", ch->sess_name, ch->qp->qp_num,
941	wc->status);
942
943	if (wc->status == IB_WC_SUCCESS) {
944	srpt_process_wait_list(ch);
945	} else {
946	if (srpt_set_ch_state(ch, new: CH_DISCONNECTED))
947	schedule_work(work: &ch->release_work);
948	else
949	pr_debug("%s-%d: already disconnected.\n",
950	ch->sess_name, ch->qp->qp_num);
951	}
952	}
953
954	static int srpt_alloc_rw_ctxs(struct srpt_send_ioctx *ioctx,
955	struct srp_direct_buf *db, int nbufs, struct scatterlist **sg,
956	unsigned *sg_cnt)
957	{
958	enum dma_data_direction dir = target_reverse_dma_direction(se_cmd: &ioctx->cmd);
959	struct srpt_rdma_ch *ch = ioctx->ch;
960	struct scatterlist *prev = NULL;
961	unsigned prev_nents;
962	int ret, i;
963
964	if (nbufs == 1) {
965	ioctx->rw_ctxs = &ioctx->s_rw_ctx;
966	} else {
967	ioctx->rw_ctxs = kmalloc_array(nbufs, sizeof(*ioctx->rw_ctxs),
968	GFP_KERNEL);
969	if (!ioctx->rw_ctxs)
970	return -ENOMEM;
971	}
972
973	for (i = ioctx->n_rw_ctx; i < nbufs; i++, db++) {
974	struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i];
975	u64 remote_addr = be64_to_cpu(db->va);
976	u32 size = be32_to_cpu(db->len);
977	u32 rkey = be32_to_cpu(db->key);
978
979	ret = target_alloc_sgl(sgl: &ctx->sg, nents: &ctx->nents, length: size, zero_page: false,
980	chainable: i < nbufs - 1);
981	if (ret)
982	goto unwind;
983
984	ret = rdma_rw_ctx_init(ctx: &ctx->rw, qp: ch->qp, port_num: ch->sport->port,
985	sg: ctx->sg, sg_cnt: ctx->nents, sg_offset: 0, remote_addr, rkey, dir);
986	if (ret < 0) {
987	target_free_sgl(sgl: ctx->sg, nents: ctx->nents);
988	goto unwind;
989	}
990
991	ioctx->n_rdma += ret;
992	ioctx->n_rw_ctx++;
993
994	if (prev) {
995	sg_unmark_end(sg: &prev[prev_nents - 1]);
996	sg_chain(prv: prev, prv_nents: prev_nents + 1, sgl: ctx->sg);
997	} else {
998	*sg = ctx->sg;
999	}
1000
1001	prev = ctx->sg;
1002	prev_nents = ctx->nents;
1003
1004	*sg_cnt += ctx->nents;
1005	}
1006
1007	return 0;
1008
1009	unwind:
1010	while (--i >= 0) {
1011	struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i];
1012
1013	rdma_rw_ctx_destroy(ctx: &ctx->rw, qp: ch->qp, port_num: ch->sport->port,
1014	sg: ctx->sg, sg_cnt: ctx->nents, dir);
1015	target_free_sgl(sgl: ctx->sg, nents: ctx->nents);
1016	}
1017	if (ioctx->rw_ctxs != &ioctx->s_rw_ctx)
1018	kfree(objp: ioctx->rw_ctxs);
1019	return ret;
1020	}
1021
1022	static void srpt_free_rw_ctxs(struct srpt_rdma_ch *ch,
1023	struct srpt_send_ioctx *ioctx)
1024	{
1025	enum dma_data_direction dir = target_reverse_dma_direction(se_cmd: &ioctx->cmd);
1026	int i;
1027
1028	for (i = 0; i < ioctx->n_rw_ctx; i++) {
1029	struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i];
1030
1031	rdma_rw_ctx_destroy(ctx: &ctx->rw, qp: ch->qp, port_num: ch->sport->port,
1032	sg: ctx->sg, sg_cnt: ctx->nents, dir);
1033	target_free_sgl(sgl: ctx->sg, nents: ctx->nents);
1034	}
1035
1036	if (ioctx->rw_ctxs != &ioctx->s_rw_ctx)
1037	kfree(objp: ioctx->rw_ctxs);
1038	}
1039
1040	static inline void *srpt_get_desc_buf(struct srp_cmd *srp_cmd)
1041	{
1042	/*
1043	* The pointer computations below will only be compiled correctly
1044	* if srp_cmd::add_data is declared as s8*, u8*, s8[] or u8[], so check
1045	* whether srp_cmd::add_data has been declared as a byte pointer.
1046	*/
1047	BUILD_BUG_ON(!__same_type(srp_cmd->add_data[0], (s8)0) &&
1048	!__same_type(srp_cmd->add_data[0], (u8)0));
1049
1050	/*
1051	* According to the SRP spec, the lower two bits of the 'ADDITIONAL
1052	* CDB LENGTH' field are reserved and the size in bytes of this field
1053	* is four times the value specified in bits 3..7. Hence the "& ~3".
1054	*/
1055	return srp_cmd->add_data + (srp_cmd->add_cdb_len & ~3);
1056	}
1057
1058	/**
1059	* srpt_get_desc_tbl - parse the data descriptors of a SRP_CMD request
1060	* @recv_ioctx: I/O context associated with the received command @srp_cmd.
1061	* @ioctx: I/O context that will be used for responding to the initiator.
1062	* @srp_cmd: Pointer to the SRP_CMD request data.
1063	* @dir: Pointer to the variable to which the transfer direction will be
1064	* written.
1065	* @sg: [out] scatterlist for the parsed SRP_CMD.
1066	* @sg_cnt: [out] length of @sg.
1067	* @data_len: Pointer to the variable to which the total data length of all
1068	* descriptors in the SRP_CMD request will be written.
1069	* @imm_data_offset: [in] Offset in SRP_CMD requests at which immediate data
1070	* starts.
1071	*
1072	* This function initializes ioctx->nrbuf and ioctx->r_bufs.
1073	*
1074	* Returns -EINVAL when the SRP_CMD request contains inconsistent descriptors;
1075	* -ENOMEM when memory allocation fails and zero upon success.
1076	*/
1077	static int srpt_get_desc_tbl(struct srpt_recv_ioctx *recv_ioctx,
1078	struct srpt_send_ioctx *ioctx,
1079	struct srp_cmd *srp_cmd, enum dma_data_direction *dir,
1080	struct scatterlist **sg, unsigned int *sg_cnt, u64 *data_len,
1081	u16 imm_data_offset)
1082	{
1083	BUG_ON(!dir);
1084	BUG_ON(!data_len);
1085
1086	/*
1087	* The lower four bits of the buffer format field contain the DATA-IN
1088	* buffer descriptor format, and the highest four bits contain the
1089	* DATA-OUT buffer descriptor format.
1090	*/
1091	if (srp_cmd->buf_fmt & 0xf)
1092	/* DATA-IN: transfer data from target to initiator (read). */
1093	*dir = DMA_FROM_DEVICE;
1094	else if (srp_cmd->buf_fmt >> 4)
1095	/* DATA-OUT: transfer data from initiator to target (write). */
1096	*dir = DMA_TO_DEVICE;
1097	else
1098	*dir = DMA_NONE;
1099
1100	/* initialize data_direction early as srpt_alloc_rw_ctxs needs it */
1101	ioctx->cmd.data_direction = *dir;
1102
1103	if (((srp_cmd->buf_fmt & 0xf) == SRP_DATA_DESC_DIRECT) ||
1104	((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_DIRECT)) {
1105	struct srp_direct_buf *db = srpt_get_desc_buf(srp_cmd);
1106
1107	*data_len = be32_to_cpu(db->len);
1108	return srpt_alloc_rw_ctxs(ioctx, db, nbufs: 1, sg, sg_cnt);
1109	} else if (((srp_cmd->buf_fmt & 0xf) == SRP_DATA_DESC_INDIRECT) ||
1110	((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_INDIRECT)) {
1111	struct srp_indirect_buf *idb = srpt_get_desc_buf(srp_cmd);
1112	int nbufs = be32_to_cpu(idb->table_desc.len) /
1113	sizeof(struct srp_direct_buf);
1114
1115	if (nbufs >
1116	(srp_cmd->data_out_desc_cnt + srp_cmd->data_in_desc_cnt)) {
1117	pr_err("received unsupported SRP_CMD request type (%u out + %u in != %u / %zu)\n",
1118	srp_cmd->data_out_desc_cnt,
1119	srp_cmd->data_in_desc_cnt,
1120	be32_to_cpu(idb->table_desc.len),
1121	sizeof(struct srp_direct_buf));
1122	return -EINVAL;
1123	}
1124
1125	*data_len = be32_to_cpu(idb->len);
1126	return srpt_alloc_rw_ctxs(ioctx, db: idb->desc_list, nbufs,
1127	sg, sg_cnt);
1128	} else if ((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_IMM) {
1129	struct srp_imm_buf *imm_buf = srpt_get_desc_buf(srp_cmd);
1130	void *data = (void *)srp_cmd + imm_data_offset;
1131	uint32_t len = be32_to_cpu(imm_buf->len);
1132	uint32_t req_size = imm_data_offset + len;
1133
1134	if (req_size > srp_max_req_size) {
1135	pr_err("Immediate data (length %d + %d) exceeds request size %d\n",
1136	imm_data_offset, len, srp_max_req_size);
1137	return -EINVAL;
1138	}
1139	if (recv_ioctx->byte_len < req_size) {
1140	pr_err("Received too few data - %d < %d\n",
1141	recv_ioctx->byte_len, req_size);
1142	return -EIO;
1143	}
1144	/*
1145	* The immediate data buffer descriptor must occur before the
1146	* immediate data itself.
1147	*/
1148	if ((void *)(imm_buf + 1) > (void *)data) {
1149	pr_err("Received invalid write request\n");
1150	return -EINVAL;
1151	}
1152	*data_len = len;
1153	ioctx->recv_ioctx = recv_ioctx;
1154	if ((uintptr_t)data & 511) {
1155	pr_warn_once("Internal error - the receive buffers are not aligned properly.\n");
1156	return -EINVAL;
1157	}
1158	sg_init_one(&ioctx->imm_sg, data, len);
1159	*sg = &ioctx->imm_sg;
1160	*sg_cnt = 1;
1161	return 0;
1162	} else {
1163	*data_len = 0;
1164	return 0;
1165	}
1166	}
1167
1168	/**
1169	* srpt_init_ch_qp - initialize queue pair attributes
1170	* @ch: SRPT RDMA channel.
1171	* @qp: Queue pair pointer.
1172	*
1173	* Initialized the attributes of queue pair 'qp' by allowing local write,
1174	* remote read and remote write. Also transitions 'qp' to state IB_QPS_INIT.
1175	*/
1176	static int srpt_init_ch_qp(struct srpt_rdma_ch *ch, struct ib_qp *qp)
1177	{
1178	struct ib_qp_attr *attr;
1179	int ret;
1180
1181	WARN_ON_ONCE(ch->using_rdma_cm);
1182
1183	attr = kzalloc(sizeof(*attr), GFP_KERNEL);
1184	if (!attr)
1185	return -ENOMEM;
1186
1187	attr->qp_state = IB_QPS_INIT;
1188	attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE;
1189	attr->port_num = ch->sport->port;
1190
1191	ret = ib_find_cached_pkey(device: ch->sport->sdev->device, port_num: ch->sport->port,
1192	pkey: ch->pkey, index: &attr->pkey_index);
1193	if (ret < 0)
1194	pr_err("Translating pkey %#x failed (%d) - using index 0\n",
1195	ch->pkey, ret);
1196
1197	ret = ib_modify_qp(qp, qp_attr: attr,
1198	qp_attr_mask: IB_QP_STATE | IB_QP_ACCESS_FLAGS | IB_QP_PORT |
1199	IB_QP_PKEY_INDEX);
1200
1201	kfree(objp: attr);
1202	return ret;
1203	}
1204
1205	/**
1206	* srpt_ch_qp_rtr - change the state of a channel to 'ready to receive' (RTR)
1207	* @ch: channel of the queue pair.
1208	* @qp: queue pair to change the state of.
1209	*
1210	* Returns zero upon success and a negative value upon failure.
1211	*
1212	* Note: currently a struct ib_qp_attr takes 136 bytes on a 64-bit system.
1213	* If this structure ever becomes larger, it might be necessary to allocate
1214	* it dynamically instead of on the stack.
1215	*/
1216	static int srpt_ch_qp_rtr(struct srpt_rdma_ch *ch, struct ib_qp *qp)
1217	{
1218	struct ib_qp_attr qp_attr;
1219	int attr_mask;
1220	int ret;
1221
1222	WARN_ON_ONCE(ch->using_rdma_cm);
1223
1224	qp_attr.qp_state = IB_QPS_RTR;
1225	ret = ib_cm_init_qp_attr(cm_id: ch->ib_cm.cm_id, qp_attr: &qp_attr, qp_attr_mask: &attr_mask);
1226	if (ret)
1227	goto out;
1228
1229	qp_attr.max_dest_rd_atomic = 4;
1230
1231	ret = ib_modify_qp(qp, qp_attr: &qp_attr, qp_attr_mask: attr_mask);
1232
1233	out:
1234	return ret;
1235	}
1236
1237	/**
1238	* srpt_ch_qp_rts - change the state of a channel to 'ready to send' (RTS)
1239	* @ch: channel of the queue pair.
1240	* @qp: queue pair to change the state of.
1241	*
1242	* Returns zero upon success and a negative value upon failure.
1243	*
1244	* Note: currently a struct ib_qp_attr takes 136 bytes on a 64-bit system.
1245	* If this structure ever becomes larger, it might be necessary to allocate
1246	* it dynamically instead of on the stack.
1247	*/
1248	static int srpt_ch_qp_rts(struct srpt_rdma_ch *ch, struct ib_qp *qp)
1249	{
1250	struct ib_qp_attr qp_attr;
1251	int attr_mask;
1252	int ret;
1253
1254	qp_attr.qp_state = IB_QPS_RTS;
1255	ret = ib_cm_init_qp_attr(cm_id: ch->ib_cm.cm_id, qp_attr: &qp_attr, qp_attr_mask: &attr_mask);
1256	if (ret)
1257	goto out;
1258
1259	qp_attr.max_rd_atomic = 4;
1260
1261	ret = ib_modify_qp(qp, qp_attr: &qp_attr, qp_attr_mask: attr_mask);
1262
1263	out:
1264	return ret;
1265	}
1266
1267	/**
1268	* srpt_ch_qp_err - set the channel queue pair state to 'error'
1269	* @ch: SRPT RDMA channel.
1270	*/
1271	static int srpt_ch_qp_err(struct srpt_rdma_ch *ch)
1272	{
1273	struct ib_qp_attr qp_attr;
1274
1275	qp_attr.qp_state = IB_QPS_ERR;
1276	return ib_modify_qp(qp: ch->qp, qp_attr: &qp_attr, qp_attr_mask: IB_QP_STATE);
1277	}
1278
1279	/**
1280	* srpt_get_send_ioctx - obtain an I/O context for sending to the initiator
1281	* @ch: SRPT RDMA channel.
1282	*/
1283	static struct srpt_send_ioctx *srpt_get_send_ioctx(struct srpt_rdma_ch *ch)
1284	{
1285	struct srpt_send_ioctx *ioctx;
1286	int tag, cpu;
1287
1288	BUG_ON(!ch);
1289
1290	tag = sbitmap_queue_get(sbq: &ch->sess->sess_tag_pool, cpu: &cpu);
1291	if (tag < 0)
1292	return NULL;
1293
1294	ioctx = ch->ioctx_ring[tag];
1295	BUG_ON(ioctx->ch != ch);
1296	ioctx->state = SRPT_STATE_NEW;
1297	WARN_ON_ONCE(ioctx->recv_ioctx);
1298	ioctx->n_rdma = 0;
1299	ioctx->n_rw_ctx = 0;
1300	ioctx->queue_status_only = false;
1301	/*
1302	* transport_init_se_cmd() does not initialize all fields, so do it
1303	* here.
1304	*/
1305	memset(&ioctx->cmd, 0, sizeof(ioctx->cmd));
1306	memset(&ioctx->sense_data, 0, sizeof(ioctx->sense_data));
1307	ioctx->cmd.map_tag = tag;
1308	ioctx->cmd.map_cpu = cpu;
1309
1310	return ioctx;
1311	}
1312
1313	/**
1314	* srpt_abort_cmd - abort a SCSI command
1315	* @ioctx: I/O context associated with the SCSI command.
1316	*/
1317	static int srpt_abort_cmd(struct srpt_send_ioctx *ioctx)
1318	{
1319	enum srpt_command_state state;
1320
1321	BUG_ON(!ioctx);
1322
1323	/*
1324	* If the command is in a state where the target core is waiting for
1325	* the ib_srpt driver, change the state to the next state.
1326	*/
1327
1328	state = ioctx->state;
1329	switch (state) {
1330	case SRPT_STATE_NEED_DATA:
1331	ioctx->state = SRPT_STATE_DATA_IN;
1332	break;
1333	case SRPT_STATE_CMD_RSP_SENT:
1334	case SRPT_STATE_MGMT_RSP_SENT:
1335	ioctx->state = SRPT_STATE_DONE;
1336	break;
1337	default:
1338	WARN_ONCE(true, "%s: unexpected I/O context state %d\n",
1339	__func__, state);
1340	break;
1341	}
1342
1343	pr_debug("Aborting cmd with state %d -> %d and tag %lld\n", state,
1344	ioctx->state, ioctx->cmd.tag);
1345
1346	switch (state) {
1347	case SRPT_STATE_NEW:
1348	case SRPT_STATE_DATA_IN:
1349	case SRPT_STATE_MGMT:
1350	case SRPT_STATE_DONE:
1351	/*
1352	* Do nothing - defer abort processing until
1353	* srpt_queue_response() is invoked.
1354	*/
1355	break;
1356	case SRPT_STATE_NEED_DATA:
1357	pr_debug("tag %#llx: RDMA read error\n", ioctx->cmd.tag);
1358	transport_generic_request_failure(&ioctx->cmd,
1359	TCM_CHECK_CONDITION_ABORT_CMD);
1360	break;
1361	case SRPT_STATE_CMD_RSP_SENT:
1362	/*
1363	* SRP_RSP sending failed or the SRP_RSP send completion has
1364	* not been received in time.
1365	*/
1366	transport_generic_free_cmd(&ioctx->cmd, 0);
1367	break;
1368	case SRPT_STATE_MGMT_RSP_SENT:
1369	transport_generic_free_cmd(&ioctx->cmd, 0);
1370	break;
1371	default:
1372	WARN(1, "Unexpected command state (%d)", state);
1373	break;
1374	}
1375
1376	return state;
1377	}
1378
1379	/**
1380	* srpt_rdma_read_done - RDMA read completion callback
1381	* @cq: Completion queue.
1382	* @wc: Work completion.
1383	*
1384	* XXX: what is now target_execute_cmd used to be asynchronous, and unmapping
1385	* the data that has been transferred via IB RDMA had to be postponed until the
1386	* check_stop_free() callback. None of this is necessary anymore and needs to
1387	* be cleaned up.
1388	*/
1389	static void srpt_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc)
1390	{
1391	struct srpt_rdma_ch *ch = wc->qp->qp_context;
1392	struct srpt_send_ioctx *ioctx =
1393	container_of(wc->wr_cqe, struct srpt_send_ioctx, rdma_cqe);
1394
1395	WARN_ON(ioctx->n_rdma <= 0);
1396	atomic_add(i: ioctx->n_rdma, v: &ch->sq_wr_avail);
1397	ioctx->n_rdma = 0;
1398
1399	if (unlikely(wc->status != IB_WC_SUCCESS)) {
1400	pr_info("RDMA_READ for ioctx 0x%p failed with status %d\n",
1401	ioctx, wc->status);
1402	srpt_abort_cmd(ioctx);
1403	return;
1404	}
1405
1406	if (srpt_test_and_set_cmd_state(ioctx, old: SRPT_STATE_NEED_DATA,
1407	new: SRPT_STATE_DATA_IN))
1408	target_execute_cmd(cmd: &ioctx->cmd);
1409	else
1410	pr_err("%s[%d]: wrong state = %d\n", __func__,
1411	__LINE__, ioctx->state);
1412	}
1413
1414	/**
1415	* srpt_build_cmd_rsp - build a SRP_RSP response
1416	* @ch: RDMA channel through which the request has been received.
1417	* @ioctx: I/O context associated with the SRP_CMD request. The response will
1418	* be built in the buffer ioctx->buf points at and hence this function will
1419	* overwrite the request data.
1420	* @tag: tag of the request for which this response is being generated.
1421	* @status: value for the STATUS field of the SRP_RSP information unit.
1422	*
1423	* Returns the size in bytes of the SRP_RSP response.
1424	*
1425	* An SRP_RSP response contains a SCSI status or service response. See also
1426	* section 6.9 in the SRP r16a document for the format of an SRP_RSP
1427	* response. See also SPC-2 for more information about sense data.
1428	*/
1429	static int srpt_build_cmd_rsp(struct srpt_rdma_ch *ch,
1430	struct srpt_send_ioctx *ioctx, u64 tag,
1431	int status)
1432	{
1433	struct se_cmd *cmd = &ioctx->cmd;
1434	struct srp_rsp *srp_rsp;
1435	const u8 *sense_data;
1436	int sense_data_len, max_sense_len;
1437	u32 resid = cmd->residual_count;
1438
1439	/*
1440	* The lowest bit of all SAM-3 status codes is zero (see also
1441	* paragraph 5.3 in SAM-3).
1442	*/
1443	WARN_ON(status & 1);
1444
1445	srp_rsp = ioctx->ioctx.buf;
1446	BUG_ON(!srp_rsp);
1447
1448	sense_data = ioctx->sense_data;
1449	sense_data_len = ioctx->cmd.scsi_sense_length;
1450	WARN_ON(sense_data_len > sizeof(ioctx->sense_data));
1451
1452	memset(srp_rsp, 0, sizeof(*srp_rsp));
1453	srp_rsp->opcode = SRP_RSP;
1454	srp_rsp->req_lim_delta =
1455	cpu_to_be32(1 + atomic_xchg(&ch->req_lim_delta, 0));
1456	srp_rsp->tag = tag;
1457	srp_rsp->status = status;
1458
1459	if (cmd->se_cmd_flags & SCF_UNDERFLOW_BIT) {
1460	if (cmd->data_direction == DMA_TO_DEVICE) {
1461	/* residual data from an underflow write */
1462	srp_rsp->flags = SRP_RSP_FLAG_DOUNDER;
1463	srp_rsp->data_out_res_cnt = cpu_to_be32(resid);
1464	} else if (cmd->data_direction == DMA_FROM_DEVICE) {
1465	/* residual data from an underflow read */
1466	srp_rsp->flags = SRP_RSP_FLAG_DIUNDER;
1467	srp_rsp->data_in_res_cnt = cpu_to_be32(resid);
1468	}
1469	} else if (cmd->se_cmd_flags & SCF_OVERFLOW_BIT) {
1470	if (cmd->data_direction == DMA_TO_DEVICE) {
1471	/* residual data from an overflow write */
1472	srp_rsp->flags = SRP_RSP_FLAG_DOOVER;
1473	srp_rsp->data_out_res_cnt = cpu_to_be32(resid);
1474	} else if (cmd->data_direction == DMA_FROM_DEVICE) {
1475	/* residual data from an overflow read */
1476	srp_rsp->flags = SRP_RSP_FLAG_DIOVER;
1477	srp_rsp->data_in_res_cnt = cpu_to_be32(resid);
1478	}
1479	}
1480
1481	if (sense_data_len) {
1482	BUILD_BUG_ON(MIN_MAX_RSP_SIZE <= sizeof(*srp_rsp));
1483	max_sense_len = ch->max_ti_iu_len - sizeof(*srp_rsp);
1484	if (sense_data_len > max_sense_len) {
1485	pr_warn("truncated sense data from %d to %d bytes\n",
1486	sense_data_len, max_sense_len);
1487	sense_data_len = max_sense_len;
1488	}
1489
1490	srp_rsp->flags |= SRP_RSP_FLAG_SNSVALID;
1491	srp_rsp->sense_data_len = cpu_to_be32(sense_data_len);
1492	memcpy(srp_rsp->data, sense_data, sense_data_len);
1493	}
1494
1495	return sizeof(*srp_rsp) + sense_data_len;
1496	}
1497
1498	/**
1499	* srpt_build_tskmgmt_rsp - build a task management response
1500	* @ch: RDMA channel through which the request has been received.
1501	* @ioctx: I/O context in which the SRP_RSP response will be built.
1502	* @rsp_code: RSP_CODE that will be stored in the response.
1503	* @tag: Tag of the request for which this response is being generated.
1504	*
1505	* Returns the size in bytes of the SRP_RSP response.
1506	*
1507	* An SRP_RSP response contains a SCSI status or service response. See also
1508	* section 6.9 in the SRP r16a document for the format of an SRP_RSP
1509	* response.
1510	*/
1511	static int srpt_build_tskmgmt_rsp(struct srpt_rdma_ch *ch,
1512	struct srpt_send_ioctx *ioctx,
1513	u8 rsp_code, u64 tag)
1514	{
1515	struct srp_rsp *srp_rsp;
1516	int resp_data_len;
1517	int resp_len;
1518
1519	resp_data_len = 4;
1520	resp_len = sizeof(*srp_rsp) + resp_data_len;
1521
1522	srp_rsp = ioctx->ioctx.buf;
1523	BUG_ON(!srp_rsp);
1524	memset(srp_rsp, 0, sizeof(*srp_rsp));
1525
1526	srp_rsp->opcode = SRP_RSP;
1527	srp_rsp->req_lim_delta =
1528	cpu_to_be32(1 + atomic_xchg(&ch->req_lim_delta, 0));
1529	srp_rsp->tag = tag;
1530
1531	srp_rsp->flags |= SRP_RSP_FLAG_RSPVALID;
1532	srp_rsp->resp_data_len = cpu_to_be32(resp_data_len);
1533	srp_rsp->data[3] = rsp_code;
1534
1535	return resp_len;
1536	}
1537
1538	static int srpt_check_stop_free(struct se_cmd *cmd)
1539	{
1540	struct srpt_send_ioctx *ioctx = container_of(cmd,
1541	struct srpt_send_ioctx, cmd);
1542
1543	return target_put_sess_cmd(&ioctx->cmd);
1544	}
1545
1546	/**
1547	* srpt_handle_cmd - process a SRP_CMD information unit
1548	* @ch: SRPT RDMA channel.
1549	* @recv_ioctx: Receive I/O context.
1550	* @send_ioctx: Send I/O context.
1551	*/
1552	static void srpt_handle_cmd(struct srpt_rdma_ch *ch,
1553	struct srpt_recv_ioctx *recv_ioctx,
1554	struct srpt_send_ioctx *send_ioctx)
1555	{
1556	struct se_cmd *cmd;
1557	struct srp_cmd *srp_cmd;
1558	struct scatterlist *sg = NULL;
1559	unsigned sg_cnt = 0;
1560	u64 data_len;
1561	enum dma_data_direction dir;
1562	int rc;
1563
1564	BUG_ON(!send_ioctx);
1565
1566	srp_cmd = recv_ioctx->ioctx.buf + recv_ioctx->ioctx.offset;
1567	cmd = &send_ioctx->cmd;
1568	cmd->tag = srp_cmd->tag;
1569
1570	switch (srp_cmd->task_attr) {
1571	case SRP_CMD_SIMPLE_Q:
1572	cmd->sam_task_attr = TCM_SIMPLE_TAG;
1573	break;
1574	case SRP_CMD_ORDERED_Q:
1575	default:
1576	cmd->sam_task_attr = TCM_ORDERED_TAG;
1577	break;
1578	case SRP_CMD_HEAD_OF_Q:
1579	cmd->sam_task_attr = TCM_HEAD_TAG;
1580	break;
1581	case SRP_CMD_ACA:
1582	cmd->sam_task_attr = TCM_ACA_TAG;
1583	break;
1584	}
1585
1586	rc = srpt_get_desc_tbl(recv_ioctx, ioctx: send_ioctx, srp_cmd, dir: &dir,
1587	sg: &sg, sg_cnt: &sg_cnt, data_len: &data_len, imm_data_offset: ch->imm_data_offset);
1588	if (rc) {
1589	if (rc != -EAGAIN) {
1590	pr_err("0x%llx: parsing SRP descriptor table failed.\n",
1591	srp_cmd->tag);
1592	}
1593	goto busy;
1594	}
1595
1596	rc = target_init_cmd(se_cmd: cmd, se_sess: ch->sess, sense: &send_ioctx->sense_data[0],
1597	unpacked_lun: scsilun_to_int(&srp_cmd->lun), data_length: data_len,
1598	TCM_SIMPLE_TAG, data_dir: dir, flags: TARGET_SCF_ACK_KREF);
1599	if (rc != 0) {
1600	pr_debug("target_submit_cmd() returned %d for tag %#llx\n", rc,
1601	srp_cmd->tag);
1602	goto busy;
1603	}
1604
1605	if (target_submit_prep(se_cmd: cmd, cdb: srp_cmd->cdb, sgl: sg, sgl_count: sg_cnt, NULL, sgl_bidi_count: 0, NULL, sgl_prot_count: 0,
1606	GFP_KERNEL))
1607	return;
1608
1609	target_submit(se_cmd: cmd);
1610	return;
1611
1612	busy:
1613	target_send_busy(cmd);
1614	}
1615
1616	static int srp_tmr_to_tcm(int fn)
1617	{
1618	switch (fn) {
1619	case SRP_TSK_ABORT_TASK:
1620	return TMR_ABORT_TASK;
1621	case SRP_TSK_ABORT_TASK_SET:
1622	return TMR_ABORT_TASK_SET;
1623	case SRP_TSK_CLEAR_TASK_SET:
1624	return TMR_CLEAR_TASK_SET;
1625	case SRP_TSK_LUN_RESET:
1626	return TMR_LUN_RESET;
1627	case SRP_TSK_CLEAR_ACA:
1628	return TMR_CLEAR_ACA;
1629	default:
1630	return -1;
1631	}
1632	}
1633
1634	/**
1635	* srpt_handle_tsk_mgmt - process a SRP_TSK_MGMT information unit
1636	* @ch: SRPT RDMA channel.
1637	* @recv_ioctx: Receive I/O context.
1638	* @send_ioctx: Send I/O context.
1639	*
1640	* Returns 0 if and only if the request will be processed by the target core.
1641	*
1642	* For more information about SRP_TSK_MGMT information units, see also section
1643	* 6.7 in the SRP r16a document.
1644	*/
1645	static void srpt_handle_tsk_mgmt(struct srpt_rdma_ch *ch,
1646	struct srpt_recv_ioctx *recv_ioctx,
1647	struct srpt_send_ioctx *send_ioctx)
1648	{
1649	struct srp_tsk_mgmt *srp_tsk;
1650	struct se_cmd *cmd;
1651	struct se_session *sess = ch->sess;
1652	int tcm_tmr;
1653	int rc;
1654
1655	BUG_ON(!send_ioctx);
1656
1657	srp_tsk = recv_ioctx->ioctx.buf + recv_ioctx->ioctx.offset;
1658	cmd = &send_ioctx->cmd;
1659
1660	pr_debug("recv tsk_mgmt fn %d for task_tag %lld and cmd tag %lld ch %p sess %p\n",
1661	srp_tsk->tsk_mgmt_func, srp_tsk->task_tag, srp_tsk->tag, ch,
1662	ch->sess);
1663
1664	srpt_set_cmd_state(ioctx: send_ioctx, new: SRPT_STATE_MGMT);
1665	send_ioctx->cmd.tag = srp_tsk->tag;
1666	tcm_tmr = srp_tmr_to_tcm(fn: srp_tsk->tsk_mgmt_func);
1667	rc = target_submit_tmr(se_cmd: &send_ioctx->cmd, se_sess: sess, NULL,
1668	unpacked_lun: scsilun_to_int(&srp_tsk->lun), fabric_tmr_ptr: srp_tsk, tm_type: tcm_tmr,
1669	GFP_KERNEL, srp_tsk->task_tag,
1670	TARGET_SCF_ACK_KREF);
1671	if (rc != 0) {
1672	send_ioctx->cmd.se_tmr_req->response = TMR_FUNCTION_REJECTED;
1673	cmd->se_tfo->queue_tm_rsp(cmd);
1674	}
1675	return;
1676	}
1677
1678	/**
1679	* srpt_handle_new_iu - process a newly received information unit
1680	* @ch: RDMA channel through which the information unit has been received.
1681	* @recv_ioctx: Receive I/O context associated with the information unit.
1682	*/
1683	static bool
1684	srpt_handle_new_iu(struct srpt_rdma_ch *ch, struct srpt_recv_ioctx *recv_ioctx)
1685	{
1686	struct srpt_send_ioctx *send_ioctx = NULL;
1687	struct srp_cmd *srp_cmd;
1688	bool res = false;
1689	u8 opcode;
1690
1691	BUG_ON(!ch);
1692	BUG_ON(!recv_ioctx);
1693
1694	if (unlikely(ch->state == CH_CONNECTING))
1695	goto push;
1696
1697	ib_dma_sync_single_for_cpu(dev: ch->sport->sdev->device,
1698	addr: recv_ioctx->ioctx.dma,
1699	size: recv_ioctx->ioctx.offset + srp_max_req_size,
1700	dir: DMA_FROM_DEVICE);
1701
1702	srp_cmd = recv_ioctx->ioctx.buf + recv_ioctx->ioctx.offset;
1703	opcode = srp_cmd->opcode;
1704	if (opcode == SRP_CMD || opcode == SRP_TSK_MGMT) {
1705	send_ioctx = srpt_get_send_ioctx(ch);
1706	if (unlikely(!send_ioctx))
1707	goto push;
1708	}
1709
1710	if (!list_empty(head: &recv_ioctx->wait_list)) {
1711	WARN_ON_ONCE(!ch->processing_wait_list);
1712	list_del_init(entry: &recv_ioctx->wait_list);
1713	}
1714
1715	switch (opcode) {
1716	case SRP_CMD:
1717	srpt_handle_cmd(ch, recv_ioctx, send_ioctx);
1718	break;
1719	case SRP_TSK_MGMT:
1720	srpt_handle_tsk_mgmt(ch, recv_ioctx, send_ioctx);
1721	break;
1722	case SRP_I_LOGOUT:
1723	pr_err("Not yet implemented: SRP_I_LOGOUT\n");
1724	break;
1725	case SRP_CRED_RSP:
1726	pr_debug("received SRP_CRED_RSP\n");
1727	break;
1728	case SRP_AER_RSP:
1729	pr_debug("received SRP_AER_RSP\n");
1730	break;
1731	case SRP_RSP:
1732	pr_err("Received SRP_RSP\n");
1733	break;
1734	default:
1735	pr_err("received IU with unknown opcode 0x%x\n", opcode);
1736	break;
1737	}
1738
1739	if (!send_ioctx || !send_ioctx->recv_ioctx)
1740	srpt_post_recv(sdev: ch->sport->sdev, ch, ioctx: recv_ioctx);
1741	res = true;
1742
1743	out:
1744	return res;
1745
1746	push:
1747	if (list_empty(head: &recv_ioctx->wait_list)) {
1748	WARN_ON_ONCE(ch->processing_wait_list);
1749	list_add_tail(new: &recv_ioctx->wait_list, head: &ch->cmd_wait_list);
1750	}
1751	goto out;
1752	}
1753
1754	static void srpt_recv_done(struct ib_cq *cq, struct ib_wc *wc)
1755	{
1756	struct srpt_rdma_ch *ch = wc->qp->qp_context;
1757	struct srpt_recv_ioctx *ioctx =
1758	container_of(wc->wr_cqe, struct srpt_recv_ioctx, ioctx.cqe);
1759
1760	if (wc->status == IB_WC_SUCCESS) {
1761	int req_lim;
1762
1763	req_lim = atomic_dec_return(v: &ch->req_lim);
1764	if (unlikely(req_lim < 0))
1765	pr_err("req_lim = %d < 0\n", req_lim);
1766	ioctx->byte_len = wc->byte_len;
1767	srpt_handle_new_iu(ch, recv_ioctx: ioctx);
1768	} else {
1769	pr_info_ratelimited("receiving failed for ioctx %p with status %d\n",
1770	ioctx, wc->status);
1771	}
1772	}
1773
1774	/*
1775	* This function must be called from the context in which RDMA completions are
1776	* processed because it accesses the wait list without protection against
1777	* access from other threads.
1778	*/
1779	static void srpt_process_wait_list(struct srpt_rdma_ch *ch)
1780	{
1781	struct srpt_recv_ioctx *recv_ioctx, *tmp;
1782
1783	WARN_ON_ONCE(ch->state == CH_CONNECTING);
1784
1785	if (list_empty(head: &ch->cmd_wait_list))
1786	return;
1787
1788	WARN_ON_ONCE(ch->processing_wait_list);
1789	ch->processing_wait_list = true;
1790	list_for_each_entry_safe(recv_ioctx, tmp, &ch->cmd_wait_list,
1791	wait_list) {
1792	if (!srpt_handle_new_iu(ch, recv_ioctx))
1793	break;
1794	}
1795	ch->processing_wait_list = false;
1796	}
1797
1798	/**
1799	* srpt_send_done - send completion callback
1800	* @cq: Completion queue.
1801	* @wc: Work completion.
1802	*
1803	* Note: Although this has not yet been observed during tests, at least in
1804	* theory it is possible that the srpt_get_send_ioctx() call invoked by
1805	* srpt_handle_new_iu() fails. This is possible because the req_lim_delta
1806	* value in each response is set to one, and it is possible that this response
1807	* makes the initiator send a new request before the send completion for that
1808	* response has been processed. This could e.g. happen if the call to
1809	* srpt_put_send_iotcx() is delayed because of a higher priority interrupt or
1810	* if IB retransmission causes generation of the send completion to be
1811	* delayed. Incoming information units for which srpt_get_send_ioctx() fails
1812	* are queued on cmd_wait_list. The code below processes these delayed
1813	* requests one at a time.
1814	*/
1815	static void srpt_send_done(struct ib_cq *cq, struct ib_wc *wc)
1816	{
1817	struct srpt_rdma_ch *ch = wc->qp->qp_context;
1818	struct srpt_send_ioctx *ioctx =
1819	container_of(wc->wr_cqe, struct srpt_send_ioctx, ioctx.cqe);
1820	enum srpt_command_state state;
1821
1822	state = srpt_set_cmd_state(ioctx, new: SRPT_STATE_DONE);
1823
1824	WARN_ON(state != SRPT_STATE_CMD_RSP_SENT &&
1825	state != SRPT_STATE_MGMT_RSP_SENT);
1826
1827	atomic_add(i: 1 + ioctx->n_rdma, v: &ch->sq_wr_avail);
1828
1829	if (wc->status != IB_WC_SUCCESS)
1830	pr_info("sending response for ioctx 0x%p failed with status %d\n",
1831	ioctx, wc->status);
1832
1833	if (state != SRPT_STATE_DONE) {
1834	transport_generic_free_cmd(&ioctx->cmd, 0);
1835	} else {
1836	pr_err("IB completion has been received too late for wr_id = %u.\n",
1837	ioctx->ioctx.index);
1838	}
1839
1840	srpt_process_wait_list(ch);
1841	}
1842
1843	/**
1844	* srpt_create_ch_ib - create receive and send completion queues
1845	* @ch: SRPT RDMA channel.
1846	*/
1847	static int srpt_create_ch_ib(struct srpt_rdma_ch *ch)
1848	{
1849	struct ib_qp_init_attr *qp_init;
1850	struct srpt_port *sport = ch->sport;
1851	struct srpt_device *sdev = sport->sdev;
1852	const struct ib_device_attr *attrs = &sdev->device->attrs;
1853	int sq_size = sport->port_attrib.srp_sq_size;
1854	int i, ret;
1855
1856	WARN_ON(ch->rq_size < 1);
1857
1858	ret = -ENOMEM;
1859	qp_init = kzalloc(sizeof(*qp_init), GFP_KERNEL);
1860	if (!qp_init)
1861	goto out;
1862
1863	retry:
1864	ch->cq = ib_cq_pool_get(dev: sdev->device, nr_cqe: ch->rq_size + sq_size, comp_vector_hint: -1,
1865	poll_ctx: IB_POLL_WORKQUEUE);
1866	if (IS_ERR(ptr: ch->cq)) {
1867	ret = PTR_ERR(ptr: ch->cq);
1868	pr_err("failed to create CQ cqe= %d ret= %pe\n",
1869	ch->rq_size + sq_size, ch->cq);
1870	goto out;
1871	}
1872	ch->cq_size = ch->rq_size + sq_size;
1873
1874	qp_init->qp_context = (void *)ch;
1875	qp_init->event_handler = srpt_qp_event;
1876	qp_init->send_cq = ch->cq;
1877	qp_init->recv_cq = ch->cq;
1878	qp_init->sq_sig_type = IB_SIGNAL_REQ_WR;
1879	qp_init->qp_type = IB_QPT_RC;
1880	/*
1881	* We divide up our send queue size into half SEND WRs to send the
1882	* completions, and half R/W contexts to actually do the RDMA
1883	* READ/WRITE transfers. Note that we need to allocate CQ slots for
1884	* both both, as RDMA contexts will also post completions for the
1885	* RDMA READ case.
1886	*/
1887	qp_init->cap.max_send_wr = min(sq_size / 2, attrs->max_qp_wr);
1888	qp_init->cap.max_rdma_ctxs = sq_size / 2;
1889	qp_init->cap.max_send_sge = attrs->max_send_sge;
1890	qp_init->cap.max_recv_sge = 1;
1891	qp_init->port_num = ch->sport->port;
1892	if (sdev->use_srq)
1893	qp_init->srq = sdev->srq;
1894	else
1895	qp_init->cap.max_recv_wr = ch->rq_size;
1896
1897	if (ch->using_rdma_cm) {
1898	ret = rdma_create_qp(id: ch->rdma_cm.cm_id, pd: sdev->pd, qp_init_attr: qp_init);
1899	ch->qp = ch->rdma_cm.cm_id->qp;
1900	} else {
1901	ch->qp = ib_create_qp(pd: sdev->pd, init_attr: qp_init);
1902	if (!IS_ERR(ptr: ch->qp)) {
1903	ret = srpt_init_ch_qp(ch, qp: ch->qp);
1904	if (ret)
1905	ib_destroy_qp(qp: ch->qp);
1906	} else {
1907	ret = PTR_ERR(ptr: ch->qp);
1908	}
1909	}
1910	if (ret) {
1911	bool retry = sq_size > MIN_SRPT_SQ_SIZE;
1912
1913	if (retry) {
1914	pr_debug("failed to create queue pair with sq_size = %d (%d) - retrying\n",
1915	sq_size, ret);
1916	ib_cq_pool_put(cq: ch->cq, nr_cqe: ch->cq_size);
1917	sq_size = max(sq_size / 2, MIN_SRPT_SQ_SIZE);
1918	goto retry;
1919	} else {
1920	pr_err("failed to create queue pair with sq_size = %d (%d)\n",
1921	sq_size, ret);
1922	goto err_destroy_cq;
1923	}
1924	}
1925
1926	atomic_set(v: &ch->sq_wr_avail, i: qp_init->cap.max_send_wr);
1927
1928	pr_debug("%s: max_cqe= %d max_sge= %d sq_size = %d ch= %p\n",
1929	__func__, ch->cq->cqe, qp_init->cap.max_send_sge,
1930	qp_init->cap.max_send_wr, ch);
1931
1932	if (!sdev->use_srq)
1933	for (i = 0; i < ch->rq_size; i++)
1934	srpt_post_recv(sdev, ch, ioctx: ch->ioctx_recv_ring[i]);
1935
1936	out:
1937	kfree(objp: qp_init);
1938	return ret;
1939
1940	err_destroy_cq:
1941	ch->qp = NULL;
1942	ib_cq_pool_put(cq: ch->cq, nr_cqe: ch->cq_size);
1943	goto out;
1944	}
1945
1946	static void srpt_destroy_ch_ib(struct srpt_rdma_ch *ch)
1947	{
1948	ib_destroy_qp(qp: ch->qp);
1949	ib_cq_pool_put(cq: ch->cq, nr_cqe: ch->cq_size);
1950	}
1951
1952	/**
1953	* srpt_close_ch - close a RDMA channel
1954	* @ch: SRPT RDMA channel.
1955	*
1956	* Make sure all resources associated with the channel will be deallocated at
1957	* an appropriate time.
1958	*
1959	* Returns true if and only if the channel state has been modified into
1960	* CH_DRAINING.
1961	*/
1962	static bool srpt_close_ch(struct srpt_rdma_ch *ch)
1963	{
1964	int ret;
1965
1966	if (!srpt_set_ch_state(ch, new: CH_DRAINING)) {
1967	pr_debug("%s: already closed\n", ch->sess_name);
1968	return false;
1969	}
1970
1971	kref_get(kref: &ch->kref);
1972
1973	ret = srpt_ch_qp_err(ch);
1974	if (ret < 0)
1975	pr_err("%s-%d: changing queue pair into error state failed: %d\n",
1976	ch->sess_name, ch->qp->qp_num, ret);
1977
1978	ret = srpt_zerolength_write(ch);
1979	if (ret < 0) {
1980	pr_err("%s-%d: queuing zero-length write failed: %d\n",
1981	ch->sess_name, ch->qp->qp_num, ret);
1982	if (srpt_set_ch_state(ch, new: CH_DISCONNECTED))
1983	schedule_work(work: &ch->release_work);
1984	else
1985	WARN_ON_ONCE(true);
1986	}
1987
1988	kref_put(kref: &ch->kref, release: srpt_free_ch);
1989
1990	return true;
1991	}
1992
1993	/*
1994	* Change the channel state into CH_DISCONNECTING. If a channel has not yet
1995	* reached the connected state, close it. If a channel is in the connected
1996	* state, send a DREQ. If a DREQ has been received, send a DREP. Note: it is
1997	* the responsibility of the caller to ensure that this function is not
1998	* invoked concurrently with the code that accepts a connection. This means
1999	* that this function must either be invoked from inside a CM callback
2000	* function or that it must be invoked with the srpt_port.mutex held.
2001	*/
2002	static int srpt_disconnect_ch(struct srpt_rdma_ch *ch)
2003	{
2004	int ret;
2005
2006	if (!srpt_set_ch_state(ch, new: CH_DISCONNECTING))
2007	return -ENOTCONN;
2008
2009	if (ch->using_rdma_cm) {
2010	ret = rdma_disconnect(id: ch->rdma_cm.cm_id);
2011	} else {
2012	ret = ib_send_cm_dreq(cm_id: ch->ib_cm.cm_id, NULL, private_data_len: 0);
2013	if (ret < 0)
2014	ret = ib_send_cm_drep(cm_id: ch->ib_cm.cm_id, NULL, private_data_len: 0);
2015	}
2016
2017	if (ret < 0 && srpt_close_ch(ch))
2018	ret = 0;
2019
2020	return ret;
2021	}
2022
2023	/* Send DREQ and wait for DREP. */
2024	static void srpt_disconnect_ch_sync(struct srpt_rdma_ch *ch)
2025	{
2026	DECLARE_COMPLETION_ONSTACK(closed);
2027	struct srpt_port *sport = ch->sport;
2028
2029	pr_debug("ch %s-%d state %d\n", ch->sess_name, ch->qp->qp_num,
2030	ch->state);
2031
2032	ch->closed = &closed;
2033
2034	mutex_lock(&sport->mutex);
2035	srpt_disconnect_ch(ch);
2036	mutex_unlock(lock: &sport->mutex);
2037
2038	while (wait_for_completion_timeout(x: &closed, timeout: 5 * HZ) == 0)
2039	pr_info("%s(%s-%d state %d): still waiting ...\n", __func__,
2040	ch->sess_name, ch->qp->qp_num, ch->state);
2041
2042	}
2043
2044	static void __srpt_close_all_ch(struct srpt_port *sport)
2045	{
2046	struct srpt_nexus *nexus;
2047	struct srpt_rdma_ch *ch;
2048
2049	lockdep_assert_held(&sport->mutex);
2050
2051	list_for_each_entry(nexus, &sport->nexus_list, entry) {
2052	list_for_each_entry(ch, &nexus->ch_list, list) {
2053	if (srpt_disconnect_ch(ch) >= 0)
2054	pr_info("Closing channel %s-%d because target %s_%d has been disabled\n",
2055	ch->sess_name, ch->qp->qp_num,
2056	dev_name(&sport->sdev->device->dev),
2057	sport->port);
2058	srpt_close_ch(ch);
2059	}
2060	}
2061	}
2062
2063	/*
2064	* Look up (i_port_id, t_port_id) in sport->nexus_list. Create an entry if
2065	* it does not yet exist.
2066	*/
2067	static struct srpt_nexus *srpt_get_nexus(struct srpt_port *sport,
2068	const u8 i_port_id[16],
2069	const u8 t_port_id[16])
2070	{
2071	struct srpt_nexus *nexus = NULL, *tmp_nexus = NULL, *n;
2072
2073	for (;;) {
2074	mutex_lock(&sport->mutex);
2075	list_for_each_entry(n, &sport->nexus_list, entry) {
2076	if (memcmp(p: n->i_port_id, q: i_port_id, size: 16) == 0 &&
2077	memcmp(p: n->t_port_id, q: t_port_id, size: 16) == 0) {
2078	nexus = n;
2079	break;
2080	}
2081	}
2082	if (!nexus && tmp_nexus) {
2083	list_add_tail_rcu(new: &tmp_nexus->entry,
2084	head: &sport->nexus_list);
2085	swap(nexus, tmp_nexus);
2086	}
2087	mutex_unlock(lock: &sport->mutex);
2088
2089	if (nexus)
2090	break;
2091	tmp_nexus = kzalloc(sizeof(*nexus), GFP_KERNEL);
2092	if (!tmp_nexus) {
2093	nexus = ERR_PTR(error: -ENOMEM);
2094	break;
2095	}
2096	INIT_LIST_HEAD(list: &tmp_nexus->ch_list);
2097	memcpy(tmp_nexus->i_port_id, i_port_id, 16);
2098	memcpy(tmp_nexus->t_port_id, t_port_id, 16);
2099	}
2100
2101	kfree(objp: tmp_nexus);
2102
2103	return nexus;
2104	}
2105
2106	static void srpt_set_enabled(struct srpt_port *sport, bool enabled)
2107	__must_hold(&sport->mutex)
2108	{
2109	lockdep_assert_held(&sport->mutex);
2110
2111	if (sport->enabled == enabled)
2112	return;
2113	sport->enabled = enabled;
2114	if (!enabled)
2115	__srpt_close_all_ch(sport);
2116	}
2117
2118	static void srpt_drop_sport_ref(struct srpt_port *sport)
2119	{
2120	if (atomic_dec_return(v: &sport->refcount) == 0 && sport->freed_channels)
2121	complete(sport->freed_channels);
2122	}
2123
2124	static void srpt_free_ch(struct kref *kref)
2125	{
2126	struct srpt_rdma_ch *ch = container_of(kref, struct srpt_rdma_ch, kref);
2127
2128	srpt_drop_sport_ref(sport: ch->sport);
2129	kfree_rcu(ch, rcu);
2130	}
2131
2132	/*
2133	* Shut down the SCSI target session, tell the connection manager to
2134	* disconnect the associated RDMA channel, transition the QP to the error
2135	* state and remove the channel from the channel list. This function is
2136	* typically called from inside srpt_zerolength_write_done(). Concurrent
2137	* srpt_zerolength_write() calls from inside srpt_close_ch() are possible
2138	* as long as the channel is on sport->nexus_list.
2139	*/
2140	static void srpt_release_channel_work(struct work_struct *w)
2141	{
2142	struct srpt_rdma_ch *ch;
2143	struct srpt_device *sdev;
2144	struct srpt_port *sport;
2145	struct se_session *se_sess;
2146
2147	ch = container_of(w, struct srpt_rdma_ch, release_work);
2148	pr_debug("%s-%d\n", ch->sess_name, ch->qp->qp_num);
2149
2150	sdev = ch->sport->sdev;
2151	BUG_ON(!sdev);
2152
2153	se_sess = ch->sess;
2154	BUG_ON(!se_sess);
2155
2156	target_stop_session(se_sess);
2157	target_wait_for_sess_cmds(se_sess);
2158
2159	target_remove_session(se_sess);
2160	ch->sess = NULL;
2161
2162	if (ch->using_rdma_cm)
2163	rdma_destroy_id(id: ch->rdma_cm.cm_id);
2164	else
2165	ib_destroy_cm_id(cm_id: ch->ib_cm.cm_id);
2166
2167	sport = ch->sport;
2168	mutex_lock(&sport->mutex);
2169	list_del_rcu(entry: &ch->list);
2170	mutex_unlock(lock: &sport->mutex);
2171
2172	if (ch->closed)
2173	complete(ch->closed);
2174
2175	srpt_destroy_ch_ib(ch);
2176
2177	srpt_free_ioctx_ring(ioctx_ring: (struct srpt_ioctx **)ch->ioctx_ring,
2178	sdev: ch->sport->sdev, ring_size: ch->rq_size,
2179	buf_cache: ch->rsp_buf_cache, dir: DMA_TO_DEVICE);
2180
2181	srpt_cache_put(c: ch->rsp_buf_cache);
2182
2183	srpt_free_ioctx_ring(ioctx_ring: (struct srpt_ioctx **)ch->ioctx_recv_ring,
2184	sdev, ring_size: ch->rq_size,
2185	buf_cache: ch->req_buf_cache, dir: DMA_FROM_DEVICE);
2186
2187	srpt_cache_put(c: ch->req_buf_cache);
2188
2189	kref_put(kref: &ch->kref, release: srpt_free_ch);
2190	}
2191
2192	/**
2193	* srpt_cm_req_recv - process the event IB_CM_REQ_RECEIVED
2194	* @sdev: HCA through which the login request was received.
2195	* @ib_cm_id: IB/CM connection identifier in case of IB/CM.
2196	* @rdma_cm_id: RDMA/CM connection identifier in case of RDMA/CM.
2197	* @port_num: Port through which the REQ message was received.
2198	* @pkey: P_Key of the incoming connection.
2199	* @req: SRP login request.
2200	* @src_addr: GID (IB/CM) or IP address (RDMA/CM) of the port that submitted
2201	* the login request.
2202	*
2203	* Ownership of the cm_id is transferred to the target session if this
2204	* function returns zero. Otherwise the caller remains the owner of cm_id.
2205	*/
2206	static int srpt_cm_req_recv(struct srpt_device *const sdev,
2207	struct ib_cm_id *ib_cm_id,
2208	struct rdma_cm_id *rdma_cm_id,
2209	u8 port_num, __be16 pkey,
2210	const struct srp_login_req *req,
2211	const char *src_addr)
2212	{
2213	struct srpt_port *sport = &sdev->port[port_num - 1];
2214	struct srpt_nexus *nexus;
2215	struct srp_login_rsp *rsp = NULL;
2216	struct srp_login_rej *rej = NULL;
2217	union {
2218	struct rdma_conn_param rdma_cm;
2219	struct ib_cm_rep_param ib_cm;
2220	} *rep_param = NULL;
2221	struct srpt_rdma_ch *ch = NULL;
2222	char i_port_id[36];
2223	u32 it_iu_len;
2224	int i, tag_num, tag_size, ret;
2225	struct srpt_tpg *stpg;
2226
2227	WARN_ON_ONCE(irqs_disabled());
2228
2229	it_iu_len = be32_to_cpu(req->req_it_iu_len);
2230
2231	pr_info("Received SRP_LOGIN_REQ with i_port_id %pI6, t_port_id %pI6 and it_iu_len %d on port %d (guid=%pI6); pkey %#04x\n",
2232	req->initiator_port_id, req->target_port_id, it_iu_len,
2233	port_num, &sport->gid, be16_to_cpu(pkey));
2234
2235	nexus = srpt_get_nexus(sport, i_port_id: req->initiator_port_id,
2236	t_port_id: req->target_port_id);
2237	if (IS_ERR(ptr: nexus)) {
2238	ret = PTR_ERR(ptr: nexus);
2239	goto out;
2240	}
2241
2242	ret = -ENOMEM;
2243	rsp = kzalloc(sizeof(*rsp), GFP_KERNEL);
2244	rej = kzalloc(sizeof(*rej), GFP_KERNEL);
2245	rep_param = kzalloc(sizeof(*rep_param), GFP_KERNEL);
2246	if (!rsp || !rej || !rep_param)
2247	goto out;
2248
2249	ret = -EINVAL;
2250	if (it_iu_len > srp_max_req_size || it_iu_len < 64) {
2251	rej->reason = cpu_to_be32(
2252	SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE);
2253	pr_err("rejected SRP_LOGIN_REQ because its length (%d bytes) is out of range (%d .. %d)\n",
2254	it_iu_len, 64, srp_max_req_size);
2255	goto reject;
2256	}
2257
2258	if (!sport->enabled) {
2259	rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
2260	pr_info("rejected SRP_LOGIN_REQ because target port %s_%d has not yet been enabled\n",
2261	dev_name(&sport->sdev->device->dev), port_num);
2262	goto reject;
2263	}
2264
2265	if (*(__be64 *)req->target_port_id != cpu_to_be64(srpt_service_guid)
2266	|| *(__be64 *)(req->target_port_id + 8) !=
2267	cpu_to_be64(srpt_service_guid)) {
2268	rej->reason = cpu_to_be32(
2269	SRP_LOGIN_REJ_UNABLE_ASSOCIATE_CHANNEL);
2270	pr_err("rejected SRP_LOGIN_REQ because it has an invalid target port identifier.\n");
2271	goto reject;
2272	}
2273
2274	ret = -ENOMEM;
2275	ch = kzalloc(sizeof(*ch), GFP_KERNEL);
2276	if (!ch) {
2277	rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
2278	pr_err("rejected SRP_LOGIN_REQ because out of memory.\n");
2279	goto reject;
2280	}
2281
2282	kref_init(kref: &ch->kref);
2283	ch->pkey = be16_to_cpu(pkey);
2284	ch->nexus = nexus;
2285	ch->zw_cqe.done = srpt_zerolength_write_done;
2286	INIT_WORK(&ch->release_work, srpt_release_channel_work);
2287	ch->sport = sport;
2288	if (rdma_cm_id) {
2289	ch->using_rdma_cm = true;
2290	ch->rdma_cm.cm_id = rdma_cm_id;
2291	rdma_cm_id->context = ch;
2292	} else {
2293	ch->ib_cm.cm_id = ib_cm_id;
2294	ib_cm_id->context = ch;
2295	}
2296	/*
2297	* ch->rq_size should be at least as large as the initiator queue
2298	* depth to avoid that the initiator driver has to report QUEUE_FULL
2299	* to the SCSI mid-layer.
2300	*/
2301	ch->rq_size = min(MAX_SRPT_RQ_SIZE, sdev->device->attrs.max_qp_wr);
2302	spin_lock_init(&ch->spinlock);
2303	ch->state = CH_CONNECTING;
2304	INIT_LIST_HEAD(list: &ch->cmd_wait_list);
2305	ch->max_rsp_size = ch->sport->port_attrib.srp_max_rsp_size;
2306
2307	ch->rsp_buf_cache = srpt_cache_get(object_size: ch->max_rsp_size);
2308	if (!ch->rsp_buf_cache)
2309	goto free_ch;
2310
2311	ch->ioctx_ring = (struct srpt_send_ioctx **)
2312	srpt_alloc_ioctx_ring(sdev: ch->sport->sdev, ring_size: ch->rq_size,
2313	ioctx_size: sizeof(*ch->ioctx_ring[0]),
2314	buf_cache: ch->rsp_buf_cache, alignment_offset: 0, dir: DMA_TO_DEVICE);
2315	if (!ch->ioctx_ring) {
2316	pr_err("rejected SRP_LOGIN_REQ because creating a new QP SQ ring failed.\n");
2317	rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
2318	goto free_rsp_cache;
2319	}
2320
2321	for (i = 0; i < ch->rq_size; i++)
2322	ch->ioctx_ring[i]->ch = ch;
2323	if (!sdev->use_srq) {
2324	u16 imm_data_offset = req->req_flags & SRP_IMMED_REQUESTED ?
2325	be16_to_cpu(req->imm_data_offset) : 0;
2326	u16 alignment_offset;
2327	u32 req_sz;
2328
2329	if (req->req_flags & SRP_IMMED_REQUESTED)
2330	pr_debug("imm_data_offset = %d\n",
2331	be16_to_cpu(req->imm_data_offset));
2332	if (imm_data_offset >= sizeof(struct srp_cmd)) {
2333	ch->imm_data_offset = imm_data_offset;
2334	rsp->rsp_flags |= SRP_LOGIN_RSP_IMMED_SUPP;
2335	} else {
2336	ch->imm_data_offset = 0;
2337	}
2338	alignment_offset = round_up(imm_data_offset, 512) -
2339	imm_data_offset;
2340	req_sz = alignment_offset + imm_data_offset + srp_max_req_size;
2341	ch->req_buf_cache = srpt_cache_get(object_size: req_sz);
2342	if (!ch->req_buf_cache)
2343	goto free_rsp_ring;
2344
2345	ch->ioctx_recv_ring = (struct srpt_recv_ioctx **)
2346	srpt_alloc_ioctx_ring(sdev: ch->sport->sdev, ring_size: ch->rq_size,
2347	ioctx_size: sizeof(*ch->ioctx_recv_ring[0]),
2348	buf_cache: ch->req_buf_cache,
2349	alignment_offset,
2350	dir: DMA_FROM_DEVICE);
2351	if (!ch->ioctx_recv_ring) {
2352	pr_err("rejected SRP_LOGIN_REQ because creating a new QP RQ ring failed.\n");
2353	rej->reason =
2354	cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
2355	goto free_recv_cache;
2356	}
2357	for (i = 0; i < ch->rq_size; i++)
2358	INIT_LIST_HEAD(list: &ch->ioctx_recv_ring[i]->wait_list);
2359	}
2360
2361	ret = srpt_create_ch_ib(ch);
2362	if (ret) {
2363	rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
2364	pr_err("rejected SRP_LOGIN_REQ because creating a new RDMA channel failed.\n");
2365	goto free_recv_ring;
2366	}
2367
2368	strscpy(ch->sess_name, src_addr, sizeof(ch->sess_name));
2369	snprintf(buf: i_port_id, size: sizeof(i_port_id), fmt: "0x%016llx%016llx",
2370	be64_to_cpu(*(__be64 *)nexus->i_port_id),
2371	be64_to_cpu(*(__be64 *)(nexus->i_port_id + 8)));
2372
2373	pr_debug("registering src addr %s or i_port_id %s\n", ch->sess_name,
2374	i_port_id);
2375
2376	tag_num = ch->rq_size;
2377	tag_size = 1; /* ib_srpt does not use se_sess->sess_cmd_map */
2378
2379	if (sport->guid_id) {
2380	mutex_lock(&sport->guid_id->mutex);
2381	list_for_each_entry(stpg, &sport->guid_id->tpg_list, entry) {
2382	if (!IS_ERR_OR_NULL(ptr: ch->sess))
2383	break;
2384	ch->sess = target_setup_session(&stpg->tpg, tag_num,
2385	tag_size, prot_op: TARGET_PROT_NORMAL,
2386	ch->sess_name, ch, NULL);
2387	}
2388	mutex_unlock(lock: &sport->guid_id->mutex);
2389	}
2390
2391	if (sport->gid_id) {
2392	mutex_lock(&sport->gid_id->mutex);
2393	list_for_each_entry(stpg, &sport->gid_id->tpg_list, entry) {
2394	if (!IS_ERR_OR_NULL(ptr: ch->sess))
2395	break;
2396	ch->sess = target_setup_session(&stpg->tpg, tag_num,
2397	tag_size, prot_op: TARGET_PROT_NORMAL, i_port_id,
2398	ch, NULL);
2399	if (!IS_ERR_OR_NULL(ptr: ch->sess))
2400	break;
2401	/* Retry without leading "0x" */
2402	ch->sess = target_setup_session(&stpg->tpg, tag_num,
2403	tag_size, prot_op: TARGET_PROT_NORMAL,
2404	i_port_id + 2, ch, NULL);
2405	}
2406	mutex_unlock(lock: &sport->gid_id->mutex);
2407	}
2408
2409	if (IS_ERR_OR_NULL(ptr: ch->sess)) {
2410	WARN_ON_ONCE(ch->sess == NULL);
2411	ret = PTR_ERR(ptr: ch->sess);
2412	ch->sess = NULL;
2413	pr_info("Rejected login for initiator %s: ret = %d.\n",
2414	ch->sess_name, ret);
2415	rej->reason = cpu_to_be32(ret == -ENOMEM ?
2416	SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES :
2417	SRP_LOGIN_REJ_CHANNEL_LIMIT_REACHED);
2418	goto destroy_ib;
2419	}
2420
2421	/*
2422	* Once a session has been created destruction of srpt_rdma_ch objects
2423	* will decrement sport->refcount. Hence increment sport->refcount now.
2424	*/
2425	atomic_inc(v: &sport->refcount);
2426
2427	mutex_lock(&sport->mutex);
2428
2429	if ((req->req_flags & SRP_MTCH_ACTION) == SRP_MULTICHAN_SINGLE) {
2430	struct srpt_rdma_ch *ch2;
2431
2432	list_for_each_entry(ch2, &nexus->ch_list, list) {
2433	if (srpt_disconnect_ch(ch: ch2) < 0)
2434	continue;
2435	pr_info("Relogin - closed existing channel %s\n",
2436	ch2->sess_name);
2437	rsp->rsp_flags |= SRP_LOGIN_RSP_MULTICHAN_TERMINATED;
2438	}
2439	} else {
2440	rsp->rsp_flags |= SRP_LOGIN_RSP_MULTICHAN_MAINTAINED;
2441	}
2442
2443	list_add_tail_rcu(new: &ch->list, head: &nexus->ch_list);
2444
2445	if (!sport->enabled) {
2446	rej->reason = cpu_to_be32(
2447	SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
2448	pr_info("rejected SRP_LOGIN_REQ because target %s_%d is not enabled\n",
2449	dev_name(&sdev->device->dev), port_num);
2450	mutex_unlock(lock: &sport->mutex);
2451	ret = -EINVAL;
2452	goto reject;
2453	}
2454
2455	mutex_unlock(lock: &sport->mutex);
2456
2457	ret = ch->using_rdma_cm ? 0 : srpt_ch_qp_rtr(ch, qp: ch->qp);
2458	if (ret) {
2459	rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
2460	pr_err("rejected SRP_LOGIN_REQ because enabling RTR failed (error code = %d)\n",
2461	ret);
2462	goto reject;
2463	}
2464
2465	pr_debug("Establish connection sess=%p name=%s ch=%p\n", ch->sess,
2466	ch->sess_name, ch);
2467
2468	/* create srp_login_response */
2469	rsp->opcode = SRP_LOGIN_RSP;
2470	rsp->tag = req->tag;
2471	rsp->max_it_iu_len = cpu_to_be32(srp_max_req_size);
2472	rsp->max_ti_iu_len = req->req_it_iu_len;
2473	ch->max_ti_iu_len = it_iu_len;
2474	rsp->buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT |
2475	SRP_BUF_FORMAT_INDIRECT);
2476	rsp->req_lim_delta = cpu_to_be32(ch->rq_size);
2477	atomic_set(v: &ch->req_lim, i: ch->rq_size);
2478	atomic_set(v: &ch->req_lim_delta, i: 0);
2479
2480	/* create cm reply */
2481	if (ch->using_rdma_cm) {
2482	rep_param->rdma_cm.private_data = (void *)rsp;
2483	rep_param->rdma_cm.private_data_len = sizeof(*rsp);
2484	rep_param->rdma_cm.rnr_retry_count = 7;
2485	rep_param->rdma_cm.flow_control = 1;
2486	rep_param->rdma_cm.responder_resources = 4;
2487	rep_param->rdma_cm.initiator_depth = 4;
2488	} else {
2489	rep_param->ib_cm.qp_num = ch->qp->qp_num;
2490	rep_param->ib_cm.private_data = (void *)rsp;
2491	rep_param->ib_cm.private_data_len = sizeof(*rsp);
2492	rep_param->ib_cm.rnr_retry_count = 7;
2493	rep_param->ib_cm.flow_control = 1;
2494	rep_param->ib_cm.failover_accepted = 0;
2495	rep_param->ib_cm.srq = 1;
2496	rep_param->ib_cm.responder_resources = 4;
2497	rep_param->ib_cm.initiator_depth = 4;
2498	}
2499
2500	/*
2501	* Hold the sport mutex while accepting a connection to avoid that
2502	* srpt_disconnect_ch() is invoked concurrently with this code.
2503	*/
2504	mutex_lock(&sport->mutex);
2505	if (sport->enabled && ch->state == CH_CONNECTING) {
2506	if (ch->using_rdma_cm)
2507	ret = rdma_accept(id: rdma_cm_id, conn_param: &rep_param->rdma_cm);
2508	else
2509	ret = ib_send_cm_rep(cm_id: ib_cm_id, param: &rep_param->ib_cm);
2510	} else {
2511	ret = -EINVAL;
2512	}
2513	mutex_unlock(lock: &sport->mutex);
2514
2515	switch (ret) {
2516	case 0:
2517	break;
2518	case -EINVAL:
2519	goto reject;
2520	default:
2521	rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
2522	pr_err("sending SRP_LOGIN_REQ response failed (error code = %d)\n",
2523	ret);
2524	goto reject;
2525	}
2526
2527	goto out;
2528
2529	destroy_ib:
2530	srpt_destroy_ch_ib(ch);
2531
2532	free_recv_ring:
2533	srpt_free_ioctx_ring(ioctx_ring: (struct srpt_ioctx **)ch->ioctx_recv_ring,
2534	sdev: ch->sport->sdev, ring_size: ch->rq_size,
2535	buf_cache: ch->req_buf_cache, dir: DMA_FROM_DEVICE);
2536
2537	free_recv_cache:
2538	srpt_cache_put(c: ch->req_buf_cache);
2539
2540	free_rsp_ring:
2541	srpt_free_ioctx_ring(ioctx_ring: (struct srpt_ioctx **)ch->ioctx_ring,
2542	sdev: ch->sport->sdev, ring_size: ch->rq_size,
2543	buf_cache: ch->rsp_buf_cache, dir: DMA_TO_DEVICE);
2544
2545	free_rsp_cache:
2546	srpt_cache_put(c: ch->rsp_buf_cache);
2547
2548	free_ch:
2549	if (rdma_cm_id)
2550	rdma_cm_id->context = NULL;
2551	else
2552	ib_cm_id->context = NULL;
2553	kfree(objp: ch);
2554	ch = NULL;
2555
2556	WARN_ON_ONCE(ret == 0);
2557
2558	reject:
2559	pr_info("Rejecting login with reason %#x\n", be32_to_cpu(rej->reason));
2560	rej->opcode = SRP_LOGIN_REJ;
2561	rej->tag = req->tag;
2562	rej->buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT |
2563	SRP_BUF_FORMAT_INDIRECT);
2564
2565	if (rdma_cm_id)
2566	rdma_reject(id: rdma_cm_id, private_data: rej, private_data_len: sizeof(*rej),
2567	reason: IB_CM_REJ_CONSUMER_DEFINED);
2568	else
2569	ib_send_cm_rej(cm_id: ib_cm_id, reason: IB_CM_REJ_CONSUMER_DEFINED, NULL, ari_length: 0,
2570	private_data: rej, private_data_len: sizeof(*rej));
2571
2572	if (ch && ch->sess) {
2573	srpt_close_ch(ch);
2574	/*
2575	* Tell the caller not to free cm_id since
2576	* srpt_release_channel_work() will do that.
2577	*/
2578	ret = 0;
2579	}
2580
2581	out:
2582	kfree(objp: rep_param);
2583	kfree(objp: rsp);
2584	kfree(objp: rej);
2585
2586	return ret;
2587	}
2588
2589	static int srpt_ib_cm_req_recv(struct ib_cm_id *cm_id,
2590	const struct ib_cm_req_event_param *param,
2591	void *private_data)
2592	{
2593	char sguid[40];
2594
2595	srpt_format_guid(buf: sguid, size: sizeof(sguid),
2596	guid: &param->primary_path->dgid.global.interface_id);
2597
2598	return srpt_cm_req_recv(sdev: cm_id->context, ib_cm_id: cm_id, NULL, port_num: param->port,
2599	pkey: param->primary_path->pkey,
2600	req: private_data, src_addr: sguid);
2601	}
2602
2603	static int srpt_rdma_cm_req_recv(struct rdma_cm_id *cm_id,
2604	struct rdma_cm_event *event)
2605	{
2606	struct srpt_device *sdev;
2607	struct srp_login_req req;
2608	const struct srp_login_req_rdma *req_rdma;
2609	struct sa_path_rec *path_rec = cm_id->route.path_rec;
2610	char src_addr[40];
2611
2612	sdev = ib_get_client_data(device: cm_id->device, client: &srpt_client);
2613	if (!sdev)
2614	return -ECONNREFUSED;
2615
2616	if (event->param.conn.private_data_len < sizeof(*req_rdma))
2617	return -EINVAL;
2618
2619	/* Transform srp_login_req_rdma into srp_login_req. */
2620	req_rdma = event->param.conn.private_data;
2621	memset(&req, 0, sizeof(req));
2622	req.opcode = req_rdma->opcode;
2623	req.tag = req_rdma->tag;
2624	req.req_it_iu_len = req_rdma->req_it_iu_len;
2625	req.req_buf_fmt = req_rdma->req_buf_fmt;
2626	req.req_flags = req_rdma->req_flags;
2627	memcpy(req.initiator_port_id, req_rdma->initiator_port_id, 16);
2628	memcpy(req.target_port_id, req_rdma->target_port_id, 16);
2629	req.imm_data_offset = req_rdma->imm_data_offset;
2630
2631	snprintf(buf: src_addr, size: sizeof(src_addr), fmt: "%pIS",
2632	&cm_id->route.addr.src_addr);
2633
2634	return srpt_cm_req_recv(sdev, NULL, rdma_cm_id: cm_id, port_num: cm_id->port_num,
2635	pkey: path_rec ? path_rec->pkey : 0, req: &req, src_addr);
2636	}
2637
2638	static void srpt_cm_rej_recv(struct srpt_rdma_ch *ch,
2639	enum ib_cm_rej_reason reason,
2640	const u8 *private_data,
2641	u8 private_data_len)
2642	{
2643	char *priv = NULL;
2644	int i;
2645
2646	if (private_data_len && (priv = kmalloc(private_data_len * 3 + 1,
2647	GFP_KERNEL))) {
2648	for (i = 0; i < private_data_len; i++)
2649	sprintf(buf: priv + 3 * i, fmt: " %02x", private_data[i]);
2650	}
2651	pr_info("Received CM REJ for ch %s-%d; reason %d%s%s.\n",
2652	ch->sess_name, ch->qp->qp_num, reason, private_data_len ?
2653	"; private data" : "", priv ? priv : " (?)");
2654	kfree(objp: priv);
2655	}
2656
2657	/**
2658	* srpt_cm_rtu_recv - process an IB_CM_RTU_RECEIVED or USER_ESTABLISHED event
2659	* @ch: SRPT RDMA channel.
2660	*
2661	* An RTU (ready to use) message indicates that the connection has been
2662	* established and that the recipient may begin transmitting.
2663	*/
2664	static void srpt_cm_rtu_recv(struct srpt_rdma_ch *ch)
2665	{
2666	int ret;
2667
2668	ret = ch->using_rdma_cm ? 0 : srpt_ch_qp_rts(ch, qp: ch->qp);
2669	if (ret < 0) {
2670	pr_err("%s-%d: QP transition to RTS failed\n", ch->sess_name,
2671	ch->qp->qp_num);
2672	srpt_close_ch(ch);
2673	return;
2674	}
2675
2676	/*
2677	* Note: calling srpt_close_ch() if the transition to the LIVE state
2678	* fails is not necessary since that means that that function has
2679	* already been invoked from another thread.
2680	*/
2681	if (!srpt_set_ch_state(ch, new: CH_LIVE)) {
2682	pr_err("%s-%d: channel transition to LIVE state failed\n",
2683	ch->sess_name, ch->qp->qp_num);
2684	return;
2685	}
2686
2687	/* Trigger wait list processing. */
2688	ret = srpt_zerolength_write(ch);
2689	WARN_ONCE(ret < 0, "%d\n", ret);
2690	}
2691
2692	/**
2693	* srpt_cm_handler - IB connection manager callback function
2694	* @cm_id: IB/CM connection identifier.
2695	* @event: IB/CM event.
2696	*
2697	* A non-zero return value will cause the caller destroy the CM ID.
2698	*
2699	* Note: srpt_cm_handler() must only return a non-zero value when transferring
2700	* ownership of the cm_id to a channel by srpt_cm_req_recv() failed. Returning
2701	* a non-zero value in any other case will trigger a race with the
2702	* ib_destroy_cm_id() call in srpt_release_channel().
2703	*/
2704	static int srpt_cm_handler(struct ib_cm_id *cm_id,
2705	const struct ib_cm_event *event)
2706	{
2707	struct srpt_rdma_ch *ch = cm_id->context;
2708	int ret;
2709
2710	ret = 0;
2711	switch (event->event) {
2712	case IB_CM_REQ_RECEIVED:
2713	ret = srpt_ib_cm_req_recv(cm_id, param: &event->param.req_rcvd,
2714	private_data: event->private_data);
2715	break;
2716	case IB_CM_REJ_RECEIVED:
2717	srpt_cm_rej_recv(ch, reason: event->param.rej_rcvd.reason,
2718	private_data: event->private_data,
2719	private_data_len: IB_CM_REJ_PRIVATE_DATA_SIZE);
2720	break;
2721	case IB_CM_RTU_RECEIVED:
2722	case IB_CM_USER_ESTABLISHED:
2723	srpt_cm_rtu_recv(ch);
2724	break;
2725	case IB_CM_DREQ_RECEIVED:
2726	srpt_disconnect_ch(ch);
2727	break;
2728	case IB_CM_DREP_RECEIVED:
2729	pr_info("Received CM DREP message for ch %s-%d.\n",
2730	ch->sess_name, ch->qp->qp_num);
2731	srpt_close_ch(ch);
2732	break;
2733	case IB_CM_TIMEWAIT_EXIT:
2734	pr_info("Received CM TimeWait exit for ch %s-%d.\n",
2735	ch->sess_name, ch->qp->qp_num);
2736	srpt_close_ch(ch);
2737	break;
2738	case IB_CM_REP_ERROR:
2739	pr_info("Received CM REP error for ch %s-%d.\n", ch->sess_name,
2740	ch->qp->qp_num);
2741	break;
2742	case IB_CM_DREQ_ERROR:
2743	pr_info("Received CM DREQ ERROR event.\n");
2744	break;
2745	case IB_CM_MRA_RECEIVED:
2746	pr_info("Received CM MRA event\n");
2747	break;
2748	default:
2749	pr_err("received unrecognized CM event %d\n", event->event);
2750	break;
2751	}
2752
2753	return ret;
2754	}
2755
2756	static int srpt_rdma_cm_handler(struct rdma_cm_id *cm_id,
2757	struct rdma_cm_event *event)
2758	{
2759	struct srpt_rdma_ch *ch = cm_id->context;
2760	int ret = 0;
2761
2762	switch (event->event) {
2763	case RDMA_CM_EVENT_CONNECT_REQUEST:
2764	ret = srpt_rdma_cm_req_recv(cm_id, event);
2765	break;
2766	case RDMA_CM_EVENT_REJECTED:
2767	srpt_cm_rej_recv(ch, reason: event->status,
2768	private_data: event->param.conn.private_data,
2769	private_data_len: event->param.conn.private_data_len);
2770	break;
2771	case RDMA_CM_EVENT_ESTABLISHED:
2772	srpt_cm_rtu_recv(ch);
2773	break;
2774	case RDMA_CM_EVENT_DISCONNECTED:
2775	if (ch->state < CH_DISCONNECTING)
2776	srpt_disconnect_ch(ch);
2777	else
2778	srpt_close_ch(ch);
2779	break;
2780	case RDMA_CM_EVENT_TIMEWAIT_EXIT:
2781	srpt_close_ch(ch);
2782	break;
2783	case RDMA_CM_EVENT_UNREACHABLE:
2784	pr_info("Received CM REP error for ch %s-%d.\n", ch->sess_name,
2785	ch->qp->qp_num);
2786	break;
2787	case RDMA_CM_EVENT_DEVICE_REMOVAL:
2788	case RDMA_CM_EVENT_ADDR_CHANGE:
2789	break;
2790	default:
2791	pr_err("received unrecognized RDMA CM event %d\n",
2792	event->event);
2793	break;
2794	}
2795
2796	return ret;
2797	}
2798
2799	/*
2800	* srpt_write_pending - Start data transfer from initiator to target (write).
2801	*/
2802	static int srpt_write_pending(struct se_cmd *se_cmd)
2803	{
2804	struct srpt_send_ioctx *ioctx =
2805	container_of(se_cmd, struct srpt_send_ioctx, cmd);
2806	struct srpt_rdma_ch *ch = ioctx->ch;
2807	struct ib_send_wr *first_wr = NULL;
2808	struct ib_cqe *cqe = &ioctx->rdma_cqe;
2809	enum srpt_command_state new_state;
2810	int ret, i;
2811
2812	if (ioctx->recv_ioctx) {
2813	srpt_set_cmd_state(ioctx, new: SRPT_STATE_DATA_IN);
2814	target_execute_cmd(cmd: &ioctx->cmd);
2815	return 0;
2816	}
2817
2818	new_state = srpt_set_cmd_state(ioctx, new: SRPT_STATE_NEED_DATA);
2819	WARN_ON(new_state == SRPT_STATE_DONE);
2820
2821	if (atomic_sub_return(i: ioctx->n_rdma, v: &ch->sq_wr_avail) < 0) {
2822	pr_warn("%s: IB send queue full (needed %d)\n",
2823	__func__, ioctx->n_rdma);
2824	ret = -ENOMEM;
2825	goto out_undo;
2826	}
2827
2828	cqe->done = srpt_rdma_read_done;
2829	for (i = ioctx->n_rw_ctx - 1; i >= 0; i--) {
2830	struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i];
2831
2832	first_wr = rdma_rw_ctx_wrs(ctx: &ctx->rw, qp: ch->qp, port_num: ch->sport->port,
2833	cqe, chain_wr: first_wr);
2834	cqe = NULL;
2835	}
2836
2837	ret = ib_post_send(qp: ch->qp, send_wr: first_wr, NULL);
2838	if (ret) {
2839	pr_err("%s: ib_post_send() returned %d for %d (avail: %d)\n",
2840	__func__, ret, ioctx->n_rdma,
2841	atomic_read(&ch->sq_wr_avail));
2842	goto out_undo;
2843	}
2844
2845	return 0;
2846	out_undo:
2847	atomic_add(i: ioctx->n_rdma, v: &ch->sq_wr_avail);
2848	return ret;
2849	}
2850
2851	static u8 tcm_to_srp_tsk_mgmt_status(const int tcm_mgmt_status)
2852	{
2853	switch (tcm_mgmt_status) {
2854	case TMR_FUNCTION_COMPLETE:
2855	return SRP_TSK_MGMT_SUCCESS;
2856	case TMR_FUNCTION_REJECTED:
2857	return SRP_TSK_MGMT_FUNC_NOT_SUPP;
2858	}
2859	return SRP_TSK_MGMT_FAILED;
2860	}
2861
2862	/**
2863	* srpt_queue_response - transmit the response to a SCSI command
2864	* @cmd: SCSI target command.
2865	*
2866	* Callback function called by the TCM core. Must not block since it can be
2867	* invoked on the context of the IB completion handler.
2868	*/
2869	static void srpt_queue_response(struct se_cmd *cmd)
2870	{
2871	struct srpt_send_ioctx *ioctx =
2872	container_of(cmd, struct srpt_send_ioctx, cmd);
2873	struct srpt_rdma_ch *ch = ioctx->ch;
2874	struct srpt_device *sdev = ch->sport->sdev;
2875	struct ib_send_wr send_wr, *first_wr = &send_wr;
2876	struct ib_sge sge;
2877	enum srpt_command_state state;
2878	int resp_len, ret, i;
2879	u8 srp_tm_status;
2880
2881	state = ioctx->state;
2882	switch (state) {
2883	case SRPT_STATE_NEW:
2884	case SRPT_STATE_DATA_IN:
2885	ioctx->state = SRPT_STATE_CMD_RSP_SENT;
2886	break;
2887	case SRPT_STATE_MGMT:
2888	ioctx->state = SRPT_STATE_MGMT_RSP_SENT;
2889	break;
2890	default:
2891	WARN(true, "ch %p; cmd %d: unexpected command state %d\n",
2892	ch, ioctx->ioctx.index, ioctx->state);
2893	break;
2894	}
2895
2896	if (WARN_ON_ONCE(state == SRPT_STATE_CMD_RSP_SENT))
2897	return;
2898
2899	/* For read commands, transfer the data to the initiator. */
2900	if (ioctx->cmd.data_direction == DMA_FROM_DEVICE &&
2901	ioctx->cmd.data_length &&
2902	!ioctx->queue_status_only) {
2903	for (i = ioctx->n_rw_ctx - 1; i >= 0; i--) {
2904	struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i];
2905
2906	first_wr = rdma_rw_ctx_wrs(ctx: &ctx->rw, qp: ch->qp,
2907	port_num: ch->sport->port, NULL, chain_wr: first_wr);
2908	}
2909	}
2910
2911	if (state != SRPT_STATE_MGMT)
2912	resp_len = srpt_build_cmd_rsp(ch, ioctx, tag: ioctx->cmd.tag,
2913	status: cmd->scsi_status);
2914	else {
2915	srp_tm_status
2916	= tcm_to_srp_tsk_mgmt_status(tcm_mgmt_status: cmd->se_tmr_req->response);
2917	resp_len = srpt_build_tskmgmt_rsp(ch, ioctx, rsp_code: srp_tm_status,
2918	tag: ioctx->cmd.tag);
2919	}
2920
2921	atomic_inc(v: &ch->req_lim);
2922
2923	if (unlikely(atomic_sub_return(1 + ioctx->n_rdma,
2924	&ch->sq_wr_avail) < 0)) {
2925	pr_warn("%s: IB send queue full (needed %d)\n",
2926	__func__, ioctx->n_rdma);
2927	goto out;
2928	}
2929
2930	ib_dma_sync_single_for_device(dev: sdev->device, addr: ioctx->ioctx.dma, size: resp_len,
2931	dir: DMA_TO_DEVICE);
2932
2933	sge.addr = ioctx->ioctx.dma;
2934	sge.length = resp_len;
2935	sge.lkey = sdev->lkey;
2936
2937	ioctx->ioctx.cqe.done = srpt_send_done;
2938	send_wr.next = NULL;
2939	send_wr.wr_cqe = &ioctx->ioctx.cqe;
2940	send_wr.sg_list = &sge;
2941	send_wr.num_sge = 1;
2942	send_wr.opcode = IB_WR_SEND;
2943	send_wr.send_flags = IB_SEND_SIGNALED;
2944
2945	ret = ib_post_send(qp: ch->qp, send_wr: first_wr, NULL);
2946	if (ret < 0) {
2947	pr_err("%s: sending cmd response failed for tag %llu (%d)\n",
2948	__func__, ioctx->cmd.tag, ret);
2949	goto out;
2950	}
2951
2952	return;
2953
2954	out:
2955	atomic_add(i: 1 + ioctx->n_rdma, v: &ch->sq_wr_avail);
2956	atomic_dec(v: &ch->req_lim);
2957	srpt_set_cmd_state(ioctx, new: SRPT_STATE_DONE);
2958	target_put_sess_cmd(&ioctx->cmd);
2959	}
2960
2961	static int srpt_queue_data_in(struct se_cmd *cmd)
2962	{
2963	srpt_queue_response(cmd);
2964	return 0;
2965	}
2966
2967	static void srpt_queue_tm_rsp(struct se_cmd *cmd)
2968	{
2969	srpt_queue_response(cmd);
2970	}
2971
2972	/*
2973	* This function is called for aborted commands if no response is sent to the
2974	* initiator. Make sure that the credits freed by aborting a command are
2975	* returned to the initiator the next time a response is sent by incrementing
2976	* ch->req_lim_delta.
2977	*/
2978	static void srpt_aborted_task(struct se_cmd *cmd)
2979	{
2980	struct srpt_send_ioctx *ioctx = container_of(cmd,
2981	struct srpt_send_ioctx, cmd);
2982	struct srpt_rdma_ch *ch = ioctx->ch;
2983
2984	atomic_inc(v: &ch->req_lim_delta);
2985	}
2986
2987	static int srpt_queue_status(struct se_cmd *cmd)
2988	{
2989	struct srpt_send_ioctx *ioctx;
2990
2991	ioctx = container_of(cmd, struct srpt_send_ioctx, cmd);
2992	BUG_ON(ioctx->sense_data != cmd->sense_buffer);
2993	if (cmd->se_cmd_flags &
2994	(SCF_TRANSPORT_TASK_SENSE | SCF_EMULATED_TASK_SENSE))
2995	WARN_ON(cmd->scsi_status != SAM_STAT_CHECK_CONDITION);
2996	ioctx->queue_status_only = true;
2997	srpt_queue_response(cmd);
2998	return 0;
2999	}
3000
3001	static void srpt_refresh_port_work(struct work_struct *work)
3002	{
3003	struct srpt_port *sport = container_of(work, struct srpt_port, work);
3004
3005	srpt_refresh_port(sport);
3006	}
3007
3008	/**
3009	* srpt_release_sport - disable login and wait for associated channels
3010	* @sport: SRPT HCA port.
3011	*/
3012	static int srpt_release_sport(struct srpt_port *sport)
3013	{
3014	DECLARE_COMPLETION_ONSTACK(c);
3015	struct srpt_nexus *nexus, *next_n;
3016	struct srpt_rdma_ch *ch;
3017
3018	WARN_ON_ONCE(irqs_disabled());
3019
3020	sport->freed_channels = &c;
3021
3022	mutex_lock(&sport->mutex);
3023	srpt_set_enabled(sport, enabled: false);
3024	mutex_unlock(lock: &sport->mutex);
3025
3026	while (atomic_read(v: &sport->refcount) > 0 &&
3027	wait_for_completion_timeout(x: &c, timeout: 5 * HZ) <= 0) {
3028	pr_info("%s_%d: waiting for unregistration of %d sessions ...\n",
3029	dev_name(&sport->sdev->device->dev), sport->port,
3030	atomic_read(&sport->refcount));
3031	rcu_read_lock();
3032	list_for_each_entry(nexus, &sport->nexus_list, entry) {
3033	list_for_each_entry(ch, &nexus->ch_list, list) {
3034	pr_info("%s-%d: state %s\n",
3035	ch->sess_name, ch->qp->qp_num,
3036	get_ch_state_name(ch->state));
3037	}
3038	}
3039	rcu_read_unlock();
3040	}
3041
3042	mutex_lock(&sport->mutex);
3043	list_for_each_entry_safe(nexus, next_n, &sport->nexus_list, entry) {
3044	list_del(entry: &nexus->entry);
3045	kfree_rcu(nexus, rcu);
3046	}
3047	mutex_unlock(lock: &sport->mutex);
3048
3049	return 0;
3050	}
3051
3052	struct port_and_port_id {
3053	struct srpt_port *sport;
3054	struct srpt_port_id **port_id;
3055	};
3056
3057	static struct port_and_port_id __srpt_lookup_port(const char *name)
3058	{
3059	struct ib_device *dev;
3060	struct srpt_device *sdev;
3061	struct srpt_port *sport;
3062	int i;
3063
3064	list_for_each_entry(sdev, &srpt_dev_list, list) {
3065	dev = sdev->device;
3066	if (!dev)
3067	continue;
3068
3069	for (i = 0; i < dev->phys_port_cnt; i++) {
3070	sport = &sdev->port[i];
3071
3072	if (strcmp(sport->guid_name, name) == 0) {
3073	kref_get(kref: &sdev->refcnt);
3074	return (struct port_and_port_id){
3075	sport, &sport->guid_id};
3076	}
3077	if (strcmp(sport->gid_name, name) == 0) {
3078	kref_get(kref: &sdev->refcnt);
3079	return (struct port_and_port_id){
3080	sport, &sport->gid_id};
3081	}
3082	}
3083	}
3084
3085	return (struct port_and_port_id){};
3086	}
3087
3088	/**
3089	* srpt_lookup_port() - Look up an RDMA port by name
3090	* @name: ASCII port name
3091	*
3092	* Increments the RDMA port reference count if an RDMA port pointer is returned.
3093	* The caller must drop that reference count by calling srpt_port_put_ref().
3094	*/
3095	static struct port_and_port_id srpt_lookup_port(const char *name)
3096	{
3097	struct port_and_port_id papi;
3098
3099	spin_lock(lock: &srpt_dev_lock);
3100	papi = __srpt_lookup_port(name);
3101	spin_unlock(lock: &srpt_dev_lock);
3102
3103	return papi;
3104	}
3105
3106	static void srpt_free_srq(struct srpt_device *sdev)
3107	{
3108	if (!sdev->srq)
3109	return;
3110
3111	ib_destroy_srq(srq: sdev->srq);
3112	srpt_free_ioctx_ring(ioctx_ring: (struct srpt_ioctx **)sdev->ioctx_ring, sdev,
3113	ring_size: sdev->srq_size, buf_cache: sdev->req_buf_cache,
3114	dir: DMA_FROM_DEVICE);
3115	srpt_cache_put(c: sdev->req_buf_cache);
3116	sdev->srq = NULL;
3117	}
3118
3119	static int srpt_alloc_srq(struct srpt_device *sdev)
3120	{
3121	struct ib_srq_init_attr srq_attr = {
3122	.event_handler = srpt_srq_event,
3123	.srq_context = (void *)sdev,
3124	.attr.max_wr = sdev->srq_size,
3125	.attr.max_sge = 1,
3126	.srq_type = IB_SRQT_BASIC,
3127	};
3128	struct ib_device *device = sdev->device;
3129	struct ib_srq *srq;
3130	int i;
3131
3132	WARN_ON_ONCE(sdev->srq);
3133	srq = ib_create_srq(pd: sdev->pd, srq_init_attr: &srq_attr);
3134	if (IS_ERR(ptr: srq)) {
3135	pr_debug("ib_create_srq() failed: %pe\n", srq);
3136	return PTR_ERR(ptr: srq);
3137	}
3138
3139	pr_debug("create SRQ #wr= %d max_allow=%d dev= %s\n", sdev->srq_size,
3140	sdev->device->attrs.max_srq_wr, dev_name(&device->dev));
3141
3142	sdev->req_buf_cache = srpt_cache_get(object_size: srp_max_req_size);
3143	if (!sdev->req_buf_cache)
3144	goto free_srq;
3145
3146	sdev->ioctx_ring = (struct srpt_recv_ioctx **)
3147	srpt_alloc_ioctx_ring(sdev, ring_size: sdev->srq_size,
3148	ioctx_size: sizeof(*sdev->ioctx_ring[0]),
3149	buf_cache: sdev->req_buf_cache, alignment_offset: 0, dir: DMA_FROM_DEVICE);
3150	if (!sdev->ioctx_ring)
3151	goto free_cache;
3152
3153	sdev->use_srq = true;
3154	sdev->srq = srq;
3155
3156	for (i = 0; i < sdev->srq_size; ++i) {
3157	INIT_LIST_HEAD(list: &sdev->ioctx_ring[i]->wait_list);
3158	srpt_post_recv(sdev, NULL, ioctx: sdev->ioctx_ring[i]);
3159	}
3160
3161	return 0;
3162
3163	free_cache:
3164	srpt_cache_put(c: sdev->req_buf_cache);
3165
3166	free_srq:
3167	ib_destroy_srq(srq);
3168	return -ENOMEM;
3169	}
3170
3171	static int srpt_use_srq(struct srpt_device *sdev, bool use_srq)
3172	{
3173	struct ib_device *device = sdev->device;
3174	int ret = 0;
3175
3176	if (!use_srq) {
3177	srpt_free_srq(sdev);
3178	sdev->use_srq = false;
3179	} else if (use_srq && !sdev->srq) {
3180	ret = srpt_alloc_srq(sdev);
3181	}
3182	pr_debug("%s(%s): use_srq = %d; ret = %d\n", __func__,
3183	dev_name(&device->dev), sdev->use_srq, ret);
3184	return ret;
3185	}
3186
3187	static void srpt_free_sdev(struct kref *refcnt)
3188	{
3189	struct srpt_device *sdev = container_of(refcnt, typeof(*sdev), refcnt);
3190
3191	kfree(objp: sdev);
3192	}
3193
3194	static void srpt_sdev_put(struct srpt_device *sdev)
3195	{
3196	kref_put(kref: &sdev->refcnt, release: srpt_free_sdev);
3197	}
3198
3199	/**
3200	* srpt_add_one - InfiniBand device addition callback function
3201	* @device: Describes a HCA.
3202	*/
3203	static int srpt_add_one(struct ib_device *device)
3204	{
3205	struct srpt_device *sdev;
3206	struct srpt_port *sport;
3207	int ret;
3208	u32 i;
3209
3210	pr_debug("device = %p\n", device);
3211
3212	sdev = kzalloc(struct_size(sdev, port, device->phys_port_cnt),
3213	GFP_KERNEL);
3214	if (!sdev)
3215	return -ENOMEM;
3216
3217	kref_init(kref: &sdev->refcnt);
3218	sdev->device = device;
3219	mutex_init(&sdev->sdev_mutex);
3220
3221	sdev->pd = ib_alloc_pd(device, 0);
3222	if (IS_ERR(ptr: sdev->pd)) {
3223	ret = PTR_ERR(ptr: sdev->pd);
3224	goto free_dev;
3225	}
3226
3227	sdev->lkey = sdev->pd->local_dma_lkey;
3228
3229	sdev->srq_size = min(srpt_srq_size, sdev->device->attrs.max_srq_wr);
3230
3231	srpt_use_srq(sdev, use_srq: sdev->port[0].port_attrib.use_srq);
3232
3233	if (!srpt_service_guid)
3234	srpt_service_guid = be64_to_cpu(device->node_guid);
3235
3236	if (rdma_port_get_link_layer(device, port_num: 1) == IB_LINK_LAYER_INFINIBAND)
3237	sdev->cm_id = ib_create_cm_id(device, cm_handler: srpt_cm_handler, context: sdev);
3238	if (IS_ERR(ptr: sdev->cm_id)) {
3239	pr_info("ib_create_cm_id() failed: %pe\n", sdev->cm_id);
3240	ret = PTR_ERR(ptr: sdev->cm_id);
3241	sdev->cm_id = NULL;
3242	if (!rdma_cm_id)
3243	goto err_ring;
3244	}
3245
3246	/* print out target login information */
3247	pr_debug("Target login info: id_ext=%016llx,ioc_guid=%016llx,pkey=ffff,service_id=%016llx\n",
3248	srpt_service_guid, srpt_service_guid, srpt_service_guid);
3249
3250	/*
3251	* We do not have a consistent service_id (ie. also id_ext of target_id)
3252	* to identify this target. We currently use the guid of the first HCA
3253	* in the system as service_id; therefore, the target_id will change
3254	* if this HCA is gone bad and replaced by different HCA
3255	*/
3256	ret = sdev->cm_id ?
3257	ib_cm_listen(cm_id: sdev->cm_id, cpu_to_be64(srpt_service_guid)) :
3258	0;
3259	if (ret < 0) {
3260	pr_err("ib_cm_listen() failed: %d (cm_id state = %d)\n", ret,
3261	sdev->cm_id->state);
3262	goto err_cm;
3263	}
3264
3265	INIT_IB_EVENT_HANDLER(&sdev->event_handler, sdev->device,
3266	srpt_event_handler);
3267
3268	for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
3269	sport = &sdev->port[i - 1];
3270	INIT_LIST_HEAD(list: &sport->nexus_list);
3271	mutex_init(&sport->mutex);
3272	sport->sdev = sdev;
3273	sport->port = i;
3274	sport->port_attrib.srp_max_rdma_size = DEFAULT_MAX_RDMA_SIZE;
3275	sport->port_attrib.srp_max_rsp_size = DEFAULT_MAX_RSP_SIZE;
3276	sport->port_attrib.srp_sq_size = DEF_SRPT_SQ_SIZE;
3277	sport->port_attrib.use_srq = false;
3278	INIT_WORK(&sport->work, srpt_refresh_port_work);
3279
3280	ret = srpt_refresh_port(sport);
3281	if (ret) {
3282	pr_err("MAD registration failed for %s-%d.\n",
3283	dev_name(&sdev->device->dev), i);
3284	i--;
3285	goto err_port;
3286	}
3287	}
3288
3289	ib_register_event_handler(event_handler: &sdev->event_handler);
3290	spin_lock(lock: &srpt_dev_lock);
3291	list_add_tail(new: &sdev->list, head: &srpt_dev_list);
3292	spin_unlock(lock: &srpt_dev_lock);
3293
3294	ib_set_client_data(device, client: &srpt_client, data: sdev);
3295	pr_debug("added %s.\n", dev_name(&device->dev));
3296	return 0;
3297
3298	err_port:
3299	srpt_unregister_mad_agent(sdev, port_cnt: i);
3300	err_cm:
3301	if (sdev->cm_id)
3302	ib_destroy_cm_id(cm_id: sdev->cm_id);
3303	err_ring:
3304	srpt_free_srq(sdev);
3305	ib_dealloc_pd(pd: sdev->pd);
3306	free_dev:
3307	srpt_sdev_put(sdev);
3308	pr_info("%s(%s) failed.\n", __func__, dev_name(&device->dev));
3309	return ret;
3310	}
3311
3312	/**
3313	* srpt_remove_one - InfiniBand device removal callback function
3314	* @device: Describes a HCA.
3315	* @client_data: The value passed as the third argument to ib_set_client_data().
3316	*/
3317	static void srpt_remove_one(struct ib_device *device, void *client_data)
3318	{
3319	struct srpt_device *sdev = client_data;
3320	int i;
3321
3322	srpt_unregister_mad_agent(sdev, port_cnt: sdev->device->phys_port_cnt);
3323
3324	ib_unregister_event_handler(event_handler: &sdev->event_handler);
3325
3326	/* Cancel any work queued by the just unregistered IB event handler. */
3327	for (i = 0; i < sdev->device->phys_port_cnt; i++)
3328	cancel_work_sync(work: &sdev->port[i].work);
3329
3330	if (sdev->cm_id)
3331	ib_destroy_cm_id(cm_id: sdev->cm_id);
3332
3333	ib_set_client_data(device, client: &srpt_client, NULL);
3334
3335	/*
3336	* Unregistering a target must happen after destroying sdev->cm_id
3337	* such that no new SRP_LOGIN_REQ information units can arrive while
3338	* destroying the target.
3339	*/
3340	spin_lock(lock: &srpt_dev_lock);
3341	list_del(entry: &sdev->list);
3342	spin_unlock(lock: &srpt_dev_lock);
3343
3344	for (i = 0; i < sdev->device->phys_port_cnt; i++)
3345	srpt_release_sport(sport: &sdev->port[i]);
3346
3347	srpt_free_srq(sdev);
3348
3349	ib_dealloc_pd(pd: sdev->pd);
3350
3351	srpt_sdev_put(sdev);
3352	}
3353
3354	static struct ib_client srpt_client = {
3355	.name = DRV_NAME,
3356	.add = srpt_add_one,
3357	.remove = srpt_remove_one
3358	};
3359
3360	static int srpt_check_true(struct se_portal_group *se_tpg)
3361	{
3362	return 1;
3363	}
3364
3365	static struct srpt_port *srpt_tpg_to_sport(struct se_portal_group *tpg)
3366	{
3367	return tpg->se_tpg_wwn->priv;
3368	}
3369
3370	static struct srpt_port_id *srpt_wwn_to_sport_id(struct se_wwn *wwn)
3371	{
3372	struct srpt_port *sport = wwn->priv;
3373
3374	if (sport->guid_id && &sport->guid_id->wwn == wwn)
3375	return sport->guid_id;
3376	if (sport->gid_id && &sport->gid_id->wwn == wwn)
3377	return sport->gid_id;
3378	WARN_ON_ONCE(true);
3379	return NULL;
3380	}
3381
3382	static char *srpt_get_fabric_wwn(struct se_portal_group *tpg)
3383	{
3384	struct srpt_tpg *stpg = container_of(tpg, typeof(*stpg), tpg);
3385
3386	return stpg->sport_id->name;
3387	}
3388
3389	static u16 srpt_get_tag(struct se_portal_group *tpg)
3390	{
3391	return 1;
3392	}
3393
3394	static void srpt_release_cmd(struct se_cmd *se_cmd)
3395	{
3396	struct srpt_send_ioctx *ioctx = container_of(se_cmd,
3397	struct srpt_send_ioctx, cmd);
3398	struct srpt_rdma_ch *ch = ioctx->ch;
3399	struct srpt_recv_ioctx *recv_ioctx = ioctx->recv_ioctx;
3400
3401	WARN_ON_ONCE(ioctx->state != SRPT_STATE_DONE &&
3402	!(ioctx->cmd.transport_state & CMD_T_ABORTED));
3403
3404	if (recv_ioctx) {
3405	WARN_ON_ONCE(!list_empty(&recv_ioctx->wait_list));
3406	ioctx->recv_ioctx = NULL;
3407	srpt_post_recv(sdev: ch->sport->sdev, ch, ioctx: recv_ioctx);
3408	}
3409
3410	if (ioctx->n_rw_ctx) {
3411	srpt_free_rw_ctxs(ch, ioctx);
3412	ioctx->n_rw_ctx = 0;
3413	}
3414
3415	target_free_tag(sess: se_cmd->se_sess, cmd: se_cmd);
3416	}
3417
3418	/**
3419	* srpt_close_session - forcibly close a session
3420	* @se_sess: SCSI target session.
3421	*
3422	* Callback function invoked by the TCM core to clean up sessions associated
3423	* with a node ACL when the user invokes
3424	* rmdir /sys/kernel/config/target/$driver/$port/$tpg/acls/$i_port_id
3425	*/
3426	static void srpt_close_session(struct se_session *se_sess)
3427	{
3428	struct srpt_rdma_ch *ch = se_sess->fabric_sess_ptr;
3429
3430	srpt_disconnect_ch_sync(ch);
3431	}
3432
3433	/* Note: only used from inside debug printk's by the TCM core. */
3434	static int srpt_get_tcm_cmd_state(struct se_cmd *se_cmd)
3435	{
3436	struct srpt_send_ioctx *ioctx;
3437
3438	ioctx = container_of(se_cmd, struct srpt_send_ioctx, cmd);
3439	return ioctx->state;
3440	}
3441
3442	static int srpt_parse_guid(u64 *guid, const char *name)
3443	{
3444	u16 w[4];
3445	int ret = -EINVAL;
3446
3447	if (sscanf(name, "%hx:%hx:%hx:%hx", &w[0], &w[1], &w[2], &w[3]) != 4)
3448	goto out;
3449	*guid = get_unaligned_be64(p: w);
3450	ret = 0;
3451	out:
3452	return ret;
3453	}
3454
3455	/**
3456	* srpt_parse_i_port_id - parse an initiator port ID
3457	* @name: ASCII representation of a 128-bit initiator port ID.
3458	* @i_port_id: Binary 128-bit port ID.
3459	*/
3460	static int srpt_parse_i_port_id(u8 i_port_id[16], const char *name)
3461	{
3462	const char *p;
3463	unsigned len, count, leading_zero_bytes;
3464	int ret;
3465
3466	p = name;
3467	if (strncasecmp(s1: p, s2: "0x", n: 2) == 0)
3468	p += 2;
3469	ret = -EINVAL;
3470	len = strlen(p);
3471	if (len % 2)
3472	goto out;
3473	count = min(len / 2, 16U);
3474	leading_zero_bytes = 16 - count;
3475	memset(i_port_id, 0, leading_zero_bytes);
3476	ret = hex2bin(dst: i_port_id + leading_zero_bytes, src: p, count);
3477
3478	out:
3479	return ret;
3480	}
3481
3482	/*
3483	* configfs callback function invoked for mkdir
3484	* /sys/kernel/config/target/$driver/$port/$tpg/acls/$i_port_id
3485	*
3486	* i_port_id must be an initiator port GUID, GID or IP address. See also the
3487	* target_alloc_session() calls in this driver. Examples of valid initiator
3488	* port IDs:
3489	* 0x0000000000000000505400fffe4a0b7b
3490	* 0000000000000000505400fffe4a0b7b
3491	* 5054:00ff:fe4a:0b7b
3492	* 192.168.122.76
3493	*/
3494	static int srpt_init_nodeacl(struct se_node_acl *se_nacl, const char *name)
3495	{
3496	struct sockaddr_storage sa;
3497	u64 guid;
3498	u8 i_port_id[16];
3499	int ret;
3500
3501	ret = srpt_parse_guid(guid: &guid, name);
3502	if (ret < 0)
3503	ret = srpt_parse_i_port_id(i_port_id, name);
3504	if (ret < 0)
3505	ret = inet_pton_with_scope(net: &init_net, AF_UNSPEC, src: name, NULL,
3506	addr: &sa);
3507	if (ret < 0)
3508	pr_err("invalid initiator port ID %s\n", name);
3509	return ret;
3510	}
3511
3512	static ssize_t srpt_tpg_attrib_srp_max_rdma_size_show(struct config_item *item,
3513	char *page)
3514	{
3515	struct se_portal_group *se_tpg = attrib_to_tpg(item);
3516	struct srpt_port *sport = srpt_tpg_to_sport(tpg: se_tpg);
3517
3518	return sysfs_emit(buf: page, fmt: "%u\n", sport->port_attrib.srp_max_rdma_size);
3519	}
3520
3521	static ssize_t srpt_tpg_attrib_srp_max_rdma_size_store(struct config_item *item,
3522	const char *page, size_t count)
3523	{
3524	struct se_portal_group *se_tpg = attrib_to_tpg(item);
3525	struct srpt_port *sport = srpt_tpg_to_sport(tpg: se_tpg);
3526	unsigned long val;
3527	int ret;
3528
3529	ret = kstrtoul(s: page, base: 0, res: &val);
3530	if (ret < 0) {
3531	pr_err("kstrtoul() failed with ret: %d\n", ret);
3532	return -EINVAL;
3533	}
3534	if (val > MAX_SRPT_RDMA_SIZE) {
3535	pr_err("val: %lu exceeds MAX_SRPT_RDMA_SIZE: %d\n", val,
3536	MAX_SRPT_RDMA_SIZE);
3537	return -EINVAL;
3538	}
3539	if (val < DEFAULT_MAX_RDMA_SIZE) {
3540	pr_err("val: %lu smaller than DEFAULT_MAX_RDMA_SIZE: %d\n",
3541	val, DEFAULT_MAX_RDMA_SIZE);
3542	return -EINVAL;
3543	}
3544	sport->port_attrib.srp_max_rdma_size = val;
3545
3546	return count;
3547	}
3548
3549	static ssize_t srpt_tpg_attrib_srp_max_rsp_size_show(struct config_item *item,
3550	char *page)
3551	{
3552	struct se_portal_group *se_tpg = attrib_to_tpg(item);
3553	struct srpt_port *sport = srpt_tpg_to_sport(tpg: se_tpg);
3554
3555	return sysfs_emit(buf: page, fmt: "%u\n", sport->port_attrib.srp_max_rsp_size);
3556	}
3557
3558	static ssize_t srpt_tpg_attrib_srp_max_rsp_size_store(struct config_item *item,
3559	const char *page, size_t count)
3560	{
3561	struct se_portal_group *se_tpg = attrib_to_tpg(item);
3562	struct srpt_port *sport = srpt_tpg_to_sport(tpg: se_tpg);
3563	unsigned long val;
3564	int ret;
3565
3566	ret = kstrtoul(s: page, base: 0, res: &val);
3567	if (ret < 0) {
3568	pr_err("kstrtoul() failed with ret: %d\n", ret);
3569	return -EINVAL;
3570	}
3571	if (val > MAX_SRPT_RSP_SIZE) {
3572	pr_err("val: %lu exceeds MAX_SRPT_RSP_SIZE: %d\n", val,
3573	MAX_SRPT_RSP_SIZE);
3574	return -EINVAL;
3575	}
3576	if (val < MIN_MAX_RSP_SIZE) {
3577	pr_err("val: %lu smaller than MIN_MAX_RSP_SIZE: %d\n", val,
3578	MIN_MAX_RSP_SIZE);
3579	return -EINVAL;
3580	}
3581	sport->port_attrib.srp_max_rsp_size = val;
3582
3583	return count;
3584	}
3585
3586	static ssize_t srpt_tpg_attrib_srp_sq_size_show(struct config_item *item,
3587	char *page)
3588	{
3589	struct se_portal_group *se_tpg = attrib_to_tpg(item);
3590	struct srpt_port *sport = srpt_tpg_to_sport(tpg: se_tpg);
3591
3592	return sysfs_emit(buf: page, fmt: "%u\n", sport->port_attrib.srp_sq_size);
3593	}
3594
3595	static ssize_t srpt_tpg_attrib_srp_sq_size_store(struct config_item *item,
3596	const char *page, size_t count)
3597	{
3598	struct se_portal_group *se_tpg = attrib_to_tpg(item);
3599	struct srpt_port *sport = srpt_tpg_to_sport(tpg: se_tpg);
3600	unsigned long val;
3601	int ret;
3602
3603	ret = kstrtoul(s: page, base: 0, res: &val);
3604	if (ret < 0) {
3605	pr_err("kstrtoul() failed with ret: %d\n", ret);
3606	return -EINVAL;
3607	}
3608	if (val > MAX_SRPT_SRQ_SIZE) {
3609	pr_err("val: %lu exceeds MAX_SRPT_SRQ_SIZE: %d\n", val,
3610	MAX_SRPT_SRQ_SIZE);
3611	return -EINVAL;
3612	}
3613	if (val < MIN_SRPT_SRQ_SIZE) {
3614	pr_err("val: %lu smaller than MIN_SRPT_SRQ_SIZE: %d\n", val,
3615	MIN_SRPT_SRQ_SIZE);
3616	return -EINVAL;
3617	}
3618	sport->port_attrib.srp_sq_size = val;
3619
3620	return count;
3621	}
3622
3623	static ssize_t srpt_tpg_attrib_use_srq_show(struct config_item *item,
3624	char *page)
3625	{
3626	struct se_portal_group *se_tpg = attrib_to_tpg(item);
3627	struct srpt_port *sport = srpt_tpg_to_sport(tpg: se_tpg);
3628
3629	return sysfs_emit(buf: page, fmt: "%d\n", sport->port_attrib.use_srq);
3630	}
3631
3632	static ssize_t srpt_tpg_attrib_use_srq_store(struct config_item *item,
3633	const char *page, size_t count)
3634	{
3635	struct se_portal_group *se_tpg = attrib_to_tpg(item);
3636	struct srpt_port *sport = srpt_tpg_to_sport(tpg: se_tpg);
3637	struct srpt_device *sdev = sport->sdev;
3638	unsigned long val;
3639	bool enabled;
3640	int ret;
3641
3642	ret = kstrtoul(s: page, base: 0, res: &val);
3643	if (ret < 0)
3644	return ret;
3645	if (val != !!val)
3646	return -EINVAL;
3647
3648	ret = mutex_lock_interruptible(&sdev->sdev_mutex);
3649	if (ret < 0)
3650	return ret;
3651	ret = mutex_lock_interruptible(&sport->mutex);
3652	if (ret < 0)
3653	goto unlock_sdev;
3654	enabled = sport->enabled;
3655	/* Log out all initiator systems before changing 'use_srq'. */
3656	srpt_set_enabled(sport, enabled: false);
3657	sport->port_attrib.use_srq = val;
3658	srpt_use_srq(sdev, use_srq: sport->port_attrib.use_srq);
3659	srpt_set_enabled(sport, enabled);
3660	ret = count;
3661	mutex_unlock(lock: &sport->mutex);
3662	unlock_sdev:
3663	mutex_unlock(lock: &sdev->sdev_mutex);
3664
3665	return ret;
3666	}
3667
3668	CONFIGFS_ATTR(srpt_tpg_attrib_, srp_max_rdma_size);
3669	CONFIGFS_ATTR(srpt_tpg_attrib_, srp_max_rsp_size);
3670	CONFIGFS_ATTR(srpt_tpg_attrib_, srp_sq_size);
3671	CONFIGFS_ATTR(srpt_tpg_attrib_, use_srq);
3672
3673	static struct configfs_attribute *srpt_tpg_attrib_attrs[] = {
3674	&srpt_tpg_attrib_attr_srp_max_rdma_size,
3675	&srpt_tpg_attrib_attr_srp_max_rsp_size,
3676	&srpt_tpg_attrib_attr_srp_sq_size,
3677	&srpt_tpg_attrib_attr_use_srq,
3678	NULL,
3679	};
3680
3681	static struct rdma_cm_id *srpt_create_rdma_id(struct sockaddr *listen_addr)
3682	{
3683	struct rdma_cm_id *rdma_cm_id;
3684	int ret;
3685
3686	rdma_cm_id = rdma_create_id(&init_net, srpt_rdma_cm_handler,
3687	NULL, RDMA_PS_TCP, IB_QPT_RC);
3688	if (IS_ERR(ptr: rdma_cm_id)) {
3689	pr_err("RDMA/CM ID creation failed: %pe\n", rdma_cm_id);
3690	goto out;
3691	}
3692
3693	ret = rdma_bind_addr(id: rdma_cm_id, addr: listen_addr);
3694	if (ret) {
3695	char addr_str[64];
3696
3697	snprintf(buf: addr_str, size: sizeof(addr_str), fmt: "%pISp", listen_addr);
3698	pr_err("Binding RDMA/CM ID to address %s failed: %d\n",
3699	addr_str, ret);
3700	rdma_destroy_id(id: rdma_cm_id);
3701	rdma_cm_id = ERR_PTR(error: ret);
3702	goto out;
3703	}
3704
3705	ret = rdma_listen(id: rdma_cm_id, backlog: 128);
3706	if (ret) {
3707	pr_err("rdma_listen() failed: %d\n", ret);
3708	rdma_destroy_id(id: rdma_cm_id);
3709	rdma_cm_id = ERR_PTR(error: ret);
3710	}
3711
3712	out:
3713	return rdma_cm_id;
3714	}
3715
3716	static ssize_t srpt_rdma_cm_port_show(struct config_item *item, char *page)
3717	{
3718	return sysfs_emit(buf: page, fmt: "%d\n", rdma_cm_port);
3719	}
3720
3721	static ssize_t srpt_rdma_cm_port_store(struct config_item *item,
3722	const char *page, size_t count)
3723	{
3724	struct sockaddr_in addr4 = { .sin_family = AF_INET };
3725	struct sockaddr_in6 addr6 = { .sin6_family = AF_INET6 };
3726	struct rdma_cm_id *new_id = NULL;
3727	u16 val;
3728	int ret;
3729
3730	ret = kstrtou16(s: page, base: 0, res: &val);
3731	if (ret < 0)
3732	return ret;
3733	ret = count;
3734	if (rdma_cm_port == val)
3735	goto out;
3736
3737	if (val) {
3738	addr6.sin6_port = cpu_to_be16(val);
3739	new_id = srpt_create_rdma_id(listen_addr: (struct sockaddr *)&addr6);
3740	if (IS_ERR(ptr: new_id)) {
3741	addr4.sin_port = cpu_to_be16(val);
3742	new_id = srpt_create_rdma_id(listen_addr: (struct sockaddr *)&addr4);
3743	if (IS_ERR(ptr: new_id)) {
3744	ret = PTR_ERR(ptr: new_id);
3745	goto out;
3746	}
3747	}
3748	}
3749
3750	mutex_lock(&rdma_cm_mutex);
3751	rdma_cm_port = val;
3752	swap(rdma_cm_id, new_id);
3753	mutex_unlock(lock: &rdma_cm_mutex);
3754
3755	if (new_id)
3756	rdma_destroy_id(id: new_id);
3757	ret = count;
3758	out:
3759	return ret;
3760	}
3761
3762	CONFIGFS_ATTR(srpt_, rdma_cm_port);
3763
3764	static struct configfs_attribute *srpt_da_attrs[] = {
3765	&srpt_attr_rdma_cm_port,
3766	NULL,
3767	};
3768
3769	static int srpt_enable_tpg(struct se_portal_group *se_tpg, bool enable)
3770	{
3771	struct srpt_port *sport = srpt_tpg_to_sport(tpg: se_tpg);
3772
3773	mutex_lock(&sport->mutex);
3774	srpt_set_enabled(sport, enabled: enable);
3775	mutex_unlock(lock: &sport->mutex);
3776
3777	return 0;
3778	}
3779
3780	/**
3781	* srpt_make_tpg - configfs callback invoked for mkdir /sys/kernel/config/target/$driver/$port/$tpg
3782	* @wwn: Corresponds to $driver/$port.
3783	* @name: $tpg.
3784	*/
3785	static struct se_portal_group *srpt_make_tpg(struct se_wwn *wwn,
3786	const char *name)
3787	{
3788	struct srpt_port_id *sport_id = srpt_wwn_to_sport_id(wwn);
3789	struct srpt_tpg *stpg;
3790	int res = -ENOMEM;
3791
3792	stpg = kzalloc(sizeof(*stpg), GFP_KERNEL);
3793	if (!stpg)
3794	return ERR_PTR(error: res);
3795	stpg->sport_id = sport_id;
3796	res = core_tpg_register(wwn, &stpg->tpg, SCSI_PROTOCOL_SRP);
3797	if (res) {
3798	kfree(objp: stpg);
3799	return ERR_PTR(error: res);
3800	}
3801
3802	mutex_lock(&sport_id->mutex);
3803	list_add_tail(new: &stpg->entry, head: &sport_id->tpg_list);
3804	mutex_unlock(lock: &sport_id->mutex);
3805
3806	return &stpg->tpg;
3807	}
3808
3809	/**
3810	* srpt_drop_tpg - configfs callback invoked for rmdir /sys/kernel/config/target/$driver/$port/$tpg
3811	* @tpg: Target portal group to deregister.
3812	*/
3813	static void srpt_drop_tpg(struct se_portal_group *tpg)
3814	{
3815	struct srpt_tpg *stpg = container_of(tpg, typeof(*stpg), tpg);
3816	struct srpt_port_id *sport_id = stpg->sport_id;
3817	struct srpt_port *sport = srpt_tpg_to_sport(tpg);
3818
3819	mutex_lock(&sport_id->mutex);
3820	list_del(entry: &stpg->entry);
3821	mutex_unlock(lock: &sport_id->mutex);
3822
3823	sport->enabled = false;
3824	core_tpg_deregister(tpg);
3825	kfree(objp: stpg);
3826	}
3827
3828	/**
3829	* srpt_make_tport - configfs callback invoked for mkdir /sys/kernel/config/target/$driver/$port
3830	* @tf: Not used.
3831	* @group: Not used.
3832	* @name: $port.
3833	*/
3834	static struct se_wwn *srpt_make_tport(struct target_fabric_configfs *tf,
3835	struct config_group *group,
3836	const char *name)
3837	{
3838	struct port_and_port_id papi = srpt_lookup_port(name);
3839	struct srpt_port *sport = papi.sport;
3840	struct srpt_port_id *port_id;
3841
3842	if (!papi.port_id)
3843	return ERR_PTR(error: -EINVAL);
3844	if (*papi.port_id) {
3845	/* Attempt to create a directory that already exists. */
3846	WARN_ON_ONCE(true);
3847	return &(*papi.port_id)->wwn;
3848	}
3849	port_id = kzalloc(sizeof(*port_id), GFP_KERNEL);
3850	if (!port_id) {
3851	srpt_sdev_put(sdev: sport->sdev);
3852	return ERR_PTR(error: -ENOMEM);
3853	}
3854	mutex_init(&port_id->mutex);
3855	INIT_LIST_HEAD(list: &port_id->tpg_list);
3856	port_id->wwn.priv = sport;
3857	memcpy(port_id->name, port_id == sport->guid_id ? sport->guid_name :
3858	sport->gid_name, ARRAY_SIZE(port_id->name));
3859
3860	*papi.port_id = port_id;
3861
3862	return &port_id->wwn;
3863	}
3864
3865	/**
3866	* srpt_drop_tport - configfs callback invoked for rmdir /sys/kernel/config/target/$driver/$port
3867	* @wwn: $port.
3868	*/
3869	static void srpt_drop_tport(struct se_wwn *wwn)
3870	{
3871	struct srpt_port_id *port_id = container_of(wwn, typeof(*port_id), wwn);
3872	struct srpt_port *sport = wwn->priv;
3873
3874	if (sport->guid_id == port_id)
3875	sport->guid_id = NULL;
3876	else if (sport->gid_id == port_id)
3877	sport->gid_id = NULL;
3878	else
3879	WARN_ON_ONCE(true);
3880
3881	srpt_sdev_put(sdev: sport->sdev);
3882	kfree(objp: port_id);
3883	}
3884
3885	static ssize_t srpt_wwn_version_show(struct config_item *item, char *buf)
3886	{
3887	return sysfs_emit(buf, fmt: "\n");
3888	}
3889
3890	CONFIGFS_ATTR_RO(srpt_wwn_, version);
3891
3892	static struct configfs_attribute *srpt_wwn_attrs[] = {
3893	&srpt_wwn_attr_version,
3894	NULL,
3895	};
3896
3897	static const struct target_core_fabric_ops srpt_template = {
3898	.module = THIS_MODULE,
3899	.fabric_name = "srpt",
3900	.tpg_get_wwn = srpt_get_fabric_wwn,
3901	.tpg_get_tag = srpt_get_tag,
3902	.tpg_check_demo_mode_cache = srpt_check_true,
3903	.tpg_check_demo_mode_write_protect = srpt_check_true,
3904	.release_cmd = srpt_release_cmd,
3905	.check_stop_free = srpt_check_stop_free,
3906	.close_session = srpt_close_session,
3907	.sess_get_initiator_sid = NULL,
3908	.write_pending = srpt_write_pending,
3909	.get_cmd_state = srpt_get_tcm_cmd_state,
3910	.queue_data_in = srpt_queue_data_in,
3911	.queue_status = srpt_queue_status,
3912	.queue_tm_rsp = srpt_queue_tm_rsp,
3913	.aborted_task = srpt_aborted_task,
3914	/*
3915	* Setup function pointers for generic logic in
3916	* target_core_fabric_configfs.c
3917	*/
3918	.fabric_make_wwn = srpt_make_tport,
3919	.fabric_drop_wwn = srpt_drop_tport,
3920	.fabric_make_tpg = srpt_make_tpg,
3921	.fabric_enable_tpg = srpt_enable_tpg,
3922	.fabric_drop_tpg = srpt_drop_tpg,
3923	.fabric_init_nodeacl = srpt_init_nodeacl,
3924
3925	.tfc_discovery_attrs = srpt_da_attrs,
3926	.tfc_wwn_attrs = srpt_wwn_attrs,
3927	.tfc_tpg_attrib_attrs = srpt_tpg_attrib_attrs,
3928
3929	.default_submit_type = TARGET_DIRECT_SUBMIT,
3930	.direct_submit_supp = 1,
3931	};
3932
3933	/**
3934	* srpt_init_module - kernel module initialization
3935	*
3936	* Note: Since ib_register_client() registers callback functions, and since at
3937	* least one of these callback functions (srpt_add_one()) calls target core
3938	* functions, this driver must be registered with the target core before
3939	* ib_register_client() is called.
3940	*/
3941	static int __init srpt_init_module(void)
3942	{
3943	int ret;
3944
3945	ret = -EINVAL;
3946	if (srp_max_req_size < MIN_MAX_REQ_SIZE) {
3947	pr_err("invalid value %d for kernel module parameter srp_max_req_size -- must be at least %d.\n",
3948	srp_max_req_size, MIN_MAX_REQ_SIZE);
3949	goto out;
3950	}
3951
3952	if (srpt_srq_size < MIN_SRPT_SRQ_SIZE
3953	|| srpt_srq_size > MAX_SRPT_SRQ_SIZE) {
3954	pr_err("invalid value %d for kernel module parameter srpt_srq_size -- must be in the range [%d..%d].\n",
3955	srpt_srq_size, MIN_SRPT_SRQ_SIZE, MAX_SRPT_SRQ_SIZE);
3956	goto out;
3957	}
3958
3959	ret = target_register_template(fo: &srpt_template);
3960	if (ret)
3961	goto out;
3962
3963	ret = ib_register_client(client: &srpt_client);
3964	if (ret) {
3965	pr_err("couldn't register IB client\n");
3966	goto out_unregister_target;
3967	}
3968
3969	return 0;
3970
3971	out_unregister_target:
3972	target_unregister_template(fo: &srpt_template);
3973	out:
3974	return ret;
3975	}
3976
3977	static void __exit srpt_cleanup_module(void)
3978	{
3979	if (rdma_cm_id)
3980	rdma_destroy_id(id: rdma_cm_id);
3981	ib_unregister_client(client: &srpt_client);
3982	target_unregister_template(fo: &srpt_template);
3983	}
3984
3985	module_init(srpt_init_module);
3986	module_exit(srpt_cleanup_module);
3987