1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Copyright (c) 2016 Avago Technologies. All rights reserved.
4	*/
5	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
6	#include <linux/module.h>
7	#include <linux/parser.h>
8	#include <uapi/scsi/fc/fc_fs.h>
9	#include <uapi/scsi/fc/fc_els.h>
10	#include <linux/delay.h>
11	#include <linux/overflow.h>
12	#include <linux/blk-cgroup.h>
13	#include "nvme.h"
14	#include "fabrics.h"
15	#include <linux/nvme-fc-driver.h>
16	#include <linux/nvme-fc.h>
17	#include "fc.h"
18	#include <scsi/scsi_transport_fc.h>
19	#include <linux/blk-mq-pci.h>
20
21	/* *************************** Data Structures/Defines ****************** */
22
23
24	enum nvme_fc_queue_flags {
25	NVME_FC_Q_CONNECTED = 0,
26	NVME_FC_Q_LIVE,
27	};
28
29	#define NVME_FC_DEFAULT_DEV_LOSS_TMO 60 /* seconds */
30	#define NVME_FC_DEFAULT_RECONNECT_TMO 2 /* delay between reconnects
31	* when connected and a
32	* connection failure.
33	*/
34
35	struct nvme_fc_queue {
36	struct nvme_fc_ctrl *ctrl;
37	struct device *dev;
38	struct blk_mq_hw_ctx *hctx;
39	void *lldd_handle;
40	size_t cmnd_capsule_len;
41	u32 qnum;
42	u32 rqcnt;
43	u32 seqno;
44
45	u64 connection_id;
46	atomic_t csn;
47
48	unsigned long flags;
49	} __aligned(sizeof(u64)); /* alignment for other things alloc'd with */
50
51	enum nvme_fcop_flags {
52	FCOP_FLAGS_TERMIO = (1 << 0),
53	FCOP_FLAGS_AEN = (1 << 1),
54	};
55
56	struct nvmefc_ls_req_op {
57	struct nvmefc_ls_req ls_req;
58
59	struct nvme_fc_rport *rport;
60	struct nvme_fc_queue *queue;
61	struct request *rq;
62	u32 flags;
63
64	int ls_error;
65	struct completion ls_done;
66	struct list_head lsreq_list; /* rport->ls_req_list */
67	bool req_queued;
68	};
69
70	struct nvmefc_ls_rcv_op {
71	struct nvme_fc_rport *rport;
72	struct nvmefc_ls_rsp *lsrsp;
73	union nvmefc_ls_requests *rqstbuf;
74	union nvmefc_ls_responses *rspbuf;
75	u16 rqstdatalen;
76	bool handled;
77	dma_addr_t rspdma;
78	struct list_head lsrcv_list; /* rport->ls_rcv_list */
79	} __aligned(sizeof(u64)); /* alignment for other things alloc'd with */
80
81	enum nvme_fcpop_state {
82	FCPOP_STATE_UNINIT = 0,
83	FCPOP_STATE_IDLE = 1,
84	FCPOP_STATE_ACTIVE = 2,
85	FCPOP_STATE_ABORTED = 3,
86	FCPOP_STATE_COMPLETE = 4,
87	};
88
89	struct nvme_fc_fcp_op {
90	struct nvme_request nreq; /*
91	* nvme/host/core.c
92	* requires this to be
93	* the 1st element in the
94	* private structure
95	* associated with the
96	* request.
97	*/
98	struct nvmefc_fcp_req fcp_req;
99
100	struct nvme_fc_ctrl *ctrl;
101	struct nvme_fc_queue *queue;
102	struct request *rq;
103
104	atomic_t state;
105	u32 flags;
106	u32 rqno;
107	u32 nents;
108
109	struct nvme_fc_cmd_iu cmd_iu;
110	struct nvme_fc_ersp_iu rsp_iu;
111	};
112
113	struct nvme_fcp_op_w_sgl {
114	struct nvme_fc_fcp_op op;
115	struct scatterlist sgl[NVME_INLINE_SG_CNT];
116	uint8_t priv[];
117	};
118
119	struct nvme_fc_lport {
120	struct nvme_fc_local_port localport;
121
122	struct ida endp_cnt;
123	struct list_head port_list; /* nvme_fc_port_list */
124	struct list_head endp_list;
125	struct device *dev; /* physical device for dma */
126	struct nvme_fc_port_template *ops;
127	struct kref ref;
128	atomic_t act_rport_cnt;
129	} __aligned(sizeof(u64)); /* alignment for other things alloc'd with */
130
131	struct nvme_fc_rport {
132	struct nvme_fc_remote_port remoteport;
133
134	struct list_head endp_list; /* for lport->endp_list */
135	struct list_head ctrl_list;
136	struct list_head ls_req_list;
137	struct list_head ls_rcv_list;
138	struct list_head disc_list;
139	struct device *dev; /* physical device for dma */
140	struct nvme_fc_lport *lport;
141	spinlock_t lock;
142	struct kref ref;
143	atomic_t act_ctrl_cnt;
144	unsigned long dev_loss_end;
145	struct work_struct lsrcv_work;
146	} __aligned(sizeof(u64)); /* alignment for other things alloc'd with */
147
148	/* fc_ctrl flags values - specified as bit positions */
149	#define ASSOC_ACTIVE 0
150	#define ASSOC_FAILED 1
151	#define FCCTRL_TERMIO 2
152
153	struct nvme_fc_ctrl {
154	spinlock_t lock;
155	struct nvme_fc_queue *queues;
156	struct device *dev;
157	struct nvme_fc_lport *lport;
158	struct nvme_fc_rport *rport;
159	u32 cnum;
160
161	bool ioq_live;
162	u64 association_id;
163	struct nvmefc_ls_rcv_op *rcv_disconn;
164
165	struct list_head ctrl_list; /* rport->ctrl_list */
166
167	struct blk_mq_tag_set admin_tag_set;
168	struct blk_mq_tag_set tag_set;
169
170	struct work_struct ioerr_work;
171	struct delayed_work connect_work;
172
173	struct kref ref;
174	unsigned long flags;
175	u32 iocnt;
176	wait_queue_head_t ioabort_wait;
177
178	struct nvme_fc_fcp_op aen_ops[NVME_NR_AEN_COMMANDS];
179
180	struct nvme_ctrl ctrl;
181	};
182
183	static inline struct nvme_fc_ctrl *
184	to_fc_ctrl(struct nvme_ctrl *ctrl)
185	{
186	return container_of(ctrl, struct nvme_fc_ctrl, ctrl);
187	}
188
189	static inline struct nvme_fc_lport *
190	localport_to_lport(struct nvme_fc_local_port *portptr)
191	{
192	return container_of(portptr, struct nvme_fc_lport, localport);
193	}
194
195	static inline struct nvme_fc_rport *
196	remoteport_to_rport(struct nvme_fc_remote_port *portptr)
197	{
198	return container_of(portptr, struct nvme_fc_rport, remoteport);
199	}
200
201	static inline struct nvmefc_ls_req_op *
202	ls_req_to_lsop(struct nvmefc_ls_req *lsreq)
203	{
204	return container_of(lsreq, struct nvmefc_ls_req_op, ls_req);
205	}
206
207	static inline struct nvme_fc_fcp_op *
208	fcp_req_to_fcp_op(struct nvmefc_fcp_req *fcpreq)
209	{
210	return container_of(fcpreq, struct nvme_fc_fcp_op, fcp_req);
211	}
212
213
214
215	/* *************************** Globals **************************** */
216
217
218	static DEFINE_SPINLOCK(nvme_fc_lock);
219
220	static LIST_HEAD(nvme_fc_lport_list);
221	static DEFINE_IDA(nvme_fc_local_port_cnt);
222	static DEFINE_IDA(nvme_fc_ctrl_cnt);
223
224	/*
225	* These items are short-term. They will eventually be moved into
226	* a generic FC class. See comments in module init.
227	*/
228	static struct device *fc_udev_device;
229
230	static void nvme_fc_complete_rq(struct request *rq);
231
232	/* *********************** FC-NVME Port Management ************************ */
233
234	static void __nvme_fc_delete_hw_queue(struct nvme_fc_ctrl *,
235	struct nvme_fc_queue *, unsigned int);
236
237	static void nvme_fc_handle_ls_rqst_work(struct work_struct *work);
238
239
240	static void
241	nvme_fc_free_lport(struct kref *ref)
242	{
243	struct nvme_fc_lport *lport =
244	container_of(ref, struct nvme_fc_lport, ref);
245	unsigned long flags;
246
247	WARN_ON(lport->localport.port_state != FC_OBJSTATE_DELETED);
248	WARN_ON(!list_empty(&lport->endp_list));
249
250	/* remove from transport list */
251	spin_lock_irqsave(&nvme_fc_lock, flags);
252	list_del(entry: &lport->port_list);
253	spin_unlock_irqrestore(lock: &nvme_fc_lock, flags);
254
255	ida_free(&nvme_fc_local_port_cnt, id: lport->localport.port_num);
256	ida_destroy(ida: &lport->endp_cnt);
257
258	put_device(dev: lport->dev);
259
260	kfree(objp: lport);
261	}
262
263	static void
264	nvme_fc_lport_put(struct nvme_fc_lport *lport)
265	{
266	kref_put(kref: &lport->ref, release: nvme_fc_free_lport);
267	}
268
269	static int
270	nvme_fc_lport_get(struct nvme_fc_lport *lport)
271	{
272	return kref_get_unless_zero(kref: &lport->ref);
273	}
274
275
276	static struct nvme_fc_lport *
277	nvme_fc_attach_to_unreg_lport(struct nvme_fc_port_info *pinfo,
278	struct nvme_fc_port_template *ops,
279	struct device *dev)
280	{
281	struct nvme_fc_lport *lport;
282	unsigned long flags;
283
284	spin_lock_irqsave(&nvme_fc_lock, flags);
285
286	list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
287	if (lport->localport.node_name != pinfo->node_name ||
288	lport->localport.port_name != pinfo->port_name)
289	continue;
290
291	if (lport->dev != dev) {
292	lport = ERR_PTR(error: -EXDEV);
293	goto out_done;
294	}
295
296	if (lport->localport.port_state != FC_OBJSTATE_DELETED) {
297	lport = ERR_PTR(error: -EEXIST);
298	goto out_done;
299	}
300
301	if (!nvme_fc_lport_get(lport)) {
302	/*
303	* fails if ref cnt already 0. If so,
304	* act as if lport already deleted
305	*/
306	lport = NULL;
307	goto out_done;
308	}
309
310	/* resume the lport */
311
312	lport->ops = ops;
313	lport->localport.port_role = pinfo->port_role;
314	lport->localport.port_id = pinfo->port_id;
315	lport->localport.port_state = FC_OBJSTATE_ONLINE;
316
317	spin_unlock_irqrestore(lock: &nvme_fc_lock, flags);
318
319	return lport;
320	}
321
322	lport = NULL;
323
324	out_done:
325	spin_unlock_irqrestore(lock: &nvme_fc_lock, flags);
326
327	return lport;
328	}
329
330	/**
331	* nvme_fc_register_localport - transport entry point called by an
332	* LLDD to register the existence of a NVME
333	* host FC port.
334	* @pinfo: pointer to information about the port to be registered
335	* @template: LLDD entrypoints and operational parameters for the port
336	* @dev: physical hardware device node port corresponds to. Will be
337	* used for DMA mappings
338	* @portptr: pointer to a local port pointer. Upon success, the routine
339	* will allocate a nvme_fc_local_port structure and place its
340	* address in the local port pointer. Upon failure, local port
341	* pointer will be set to 0.
342	*
343	* Returns:
344	* a completion status. Must be 0 upon success; a negative errno
345	* (ex: -ENXIO) upon failure.
346	*/
347	int
348	nvme_fc_register_localport(struct nvme_fc_port_info *pinfo,
349	struct nvme_fc_port_template *template,
350	struct device *dev,
351	struct nvme_fc_local_port **portptr)
352	{
353	struct nvme_fc_lport *newrec;
354	unsigned long flags;
355	int ret, idx;
356
357	if (!template->localport_delete || !template->remoteport_delete ||
358	!template->ls_req || !template->fcp_io ||
359	!template->ls_abort || !template->fcp_abort ||
360	!template->max_hw_queues || !template->max_sgl_segments ||
361	!template->max_dif_sgl_segments || !template->dma_boundary) {
362	ret = -EINVAL;
363	goto out_reghost_failed;
364	}
365
366	/*
367	* look to see if there is already a localport that had been
368	* deregistered and in the process of waiting for all the
369	* references to fully be removed. If the references haven't
370	* expired, we can simply re-enable the localport. Remoteports
371	* and controller reconnections should resume naturally.
372	*/
373	newrec = nvme_fc_attach_to_unreg_lport(pinfo, ops: template, dev);
374
375	/* found an lport, but something about its state is bad */
376	if (IS_ERR(ptr: newrec)) {
377	ret = PTR_ERR(ptr: newrec);
378	goto out_reghost_failed;
379
380	/* found existing lport, which was resumed */
381	} else if (newrec) {
382	*portptr = &newrec->localport;
383	return 0;
384	}
385
386	/* nothing found - allocate a new localport struct */
387
388	newrec = kmalloc(size: (sizeof(*newrec) + template->local_priv_sz),
389	GFP_KERNEL);
390	if (!newrec) {
391	ret = -ENOMEM;
392	goto out_reghost_failed;
393	}
394
395	idx = ida_alloc(ida: &nvme_fc_local_port_cnt, GFP_KERNEL);
396	if (idx < 0) {
397	ret = -ENOSPC;
398	goto out_fail_kfree;
399	}
400
401	if (!get_device(dev) && dev) {
402	ret = -ENODEV;
403	goto out_ida_put;
404	}
405
406	INIT_LIST_HEAD(list: &newrec->port_list);
407	INIT_LIST_HEAD(list: &newrec->endp_list);
408	kref_init(kref: &newrec->ref);
409	atomic_set(v: &newrec->act_rport_cnt, i: 0);
410	newrec->ops = template;
411	newrec->dev = dev;
412	ida_init(ida: &newrec->endp_cnt);
413	if (template->local_priv_sz)
414	newrec->localport.private = &newrec[1];
415	else
416	newrec->localport.private = NULL;
417	newrec->localport.node_name = pinfo->node_name;
418	newrec->localport.port_name = pinfo->port_name;
419	newrec->localport.port_role = pinfo->port_role;
420	newrec->localport.port_id = pinfo->port_id;
421	newrec->localport.port_state = FC_OBJSTATE_ONLINE;
422	newrec->localport.port_num = idx;
423
424	spin_lock_irqsave(&nvme_fc_lock, flags);
425	list_add_tail(new: &newrec->port_list, head: &nvme_fc_lport_list);
426	spin_unlock_irqrestore(lock: &nvme_fc_lock, flags);
427
428	if (dev)
429	dma_set_seg_boundary(dev, mask: template->dma_boundary);
430
431	*portptr = &newrec->localport;
432	return 0;
433
434	out_ida_put:
435	ida_free(&nvme_fc_local_port_cnt, id: idx);
436	out_fail_kfree:
437	kfree(objp: newrec);
438	out_reghost_failed:
439	*portptr = NULL;
440
441	return ret;
442	}
443	EXPORT_SYMBOL_GPL(nvme_fc_register_localport);
444
445	/**
446	* nvme_fc_unregister_localport - transport entry point called by an
447	* LLDD to deregister/remove a previously
448	* registered a NVME host FC port.
449	* @portptr: pointer to the (registered) local port that is to be deregistered.
450	*
451	* Returns:
452	* a completion status. Must be 0 upon success; a negative errno
453	* (ex: -ENXIO) upon failure.
454	*/
455	int
456	nvme_fc_unregister_localport(struct nvme_fc_local_port *portptr)
457	{
458	struct nvme_fc_lport *lport = localport_to_lport(portptr);
459	unsigned long flags;
460
461	if (!portptr)
462	return -EINVAL;
463
464	spin_lock_irqsave(&nvme_fc_lock, flags);
465
466	if (portptr->port_state != FC_OBJSTATE_ONLINE) {
467	spin_unlock_irqrestore(lock: &nvme_fc_lock, flags);
468	return -EINVAL;
469	}
470	portptr->port_state = FC_OBJSTATE_DELETED;
471
472	spin_unlock_irqrestore(lock: &nvme_fc_lock, flags);
473
474	if (atomic_read(v: &lport->act_rport_cnt) == 0)
475	lport->ops->localport_delete(&lport->localport);
476
477	nvme_fc_lport_put(lport);
478
479	return 0;
480	}
481	EXPORT_SYMBOL_GPL(nvme_fc_unregister_localport);
482
483	/*
484	* TRADDR strings, per FC-NVME are fixed format:
485	* "nn-0x<16hexdigits>:pn-0x<16hexdigits>" - 43 characters
486	* udev event will only differ by prefix of what field is
487	* being specified:
488	* "NVMEFC_HOST_TRADDR=" or "NVMEFC_TRADDR=" - 19 max characters
489	* 19 + 43 + null_fudge = 64 characters
490	*/
491	#define FCNVME_TRADDR_LENGTH 64
492
493	static void
494	nvme_fc_signal_discovery_scan(struct nvme_fc_lport *lport,
495	struct nvme_fc_rport *rport)
496	{
497	char hostaddr[FCNVME_TRADDR_LENGTH]; /* NVMEFC_HOST_TRADDR=...*/
498	char tgtaddr[FCNVME_TRADDR_LENGTH]; /* NVMEFC_TRADDR=...*/
499	char *envp[4] = { "FC_EVENT=nvmediscovery", hostaddr, tgtaddr, NULL };
500
501	if (!(rport->remoteport.port_role & FC_PORT_ROLE_NVME_DISCOVERY))
502	return;
503
504	snprintf(buf: hostaddr, size: sizeof(hostaddr),
505	fmt: "NVMEFC_HOST_TRADDR=nn-0x%016llx:pn-0x%016llx",
506	lport->localport.node_name, lport->localport.port_name);
507	snprintf(buf: tgtaddr, size: sizeof(tgtaddr),
508	fmt: "NVMEFC_TRADDR=nn-0x%016llx:pn-0x%016llx",
509	rport->remoteport.node_name, rport->remoteport.port_name);
510	kobject_uevent_env(kobj: &fc_udev_device->kobj, action: KOBJ_CHANGE, envp);
511	}
512
513	static void
514	nvme_fc_free_rport(struct kref *ref)
515	{
516	struct nvme_fc_rport *rport =
517	container_of(ref, struct nvme_fc_rport, ref);
518	struct nvme_fc_lport *lport =
519	localport_to_lport(portptr: rport->remoteport.localport);
520	unsigned long flags;
521
522	WARN_ON(rport->remoteport.port_state != FC_OBJSTATE_DELETED);
523	WARN_ON(!list_empty(&rport->ctrl_list));
524
525	/* remove from lport list */
526	spin_lock_irqsave(&nvme_fc_lock, flags);
527	list_del(entry: &rport->endp_list);
528	spin_unlock_irqrestore(lock: &nvme_fc_lock, flags);
529
530	WARN_ON(!list_empty(&rport->disc_list));
531	ida_free(&lport->endp_cnt, id: rport->remoteport.port_num);
532
533	kfree(objp: rport);
534
535	nvme_fc_lport_put(lport);
536	}
537
538	static void
539	nvme_fc_rport_put(struct nvme_fc_rport *rport)
540	{
541	kref_put(kref: &rport->ref, release: nvme_fc_free_rport);
542	}
543
544	static int
545	nvme_fc_rport_get(struct nvme_fc_rport *rport)
546	{
547	return kref_get_unless_zero(kref: &rport->ref);
548	}
549
550	static void
551	nvme_fc_resume_controller(struct nvme_fc_ctrl *ctrl)
552	{
553	switch (nvme_ctrl_state(ctrl: &ctrl->ctrl)) {
554	case NVME_CTRL_NEW:
555	case NVME_CTRL_CONNECTING:
556	/*
557	* As all reconnects were suppressed, schedule a
558	* connect.
559	*/
560	dev_info(ctrl->ctrl.device,
561	"NVME-FC{%d}: connectivity re-established. "
562	"Attempting reconnect\n", ctrl->cnum);
563
564	queue_delayed_work(wq: nvme_wq, dwork: &ctrl->connect_work, delay: 0);
565	break;
566
567	case NVME_CTRL_RESETTING:
568	/*
569	* Controller is already in the process of terminating the
570	* association. No need to do anything further. The reconnect
571	* step will naturally occur after the reset completes.
572	*/
573	break;
574
575	default:
576	/* no action to take - let it delete */
577	break;
578	}
579	}
580
581	static struct nvme_fc_rport *
582	nvme_fc_attach_to_suspended_rport(struct nvme_fc_lport *lport,
583	struct nvme_fc_port_info *pinfo)
584	{
585	struct nvme_fc_rport *rport;
586	struct nvme_fc_ctrl *ctrl;
587	unsigned long flags;
588
589	spin_lock_irqsave(&nvme_fc_lock, flags);
590
591	list_for_each_entry(rport, &lport->endp_list, endp_list) {
592	if (rport->remoteport.node_name != pinfo->node_name ||
593	rport->remoteport.port_name != pinfo->port_name)
594	continue;
595
596	if (!nvme_fc_rport_get(rport)) {
597	rport = ERR_PTR(error: -ENOLCK);
598	goto out_done;
599	}
600
601	spin_unlock_irqrestore(lock: &nvme_fc_lock, flags);
602
603	spin_lock_irqsave(&rport->lock, flags);
604
605	/* has it been unregistered */
606	if (rport->remoteport.port_state != FC_OBJSTATE_DELETED) {
607	/* means lldd called us twice */
608	spin_unlock_irqrestore(lock: &rport->lock, flags);
609	nvme_fc_rport_put(rport);
610	return ERR_PTR(error: -ESTALE);
611	}
612
613	rport->remoteport.port_role = pinfo->port_role;
614	rport->remoteport.port_id = pinfo->port_id;
615	rport->remoteport.port_state = FC_OBJSTATE_ONLINE;
616	rport->dev_loss_end = 0;
617
618	/*
619	* kick off a reconnect attempt on all associations to the
620	* remote port. A successful reconnects will resume i/o.
621	*/
622	list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list)
623	nvme_fc_resume_controller(ctrl);
624
625	spin_unlock_irqrestore(lock: &rport->lock, flags);
626
627	return rport;
628	}
629
630	rport = NULL;
631
632	out_done:
633	spin_unlock_irqrestore(lock: &nvme_fc_lock, flags);
634
635	return rport;
636	}
637
638	static inline void
639	__nvme_fc_set_dev_loss_tmo(struct nvme_fc_rport *rport,
640	struct nvme_fc_port_info *pinfo)
641	{
642	if (pinfo->dev_loss_tmo)
643	rport->remoteport.dev_loss_tmo = pinfo->dev_loss_tmo;
644	else
645	rport->remoteport.dev_loss_tmo = NVME_FC_DEFAULT_DEV_LOSS_TMO;
646	}
647
648	/**
649	* nvme_fc_register_remoteport - transport entry point called by an
650	* LLDD to register the existence of a NVME
651	* subsystem FC port on its fabric.
652	* @localport: pointer to the (registered) local port that the remote
653	* subsystem port is connected to.
654	* @pinfo: pointer to information about the port to be registered
655	* @portptr: pointer to a remote port pointer. Upon success, the routine
656	* will allocate a nvme_fc_remote_port structure and place its
657	* address in the remote port pointer. Upon failure, remote port
658	* pointer will be set to 0.
659	*
660	* Returns:
661	* a completion status. Must be 0 upon success; a negative errno
662	* (ex: -ENXIO) upon failure.
663	*/
664	int
665	nvme_fc_register_remoteport(struct nvme_fc_local_port *localport,
666	struct nvme_fc_port_info *pinfo,
667	struct nvme_fc_remote_port **portptr)
668	{
669	struct nvme_fc_lport *lport = localport_to_lport(portptr: localport);
670	struct nvme_fc_rport *newrec;
671	unsigned long flags;
672	int ret, idx;
673
674	if (!nvme_fc_lport_get(lport)) {
675	ret = -ESHUTDOWN;
676	goto out_reghost_failed;
677	}
678
679	/*
680	* look to see if there is already a remoteport that is waiting
681	* for a reconnect (within dev_loss_tmo) with the same WWN's.
682	* If so, transition to it and reconnect.
683	*/
684	newrec = nvme_fc_attach_to_suspended_rport(lport, pinfo);
685
686	/* found an rport, but something about its state is bad */
687	if (IS_ERR(ptr: newrec)) {
688	ret = PTR_ERR(ptr: newrec);
689	goto out_lport_put;
690
691	/* found existing rport, which was resumed */
692	} else if (newrec) {
693	nvme_fc_lport_put(lport);
694	__nvme_fc_set_dev_loss_tmo(rport: newrec, pinfo);
695	nvme_fc_signal_discovery_scan(lport, rport: newrec);
696	*portptr = &newrec->remoteport;
697	return 0;
698	}
699
700	/* nothing found - allocate a new remoteport struct */
701
702	newrec = kmalloc(size: (sizeof(*newrec) + lport->ops->remote_priv_sz),
703	GFP_KERNEL);
704	if (!newrec) {
705	ret = -ENOMEM;
706	goto out_lport_put;
707	}
708
709	idx = ida_alloc(ida: &lport->endp_cnt, GFP_KERNEL);
710	if (idx < 0) {
711	ret = -ENOSPC;
712	goto out_kfree_rport;
713	}
714
715	INIT_LIST_HEAD(list: &newrec->endp_list);
716	INIT_LIST_HEAD(list: &newrec->ctrl_list);
717	INIT_LIST_HEAD(list: &newrec->ls_req_list);
718	INIT_LIST_HEAD(list: &newrec->disc_list);
719	kref_init(kref: &newrec->ref);
720	atomic_set(v: &newrec->act_ctrl_cnt, i: 0);
721	spin_lock_init(&newrec->lock);
722	newrec->remoteport.localport = &lport->localport;
723	INIT_LIST_HEAD(list: &newrec->ls_rcv_list);
724	newrec->dev = lport->dev;
725	newrec->lport = lport;
726	if (lport->ops->remote_priv_sz)
727	newrec->remoteport.private = &newrec[1];
728	else
729	newrec->remoteport.private = NULL;
730	newrec->remoteport.port_role = pinfo->port_role;
731	newrec->remoteport.node_name = pinfo->node_name;
732	newrec->remoteport.port_name = pinfo->port_name;
733	newrec->remoteport.port_id = pinfo->port_id;
734	newrec->remoteport.port_state = FC_OBJSTATE_ONLINE;
735	newrec->remoteport.port_num = idx;
736	__nvme_fc_set_dev_loss_tmo(rport: newrec, pinfo);
737	INIT_WORK(&newrec->lsrcv_work, nvme_fc_handle_ls_rqst_work);
738
739	spin_lock_irqsave(&nvme_fc_lock, flags);
740	list_add_tail(new: &newrec->endp_list, head: &lport->endp_list);
741	spin_unlock_irqrestore(lock: &nvme_fc_lock, flags);
742
743	nvme_fc_signal_discovery_scan(lport, rport: newrec);
744
745	*portptr = &newrec->remoteport;
746	return 0;
747
748	out_kfree_rport:
749	kfree(objp: newrec);
750	out_lport_put:
751	nvme_fc_lport_put(lport);
752	out_reghost_failed:
753	*portptr = NULL;
754	return ret;
755	}
756	EXPORT_SYMBOL_GPL(nvme_fc_register_remoteport);
757
758	static int
759	nvme_fc_abort_lsops(struct nvme_fc_rport *rport)
760	{
761	struct nvmefc_ls_req_op *lsop;
762	unsigned long flags;
763
764	restart:
765	spin_lock_irqsave(&rport->lock, flags);
766
767	list_for_each_entry(lsop, &rport->ls_req_list, lsreq_list) {
768	if (!(lsop->flags & FCOP_FLAGS_TERMIO)) {
769	lsop->flags |= FCOP_FLAGS_TERMIO;
770	spin_unlock_irqrestore(lock: &rport->lock, flags);
771	rport->lport->ops->ls_abort(&rport->lport->localport,
772	&rport->remoteport,
773	&lsop->ls_req);
774	goto restart;
775	}
776	}
777	spin_unlock_irqrestore(lock: &rport->lock, flags);
778
779	return 0;
780	}
781
782	static void
783	nvme_fc_ctrl_connectivity_loss(struct nvme_fc_ctrl *ctrl)
784	{
785	dev_info(ctrl->ctrl.device,
786	"NVME-FC{%d}: controller connectivity lost. Awaiting "
787	"Reconnect", ctrl->cnum);
788
789	switch (nvme_ctrl_state(ctrl: &ctrl->ctrl)) {
790	case NVME_CTRL_NEW:
791	case NVME_CTRL_LIVE:
792	/*
793	* Schedule a controller reset. The reset will terminate the
794	* association and schedule the reconnect timer. Reconnects
795	* will be attempted until either the ctlr_loss_tmo
796	* (max_retries * connect_delay) expires or the remoteport's
797	* dev_loss_tmo expires.
798	*/
799	if (nvme_reset_ctrl(ctrl: &ctrl->ctrl)) {
800	dev_warn(ctrl->ctrl.device,
801	"NVME-FC{%d}: Couldn't schedule reset.\n",
802	ctrl->cnum);
803	nvme_delete_ctrl(ctrl: &ctrl->ctrl);
804	}
805	break;
806
807	case NVME_CTRL_CONNECTING:
808	/*
809	* The association has already been terminated and the
810	* controller is attempting reconnects. No need to do anything
811	* futher. Reconnects will be attempted until either the
812	* ctlr_loss_tmo (max_retries * connect_delay) expires or the
813	* remoteport's dev_loss_tmo expires.
814	*/
815	break;
816
817	case NVME_CTRL_RESETTING:
818	/*
819	* Controller is already in the process of terminating the
820	* association. No need to do anything further. The reconnect
821	* step will kick in naturally after the association is
822	* terminated.
823	*/
824	break;
825
826	case NVME_CTRL_DELETING:
827	case NVME_CTRL_DELETING_NOIO:
828	default:
829	/* no action to take - let it delete */
830	break;
831	}
832	}
833
834	/**
835	* nvme_fc_unregister_remoteport - transport entry point called by an
836	* LLDD to deregister/remove a previously
837	* registered a NVME subsystem FC port.
838	* @portptr: pointer to the (registered) remote port that is to be
839	* deregistered.
840	*
841	* Returns:
842	* a completion status. Must be 0 upon success; a negative errno
843	* (ex: -ENXIO) upon failure.
844	*/
845	int
846	nvme_fc_unregister_remoteport(struct nvme_fc_remote_port *portptr)
847	{
848	struct nvme_fc_rport *rport = remoteport_to_rport(portptr);
849	struct nvme_fc_ctrl *ctrl;
850	unsigned long flags;
851
852	if (!portptr)
853	return -EINVAL;
854
855	spin_lock_irqsave(&rport->lock, flags);
856
857	if (portptr->port_state != FC_OBJSTATE_ONLINE) {
858	spin_unlock_irqrestore(lock: &rport->lock, flags);
859	return -EINVAL;
860	}
861	portptr->port_state = FC_OBJSTATE_DELETED;
862
863	rport->dev_loss_end = jiffies + (portptr->dev_loss_tmo * HZ);
864
865	list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
866	/* if dev_loss_tmo==0, dev loss is immediate */
867	if (!portptr->dev_loss_tmo) {
868	dev_warn(ctrl->ctrl.device,
869	"NVME-FC{%d}: controller connectivity lost.\n",
870	ctrl->cnum);
871	nvme_delete_ctrl(ctrl: &ctrl->ctrl);
872	} else
873	nvme_fc_ctrl_connectivity_loss(ctrl);
874	}
875
876	spin_unlock_irqrestore(lock: &rport->lock, flags);
877
878	nvme_fc_abort_lsops(rport);
879
880	if (atomic_read(v: &rport->act_ctrl_cnt) == 0)
881	rport->lport->ops->remoteport_delete(portptr);
882
883	/*
884	* release the reference, which will allow, if all controllers
885	* go away, which should only occur after dev_loss_tmo occurs,
886	* for the rport to be torn down.
887	*/
888	nvme_fc_rport_put(rport);
889
890	return 0;
891	}
892	EXPORT_SYMBOL_GPL(nvme_fc_unregister_remoteport);
893
894	/**
895	* nvme_fc_rescan_remoteport - transport entry point called by an
896	* LLDD to request a nvme device rescan.
897	* @remoteport: pointer to the (registered) remote port that is to be
898	* rescanned.
899	*
900	* Returns: N/A
901	*/
902	void
903	nvme_fc_rescan_remoteport(struct nvme_fc_remote_port *remoteport)
904	{
905	struct nvme_fc_rport *rport = remoteport_to_rport(portptr: remoteport);
906
907	nvme_fc_signal_discovery_scan(lport: rport->lport, rport);
908	}
909	EXPORT_SYMBOL_GPL(nvme_fc_rescan_remoteport);
910
911	int
912	nvme_fc_set_remoteport_devloss(struct nvme_fc_remote_port *portptr,
913	u32 dev_loss_tmo)
914	{
915	struct nvme_fc_rport *rport = remoteport_to_rport(portptr);
916	unsigned long flags;
917
918	spin_lock_irqsave(&rport->lock, flags);
919
920	if (portptr->port_state != FC_OBJSTATE_ONLINE) {
921	spin_unlock_irqrestore(lock: &rport->lock, flags);
922	return -EINVAL;
923	}
924
925	/* a dev_loss_tmo of 0 (immediate) is allowed to be set */
926	rport->remoteport.dev_loss_tmo = dev_loss_tmo;
927
928	spin_unlock_irqrestore(lock: &rport->lock, flags);
929
930	return 0;
931	}
932	EXPORT_SYMBOL_GPL(nvme_fc_set_remoteport_devloss);
933
934
935	/* *********************** FC-NVME DMA Handling **************************** */
936
937	/*
938	* The fcloop device passes in a NULL device pointer. Real LLD's will
939	* pass in a valid device pointer. If NULL is passed to the dma mapping
940	* routines, depending on the platform, it may or may not succeed, and
941	* may crash.
942	*
943	* As such:
944	* Wrapper all the dma routines and check the dev pointer.
945	*
946	* If simple mappings (return just a dma address, we'll noop them,
947	* returning a dma address of 0.
948	*
949	* On more complex mappings (dma_map_sg), a pseudo routine fills
950	* in the scatter list, setting all dma addresses to 0.
951	*/
952
953	static inline dma_addr_t
954	fc_dma_map_single(struct device *dev, void *ptr, size_t size,
955	enum dma_data_direction dir)
956	{
957	return dev ? dma_map_single(dev, ptr, size, dir) : (dma_addr_t)0L;
958	}
959
960	static inline int
961	fc_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
962	{
963	return dev ? dma_mapping_error(dev, dma_addr) : 0;
964	}
965
966	static inline void
967	fc_dma_unmap_single(struct device *dev, dma_addr_t addr, size_t size,
968	enum dma_data_direction dir)
969	{
970	if (dev)
971	dma_unmap_single(dev, addr, size, dir);
972	}
973
974	static inline void
975	fc_dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
976	enum dma_data_direction dir)
977	{
978	if (dev)
979	dma_sync_single_for_cpu(dev, addr, size, dir);
980	}
981
982	static inline void
983	fc_dma_sync_single_for_device(struct device *dev, dma_addr_t addr, size_t size,
984	enum dma_data_direction dir)
985	{
986	if (dev)
987	dma_sync_single_for_device(dev, addr, size, dir);
988	}
989
990	/* pseudo dma_map_sg call */
991	static int
992	fc_map_sg(struct scatterlist *sg, int nents)
993	{
994	struct scatterlist *s;
995	int i;
996
997	WARN_ON(nents == 0 || sg[0].length == 0);
998
999	for_each_sg(sg, s, nents, i) {
1000	s->dma_address = 0L;
1001	#ifdef CONFIG_NEED_SG_DMA_LENGTH
1002	s->dma_length = s->length;
1003	#endif
1004	}
1005	return nents;
1006	}
1007
1008	static inline int
1009	fc_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
1010	enum dma_data_direction dir)
1011	{
1012	return dev ? dma_map_sg(dev, sg, nents, dir) : fc_map_sg(sg, nents);
1013	}
1014
1015	static inline void
1016	fc_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
1017	enum dma_data_direction dir)
1018	{
1019	if (dev)
1020	dma_unmap_sg(dev, sg, nents, dir);
1021	}
1022
1023	/* *********************** FC-NVME LS Handling **************************** */
1024
1025	static void nvme_fc_ctrl_put(struct nvme_fc_ctrl *);
1026	static int nvme_fc_ctrl_get(struct nvme_fc_ctrl *);
1027
1028	static void nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl, char *errmsg);
1029
1030	static void
1031	__nvme_fc_finish_ls_req(struct nvmefc_ls_req_op *lsop)
1032	{
1033	struct nvme_fc_rport *rport = lsop->rport;
1034	struct nvmefc_ls_req *lsreq = &lsop->ls_req;
1035	unsigned long flags;
1036
1037	spin_lock_irqsave(&rport->lock, flags);
1038
1039	if (!lsop->req_queued) {
1040	spin_unlock_irqrestore(lock: &rport->lock, flags);
1041	return;
1042	}
1043
1044	list_del(entry: &lsop->lsreq_list);
1045
1046	lsop->req_queued = false;
1047
1048	spin_unlock_irqrestore(lock: &rport->lock, flags);
1049
1050	fc_dma_unmap_single(dev: rport->dev, addr: lsreq->rqstdma,
1051	size: (lsreq->rqstlen + lsreq->rsplen),
1052	dir: DMA_BIDIRECTIONAL);
1053
1054	nvme_fc_rport_put(rport);
1055	}
1056
1057	static int
1058	__nvme_fc_send_ls_req(struct nvme_fc_rport *rport,
1059	struct nvmefc_ls_req_op *lsop,
1060	void (*done)(struct nvmefc_ls_req *req, int status))
1061	{
1062	struct nvmefc_ls_req *lsreq = &lsop->ls_req;
1063	unsigned long flags;
1064	int ret = 0;
1065
1066	if (rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
1067	return -ECONNREFUSED;
1068
1069	if (!nvme_fc_rport_get(rport))
1070	return -ESHUTDOWN;
1071
1072	lsreq->done = done;
1073	lsop->rport = rport;
1074	lsop->req_queued = false;
1075	INIT_LIST_HEAD(list: &lsop->lsreq_list);
1076	init_completion(x: &lsop->ls_done);
1077
1078	lsreq->rqstdma = fc_dma_map_single(dev: rport->dev, ptr: lsreq->rqstaddr,
1079	size: lsreq->rqstlen + lsreq->rsplen,
1080	dir: DMA_BIDIRECTIONAL);
1081	if (fc_dma_mapping_error(dev: rport->dev, dma_addr: lsreq->rqstdma)) {
1082	ret = -EFAULT;
1083	goto out_putrport;
1084	}
1085	lsreq->rspdma = lsreq->rqstdma + lsreq->rqstlen;
1086
1087	spin_lock_irqsave(&rport->lock, flags);
1088
1089	list_add_tail(new: &lsop->lsreq_list, head: &rport->ls_req_list);
1090
1091	lsop->req_queued = true;
1092
1093	spin_unlock_irqrestore(lock: &rport->lock, flags);
1094
1095	ret = rport->lport->ops->ls_req(&rport->lport->localport,
1096	&rport->remoteport, lsreq);
1097	if (ret)
1098	goto out_unlink;
1099
1100	return 0;
1101
1102	out_unlink:
1103	lsop->ls_error = ret;
1104	spin_lock_irqsave(&rport->lock, flags);
1105	lsop->req_queued = false;
1106	list_del(entry: &lsop->lsreq_list);
1107	spin_unlock_irqrestore(lock: &rport->lock, flags);
1108	fc_dma_unmap_single(dev: rport->dev, addr: lsreq->rqstdma,
1109	size: (lsreq->rqstlen + lsreq->rsplen),
1110	dir: DMA_BIDIRECTIONAL);
1111	out_putrport:
1112	nvme_fc_rport_put(rport);
1113
1114	return ret;
1115	}
1116
1117	static void
1118	nvme_fc_send_ls_req_done(struct nvmefc_ls_req *lsreq, int status)
1119	{
1120	struct nvmefc_ls_req_op *lsop = ls_req_to_lsop(lsreq);
1121
1122	lsop->ls_error = status;
1123	complete(&lsop->ls_done);
1124	}
1125
1126	static int
1127	nvme_fc_send_ls_req(struct nvme_fc_rport *rport, struct nvmefc_ls_req_op *lsop)
1128	{
1129	struct nvmefc_ls_req *lsreq = &lsop->ls_req;
1130	struct fcnvme_ls_rjt *rjt = lsreq->rspaddr;
1131	int ret;
1132
1133	ret = __nvme_fc_send_ls_req(rport, lsop, done: nvme_fc_send_ls_req_done);
1134
1135	if (!ret) {
1136	/*
1137	* No timeout/not interruptible as we need the struct
1138	* to exist until the lldd calls us back. Thus mandate
1139	* wait until driver calls back. lldd responsible for
1140	* the timeout action
1141	*/
1142	wait_for_completion(&lsop->ls_done);
1143
1144	__nvme_fc_finish_ls_req(lsop);
1145
1146	ret = lsop->ls_error;
1147	}
1148
1149	if (ret)
1150	return ret;
1151
1152	/* ACC or RJT payload ? */
1153	if (rjt->w0.ls_cmd == FCNVME_LS_RJT)
1154	return -ENXIO;
1155
1156	return 0;
1157	}
1158
1159	static int
1160	nvme_fc_send_ls_req_async(struct nvme_fc_rport *rport,
1161	struct nvmefc_ls_req_op *lsop,
1162	void (*done)(struct nvmefc_ls_req *req, int status))
1163	{
1164	/* don't wait for completion */
1165
1166	return __nvme_fc_send_ls_req(rport, lsop, done);
1167	}
1168
1169	static int
1170	nvme_fc_connect_admin_queue(struct nvme_fc_ctrl *ctrl,
1171	struct nvme_fc_queue *queue, u16 qsize, u16 ersp_ratio)
1172	{
1173	struct nvmefc_ls_req_op *lsop;
1174	struct nvmefc_ls_req *lsreq;
1175	struct fcnvme_ls_cr_assoc_rqst *assoc_rqst;
1176	struct fcnvme_ls_cr_assoc_acc *assoc_acc;
1177	unsigned long flags;
1178	int ret, fcret = 0;
1179
1180	lsop = kzalloc(size: (sizeof(*lsop) +
1181	sizeof(*assoc_rqst) + sizeof(*assoc_acc) +
1182	ctrl->lport->ops->lsrqst_priv_sz), GFP_KERNEL);
1183	if (!lsop) {
1184	dev_info(ctrl->ctrl.device,
1185	"NVME-FC{%d}: send Create Association failed: ENOMEM\n",
1186	ctrl->cnum);
1187	ret = -ENOMEM;
1188	goto out_no_memory;
1189	}
1190
1191	assoc_rqst = (struct fcnvme_ls_cr_assoc_rqst *)&lsop[1];
1192	assoc_acc = (struct fcnvme_ls_cr_assoc_acc *)&assoc_rqst[1];
1193	lsreq = &lsop->ls_req;
1194	if (ctrl->lport->ops->lsrqst_priv_sz)
1195	lsreq->private = &assoc_acc[1];
1196	else
1197	lsreq->private = NULL;
1198
1199	assoc_rqst->w0.ls_cmd = FCNVME_LS_CREATE_ASSOCIATION;
1200	assoc_rqst->desc_list_len =
1201	cpu_to_be32(sizeof(struct fcnvme_lsdesc_cr_assoc_cmd));
1202
1203	assoc_rqst->assoc_cmd.desc_tag =
1204	cpu_to_be32(FCNVME_LSDESC_CREATE_ASSOC_CMD);
1205	assoc_rqst->assoc_cmd.desc_len =
1206	fcnvme_lsdesc_len(
1207	sz: sizeof(struct fcnvme_lsdesc_cr_assoc_cmd));
1208
1209	assoc_rqst->assoc_cmd.ersp_ratio = cpu_to_be16(ersp_ratio);
1210	assoc_rqst->assoc_cmd.sqsize = cpu_to_be16(qsize - 1);
1211	/* Linux supports only Dynamic controllers */
1212	assoc_rqst->assoc_cmd.cntlid = cpu_to_be16(0xffff);
1213	uuid_copy(dst: &assoc_rqst->assoc_cmd.hostid, src: &ctrl->ctrl.opts->host->id);
1214	strscpy(assoc_rqst->assoc_cmd.hostnqn, ctrl->ctrl.opts->host->nqn,
1215	sizeof(assoc_rqst->assoc_cmd.hostnqn));
1216	strscpy(assoc_rqst->assoc_cmd.subnqn, ctrl->ctrl.opts->subsysnqn,
1217	sizeof(assoc_rqst->assoc_cmd.subnqn));
1218
1219	lsop->queue = queue;
1220	lsreq->rqstaddr = assoc_rqst;
1221	lsreq->rqstlen = sizeof(*assoc_rqst);
1222	lsreq->rspaddr = assoc_acc;
1223	lsreq->rsplen = sizeof(*assoc_acc);
1224	lsreq->timeout = NVME_FC_LS_TIMEOUT_SEC;
1225
1226	ret = nvme_fc_send_ls_req(rport: ctrl->rport, lsop);
1227	if (ret)
1228	goto out_free_buffer;
1229
1230	/* process connect LS completion */
1231
1232	/* validate the ACC response */
1233	if (assoc_acc->hdr.w0.ls_cmd != FCNVME_LS_ACC)
1234	fcret = VERR_LSACC;
1235	else if (assoc_acc->hdr.desc_list_len !=
1236	fcnvme_lsdesc_len(
1237	sz: sizeof(struct fcnvme_ls_cr_assoc_acc)))
1238	fcret = VERR_CR_ASSOC_ACC_LEN;
1239	else if (assoc_acc->hdr.rqst.desc_tag !=
1240	cpu_to_be32(FCNVME_LSDESC_RQST))
1241	fcret = VERR_LSDESC_RQST;
1242	else if (assoc_acc->hdr.rqst.desc_len !=
1243	fcnvme_lsdesc_len(sz: sizeof(struct fcnvme_lsdesc_rqst)))
1244	fcret = VERR_LSDESC_RQST_LEN;
1245	else if (assoc_acc->hdr.rqst.w0.ls_cmd != FCNVME_LS_CREATE_ASSOCIATION)
1246	fcret = VERR_CR_ASSOC;
1247	else if (assoc_acc->associd.desc_tag !=
1248	cpu_to_be32(FCNVME_LSDESC_ASSOC_ID))
1249	fcret = VERR_ASSOC_ID;
1250	else if (assoc_acc->associd.desc_len !=
1251	fcnvme_lsdesc_len(
1252	sz: sizeof(struct fcnvme_lsdesc_assoc_id)))
1253	fcret = VERR_ASSOC_ID_LEN;
1254	else if (assoc_acc->connectid.desc_tag !=
1255	cpu_to_be32(FCNVME_LSDESC_CONN_ID))
1256	fcret = VERR_CONN_ID;
1257	else if (assoc_acc->connectid.desc_len !=
1258	fcnvme_lsdesc_len(sz: sizeof(struct fcnvme_lsdesc_conn_id)))
1259	fcret = VERR_CONN_ID_LEN;
1260
1261	if (fcret) {
1262	ret = -EBADF;
1263	dev_err(ctrl->dev,
1264	"q %d Create Association LS failed: %s\n",
1265	queue->qnum, validation_errors[fcret]);
1266	} else {
1267	spin_lock_irqsave(&ctrl->lock, flags);
1268	ctrl->association_id =
1269	be64_to_cpu(assoc_acc->associd.association_id);
1270	queue->connection_id =
1271	be64_to_cpu(assoc_acc->connectid.connection_id);
1272	set_bit(nr: NVME_FC_Q_CONNECTED, addr: &queue->flags);
1273	spin_unlock_irqrestore(lock: &ctrl->lock, flags);
1274	}
1275
1276	out_free_buffer:
1277	kfree(objp: lsop);
1278	out_no_memory:
1279	if (ret)
1280	dev_err(ctrl->dev,
1281	"queue %d connect admin queue failed (%d).\n",
1282	queue->qnum, ret);
1283	return ret;
1284	}
1285
1286	static int
1287	nvme_fc_connect_queue(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue,
1288	u16 qsize, u16 ersp_ratio)
1289	{
1290	struct nvmefc_ls_req_op *lsop;
1291	struct nvmefc_ls_req *lsreq;
1292	struct fcnvme_ls_cr_conn_rqst *conn_rqst;
1293	struct fcnvme_ls_cr_conn_acc *conn_acc;
1294	int ret, fcret = 0;
1295
1296	lsop = kzalloc(size: (sizeof(*lsop) +
1297	sizeof(*conn_rqst) + sizeof(*conn_acc) +
1298	ctrl->lport->ops->lsrqst_priv_sz), GFP_KERNEL);
1299	if (!lsop) {
1300	dev_info(ctrl->ctrl.device,
1301	"NVME-FC{%d}: send Create Connection failed: ENOMEM\n",
1302	ctrl->cnum);
1303	ret = -ENOMEM;
1304	goto out_no_memory;
1305	}
1306
1307	conn_rqst = (struct fcnvme_ls_cr_conn_rqst *)&lsop[1];
1308	conn_acc = (struct fcnvme_ls_cr_conn_acc *)&conn_rqst[1];
1309	lsreq = &lsop->ls_req;
1310	if (ctrl->lport->ops->lsrqst_priv_sz)
1311	lsreq->private = (void *)&conn_acc[1];
1312	else
1313	lsreq->private = NULL;
1314
1315	conn_rqst->w0.ls_cmd = FCNVME_LS_CREATE_CONNECTION;
1316	conn_rqst->desc_list_len = cpu_to_be32(
1317	sizeof(struct fcnvme_lsdesc_assoc_id) +
1318	sizeof(struct fcnvme_lsdesc_cr_conn_cmd));
1319
1320	conn_rqst->associd.desc_tag = cpu_to_be32(FCNVME_LSDESC_ASSOC_ID);
1321	conn_rqst->associd.desc_len =
1322	fcnvme_lsdesc_len(
1323	sz: sizeof(struct fcnvme_lsdesc_assoc_id));
1324	conn_rqst->associd.association_id = cpu_to_be64(ctrl->association_id);
1325	conn_rqst->connect_cmd.desc_tag =
1326	cpu_to_be32(FCNVME_LSDESC_CREATE_CONN_CMD);
1327	conn_rqst->connect_cmd.desc_len =
1328	fcnvme_lsdesc_len(
1329	sz: sizeof(struct fcnvme_lsdesc_cr_conn_cmd));
1330	conn_rqst->connect_cmd.ersp_ratio = cpu_to_be16(ersp_ratio);
1331	conn_rqst->connect_cmd.qid = cpu_to_be16(queue->qnum);
1332	conn_rqst->connect_cmd.sqsize = cpu_to_be16(qsize - 1);
1333
1334	lsop->queue = queue;
1335	lsreq->rqstaddr = conn_rqst;
1336	lsreq->rqstlen = sizeof(*conn_rqst);
1337	lsreq->rspaddr = conn_acc;
1338	lsreq->rsplen = sizeof(*conn_acc);
1339	lsreq->timeout = NVME_FC_LS_TIMEOUT_SEC;
1340
1341	ret = nvme_fc_send_ls_req(rport: ctrl->rport, lsop);
1342	if (ret)
1343	goto out_free_buffer;
1344
1345	/* process connect LS completion */
1346
1347	/* validate the ACC response */
1348	if (conn_acc->hdr.w0.ls_cmd != FCNVME_LS_ACC)
1349	fcret = VERR_LSACC;
1350	else if (conn_acc->hdr.desc_list_len !=
1351	fcnvme_lsdesc_len(sz: sizeof(struct fcnvme_ls_cr_conn_acc)))
1352	fcret = VERR_CR_CONN_ACC_LEN;
1353	else if (conn_acc->hdr.rqst.desc_tag != cpu_to_be32(FCNVME_LSDESC_RQST))
1354	fcret = VERR_LSDESC_RQST;
1355	else if (conn_acc->hdr.rqst.desc_len !=
1356	fcnvme_lsdesc_len(sz: sizeof(struct fcnvme_lsdesc_rqst)))
1357	fcret = VERR_LSDESC_RQST_LEN;
1358	else if (conn_acc->hdr.rqst.w0.ls_cmd != FCNVME_LS_CREATE_CONNECTION)
1359	fcret = VERR_CR_CONN;
1360	else if (conn_acc->connectid.desc_tag !=
1361	cpu_to_be32(FCNVME_LSDESC_CONN_ID))
1362	fcret = VERR_CONN_ID;
1363	else if (conn_acc->connectid.desc_len !=
1364	fcnvme_lsdesc_len(sz: sizeof(struct fcnvme_lsdesc_conn_id)))
1365	fcret = VERR_CONN_ID_LEN;
1366
1367	if (fcret) {
1368	ret = -EBADF;
1369	dev_err(ctrl->dev,
1370	"q %d Create I/O Connection LS failed: %s\n",
1371	queue->qnum, validation_errors[fcret]);
1372	} else {
1373	queue->connection_id =
1374	be64_to_cpu(conn_acc->connectid.connection_id);
1375	set_bit(nr: NVME_FC_Q_CONNECTED, addr: &queue->flags);
1376	}
1377
1378	out_free_buffer:
1379	kfree(objp: lsop);
1380	out_no_memory:
1381	if (ret)
1382	dev_err(ctrl->dev,
1383	"queue %d connect I/O queue failed (%d).\n",
1384	queue->qnum, ret);
1385	return ret;
1386	}
1387
1388	static void
1389	nvme_fc_disconnect_assoc_done(struct nvmefc_ls_req *lsreq, int status)
1390	{
1391	struct nvmefc_ls_req_op *lsop = ls_req_to_lsop(lsreq);
1392
1393	__nvme_fc_finish_ls_req(lsop);
1394
1395	/* fc-nvme initiator doesn't care about success or failure of cmd */
1396
1397	kfree(objp: lsop);
1398	}
1399
1400	/*
1401	* This routine sends a FC-NVME LS to disconnect (aka terminate)
1402	* the FC-NVME Association. Terminating the association also
1403	* terminates the FC-NVME connections (per queue, both admin and io
1404	* queues) that are part of the association. E.g. things are torn
1405	* down, and the related FC-NVME Association ID and Connection IDs
1406	* become invalid.
1407	*
1408	* The behavior of the fc-nvme initiator is such that it's
1409	* understanding of the association and connections will implicitly
1410	* be torn down. The action is implicit as it may be due to a loss of
1411	* connectivity with the fc-nvme target, so you may never get a
1412	* response even if you tried. As such, the action of this routine
1413	* is to asynchronously send the LS, ignore any results of the LS, and
1414	* continue on with terminating the association. If the fc-nvme target
1415	* is present and receives the LS, it too can tear down.
1416	*/
1417	static void
1418	nvme_fc_xmt_disconnect_assoc(struct nvme_fc_ctrl *ctrl)
1419	{
1420	struct fcnvme_ls_disconnect_assoc_rqst *discon_rqst;
1421	struct fcnvme_ls_disconnect_assoc_acc *discon_acc;
1422	struct nvmefc_ls_req_op *lsop;
1423	struct nvmefc_ls_req *lsreq;
1424	int ret;
1425
1426	lsop = kzalloc(size: (sizeof(*lsop) +
1427	sizeof(*discon_rqst) + sizeof(*discon_acc) +
1428	ctrl->lport->ops->lsrqst_priv_sz), GFP_KERNEL);
1429	if (!lsop) {
1430	dev_info(ctrl->ctrl.device,
1431	"NVME-FC{%d}: send Disconnect Association "
1432	"failed: ENOMEM\n",
1433	ctrl->cnum);
1434	return;
1435	}
1436
1437	discon_rqst = (struct fcnvme_ls_disconnect_assoc_rqst *)&lsop[1];
1438	discon_acc = (struct fcnvme_ls_disconnect_assoc_acc *)&discon_rqst[1];
1439	lsreq = &lsop->ls_req;
1440	if (ctrl->lport->ops->lsrqst_priv_sz)
1441	lsreq->private = (void *)&discon_acc[1];
1442	else
1443	lsreq->private = NULL;
1444
1445	nvmefc_fmt_lsreq_discon_assoc(lsreq, discon_rqst, discon_acc,
1446	association_id: ctrl->association_id);
1447
1448	ret = nvme_fc_send_ls_req_async(rport: ctrl->rport, lsop,
1449	done: nvme_fc_disconnect_assoc_done);
1450	if (ret)
1451	kfree(objp: lsop);
1452	}
1453
1454	static void
1455	nvme_fc_xmt_ls_rsp_done(struct nvmefc_ls_rsp *lsrsp)
1456	{
1457	struct nvmefc_ls_rcv_op *lsop = lsrsp->nvme_fc_private;
1458	struct nvme_fc_rport *rport = lsop->rport;
1459	struct nvme_fc_lport *lport = rport->lport;
1460	unsigned long flags;
1461
1462	spin_lock_irqsave(&rport->lock, flags);
1463	list_del(entry: &lsop->lsrcv_list);
1464	spin_unlock_irqrestore(lock: &rport->lock, flags);
1465
1466	fc_dma_sync_single_for_cpu(dev: lport->dev, addr: lsop->rspdma,
1467	size: sizeof(*lsop->rspbuf), dir: DMA_TO_DEVICE);
1468	fc_dma_unmap_single(dev: lport->dev, addr: lsop->rspdma,
1469	size: sizeof(*lsop->rspbuf), dir: DMA_TO_DEVICE);
1470
1471	kfree(objp: lsop->rspbuf);
1472	kfree(objp: lsop->rqstbuf);
1473	kfree(objp: lsop);
1474
1475	nvme_fc_rport_put(rport);
1476	}
1477
1478	static void
1479	nvme_fc_xmt_ls_rsp(struct nvmefc_ls_rcv_op *lsop)
1480	{
1481	struct nvme_fc_rport *rport = lsop->rport;
1482	struct nvme_fc_lport *lport = rport->lport;
1483	struct fcnvme_ls_rqst_w0 *w0 = &lsop->rqstbuf->w0;
1484	int ret;
1485
1486	fc_dma_sync_single_for_device(dev: lport->dev, addr: lsop->rspdma,
1487	size: sizeof(*lsop->rspbuf), dir: DMA_TO_DEVICE);
1488
1489	ret = lport->ops->xmt_ls_rsp(&lport->localport, &rport->remoteport,
1490	lsop->lsrsp);
1491	if (ret) {
1492	dev_warn(lport->dev,
1493	"LLDD rejected LS RSP xmt: LS %d status %d\n",
1494	w0->ls_cmd, ret);
1495	nvme_fc_xmt_ls_rsp_done(lsrsp: lsop->lsrsp);
1496	return;
1497	}
1498	}
1499
1500	static struct nvme_fc_ctrl *
1501	nvme_fc_match_disconn_ls(struct nvme_fc_rport *rport,
1502	struct nvmefc_ls_rcv_op *lsop)
1503	{
1504	struct fcnvme_ls_disconnect_assoc_rqst *rqst =
1505	&lsop->rqstbuf->rq_dis_assoc;
1506	struct nvme_fc_ctrl *ctrl, *ret = NULL;
1507	struct nvmefc_ls_rcv_op *oldls = NULL;
1508	u64 association_id = be64_to_cpu(rqst->associd.association_id);
1509	unsigned long flags;
1510
1511	spin_lock_irqsave(&rport->lock, flags);
1512
1513	list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
1514	if (!nvme_fc_ctrl_get(ctrl))
1515	continue;
1516	spin_lock(lock: &ctrl->lock);
1517	if (association_id == ctrl->association_id) {
1518	oldls = ctrl->rcv_disconn;
1519	ctrl->rcv_disconn = lsop;
1520	ret = ctrl;
1521	}
1522	spin_unlock(lock: &ctrl->lock);
1523	if (ret)
1524	/* leave the ctrl get reference */
1525	break;
1526	nvme_fc_ctrl_put(ctrl);
1527	}
1528
1529	spin_unlock_irqrestore(lock: &rport->lock, flags);
1530
1531	/* transmit a response for anything that was pending */
1532	if (oldls) {
1533	dev_info(rport->lport->dev,
1534	"NVME-FC{%d}: Multiple Disconnect Association "
1535	"LS's received\n", ctrl->cnum);
1536	/* overwrite good response with bogus failure */
1537	oldls->lsrsp->rsplen = nvme_fc_format_rjt(buf: oldls->rspbuf,
1538	buflen: sizeof(*oldls->rspbuf),
1539	ls_cmd: rqst->w0.ls_cmd,
1540	reason: FCNVME_RJT_RC_UNAB,
1541	explanation: FCNVME_RJT_EXP_NONE, vendor: 0);
1542	nvme_fc_xmt_ls_rsp(lsop: oldls);
1543	}
1544
1545	return ret;
1546	}
1547
1548	/*
1549	* returns true to mean LS handled and ls_rsp can be sent
1550	* returns false to defer ls_rsp xmt (will be done as part of
1551	* association termination)
1552	*/
1553	static bool
1554	nvme_fc_ls_disconnect_assoc(struct nvmefc_ls_rcv_op *lsop)
1555	{
1556	struct nvme_fc_rport *rport = lsop->rport;
1557	struct fcnvme_ls_disconnect_assoc_rqst *rqst =
1558	&lsop->rqstbuf->rq_dis_assoc;
1559	struct fcnvme_ls_disconnect_assoc_acc *acc =
1560	&lsop->rspbuf->rsp_dis_assoc;
1561	struct nvme_fc_ctrl *ctrl = NULL;
1562	int ret = 0;
1563
1564	memset(acc, 0, sizeof(*acc));
1565
1566	ret = nvmefc_vldt_lsreq_discon_assoc(rqstlen: lsop->rqstdatalen, rqst);
1567	if (!ret) {
1568	/* match an active association */
1569	ctrl = nvme_fc_match_disconn_ls(rport, lsop);
1570	if (!ctrl)
1571	ret = VERR_NO_ASSOC;
1572	}
1573
1574	if (ret) {
1575	dev_info(rport->lport->dev,
1576	"Disconnect LS failed: %s\n",
1577	validation_errors[ret]);
1578	lsop->lsrsp->rsplen = nvme_fc_format_rjt(buf: acc,
1579	buflen: sizeof(*acc), ls_cmd: rqst->w0.ls_cmd,
1580	reason: (ret == VERR_NO_ASSOC) ?
1581	FCNVME_RJT_RC_INV_ASSOC :
1582	FCNVME_RJT_RC_LOGIC,
1583	explanation: FCNVME_RJT_EXP_NONE, vendor: 0);
1584	return true;
1585	}
1586
1587	/* format an ACCept response */
1588
1589	lsop->lsrsp->rsplen = sizeof(*acc);
1590
1591	nvme_fc_format_rsp_hdr(buf: acc, ls_cmd: FCNVME_LS_ACC,
1592	desc_len: fcnvme_lsdesc_len(
1593	sz: sizeof(struct fcnvme_ls_disconnect_assoc_acc)),
1594	rqst_ls_cmd: FCNVME_LS_DISCONNECT_ASSOC);
1595
1596	/*
1597	* the transmit of the response will occur after the exchanges
1598	* for the association have been ABTS'd by
1599	* nvme_fc_delete_association().
1600	*/
1601
1602	/* fail the association */
1603	nvme_fc_error_recovery(ctrl, errmsg: "Disconnect Association LS received");
1604
1605	/* release the reference taken by nvme_fc_match_disconn_ls() */
1606	nvme_fc_ctrl_put(ctrl);
1607
1608	return false;
1609	}
1610
1611	/*
1612	* Actual Processing routine for received FC-NVME LS Requests from the LLD
1613	* returns true if a response should be sent afterward, false if rsp will
1614	* be sent asynchronously.
1615	*/
1616	static bool
1617	nvme_fc_handle_ls_rqst(struct nvmefc_ls_rcv_op *lsop)
1618	{
1619	struct fcnvme_ls_rqst_w0 *w0 = &lsop->rqstbuf->w0;
1620	bool ret = true;
1621
1622	lsop->lsrsp->nvme_fc_private = lsop;
1623	lsop->lsrsp->rspbuf = lsop->rspbuf;
1624	lsop->lsrsp->rspdma = lsop->rspdma;
1625	lsop->lsrsp->done = nvme_fc_xmt_ls_rsp_done;
1626	/* Be preventative. handlers will later set to valid length */
1627	lsop->lsrsp->rsplen = 0;
1628
1629	/*
1630	* handlers:
1631	* parse request input, execute the request, and format the
1632	* LS response
1633	*/
1634	switch (w0->ls_cmd) {
1635	case FCNVME_LS_DISCONNECT_ASSOC:
1636	ret = nvme_fc_ls_disconnect_assoc(lsop);
1637	break;
1638	case FCNVME_LS_DISCONNECT_CONN:
1639	lsop->lsrsp->rsplen = nvme_fc_format_rjt(buf: lsop->rspbuf,
1640	buflen: sizeof(*lsop->rspbuf), ls_cmd: w0->ls_cmd,
1641	reason: FCNVME_RJT_RC_UNSUP, explanation: FCNVME_RJT_EXP_NONE, vendor: 0);
1642	break;
1643	case FCNVME_LS_CREATE_ASSOCIATION:
1644	case FCNVME_LS_CREATE_CONNECTION:
1645	lsop->lsrsp->rsplen = nvme_fc_format_rjt(buf: lsop->rspbuf,
1646	buflen: sizeof(*lsop->rspbuf), ls_cmd: w0->ls_cmd,
1647	reason: FCNVME_RJT_RC_LOGIC, explanation: FCNVME_RJT_EXP_NONE, vendor: 0);
1648	break;
1649	default:
1650	lsop->lsrsp->rsplen = nvme_fc_format_rjt(buf: lsop->rspbuf,
1651	buflen: sizeof(*lsop->rspbuf), ls_cmd: w0->ls_cmd,
1652	reason: FCNVME_RJT_RC_INVAL, explanation: FCNVME_RJT_EXP_NONE, vendor: 0);
1653	break;
1654	}
1655
1656	return(ret);
1657	}
1658
1659	static void
1660	nvme_fc_handle_ls_rqst_work(struct work_struct *work)
1661	{
1662	struct nvme_fc_rport *rport =
1663	container_of(work, struct nvme_fc_rport, lsrcv_work);
1664	struct fcnvme_ls_rqst_w0 *w0;
1665	struct nvmefc_ls_rcv_op *lsop;
1666	unsigned long flags;
1667	bool sendrsp;
1668
1669	restart:
1670	sendrsp = true;
1671	spin_lock_irqsave(&rport->lock, flags);
1672	list_for_each_entry(lsop, &rport->ls_rcv_list, lsrcv_list) {
1673	if (lsop->handled)
1674	continue;
1675
1676	lsop->handled = true;
1677	if (rport->remoteport.port_state == FC_OBJSTATE_ONLINE) {
1678	spin_unlock_irqrestore(lock: &rport->lock, flags);
1679	sendrsp = nvme_fc_handle_ls_rqst(lsop);
1680	} else {
1681	spin_unlock_irqrestore(lock: &rport->lock, flags);
1682	w0 = &lsop->rqstbuf->w0;
1683	lsop->lsrsp->rsplen = nvme_fc_format_rjt(
1684	buf: lsop->rspbuf,
1685	buflen: sizeof(*lsop->rspbuf),
1686	ls_cmd: w0->ls_cmd,
1687	reason: FCNVME_RJT_RC_UNAB,
1688	explanation: FCNVME_RJT_EXP_NONE, vendor: 0);
1689	}
1690	if (sendrsp)
1691	nvme_fc_xmt_ls_rsp(lsop);
1692	goto restart;
1693	}
1694	spin_unlock_irqrestore(lock: &rport->lock, flags);
1695	}
1696
1697	static
1698	void nvme_fc_rcv_ls_req_err_msg(struct nvme_fc_lport *lport,
1699	struct fcnvme_ls_rqst_w0 *w0)
1700	{
1701	dev_info(lport->dev, "RCV %s LS failed: No memory\n",
1702	(w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
1703	nvmefc_ls_names[w0->ls_cmd] : "");
1704	}
1705
1706	/**
1707	* nvme_fc_rcv_ls_req - transport entry point called by an LLDD
1708	* upon the reception of a NVME LS request.
1709	*
1710	* The nvme-fc layer will copy payload to an internal structure for
1711	* processing. As such, upon completion of the routine, the LLDD may
1712	* immediately free/reuse the LS request buffer passed in the call.
1713	*
1714	* If this routine returns error, the LLDD should abort the exchange.
1715	*
1716	* @portptr: pointer to the (registered) remote port that the LS
1717	* was received from. The remoteport is associated with
1718	* a specific localport.
1719	* @lsrsp: pointer to a nvmefc_ls_rsp response structure to be
1720	* used to reference the exchange corresponding to the LS
1721	* when issuing an ls response.
1722	* @lsreqbuf: pointer to the buffer containing the LS Request
1723	* @lsreqbuf_len: length, in bytes, of the received LS request
1724	*/
1725	int
1726	nvme_fc_rcv_ls_req(struct nvme_fc_remote_port *portptr,
1727	struct nvmefc_ls_rsp *lsrsp,
1728	void *lsreqbuf, u32 lsreqbuf_len)
1729	{
1730	struct nvme_fc_rport *rport = remoteport_to_rport(portptr);
1731	struct nvme_fc_lport *lport = rport->lport;
1732	struct fcnvme_ls_rqst_w0 *w0 = (struct fcnvme_ls_rqst_w0 *)lsreqbuf;
1733	struct nvmefc_ls_rcv_op *lsop;
1734	unsigned long flags;
1735	int ret;
1736
1737	nvme_fc_rport_get(rport);
1738
1739	/* validate there's a routine to transmit a response */
1740	if (!lport->ops->xmt_ls_rsp) {
1741	dev_info(lport->dev,
1742	"RCV %s LS failed: no LLDD xmt_ls_rsp\n",
1743	(w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
1744	nvmefc_ls_names[w0->ls_cmd] : "");
1745	ret = -EINVAL;
1746	goto out_put;
1747	}
1748
1749	if (lsreqbuf_len > sizeof(union nvmefc_ls_requests)) {
1750	dev_info(lport->dev,
1751	"RCV %s LS failed: payload too large\n",
1752	(w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
1753	nvmefc_ls_names[w0->ls_cmd] : "");
1754	ret = -E2BIG;
1755	goto out_put;
1756	}
1757
1758	lsop = kzalloc(size: sizeof(*lsop), GFP_KERNEL);
1759	if (!lsop) {
1760	nvme_fc_rcv_ls_req_err_msg(lport, w0);
1761	ret = -ENOMEM;
1762	goto out_put;
1763	}
1764
1765	lsop->rqstbuf = kzalloc(size: sizeof(*lsop->rqstbuf), GFP_KERNEL);
1766	lsop->rspbuf = kzalloc(size: sizeof(*lsop->rspbuf), GFP_KERNEL);
1767	if (!lsop->rqstbuf || !lsop->rspbuf) {
1768	nvme_fc_rcv_ls_req_err_msg(lport, w0);
1769	ret = -ENOMEM;
1770	goto out_free;
1771	}
1772
1773	lsop->rspdma = fc_dma_map_single(dev: lport->dev, ptr: lsop->rspbuf,
1774	size: sizeof(*lsop->rspbuf),
1775	dir: DMA_TO_DEVICE);
1776	if (fc_dma_mapping_error(dev: lport->dev, dma_addr: lsop->rspdma)) {
1777	dev_info(lport->dev,
1778	"RCV %s LS failed: DMA mapping failure\n",
1779	(w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
1780	nvmefc_ls_names[w0->ls_cmd] : "");
1781	ret = -EFAULT;
1782	goto out_free;
1783	}
1784
1785	lsop->rport = rport;
1786	lsop->lsrsp = lsrsp;
1787
1788	memcpy(lsop->rqstbuf, lsreqbuf, lsreqbuf_len);
1789	lsop->rqstdatalen = lsreqbuf_len;
1790
1791	spin_lock_irqsave(&rport->lock, flags);
1792	if (rport->remoteport.port_state != FC_OBJSTATE_ONLINE) {
1793	spin_unlock_irqrestore(lock: &rport->lock, flags);
1794	ret = -ENOTCONN;
1795	goto out_unmap;
1796	}
1797	list_add_tail(new: &lsop->lsrcv_list, head: &rport->ls_rcv_list);
1798	spin_unlock_irqrestore(lock: &rport->lock, flags);
1799
1800	schedule_work(work: &rport->lsrcv_work);
1801
1802	return 0;
1803
1804	out_unmap:
1805	fc_dma_unmap_single(dev: lport->dev, addr: lsop->rspdma,
1806	size: sizeof(*lsop->rspbuf), dir: DMA_TO_DEVICE);
1807	out_free:
1808	kfree(objp: lsop->rspbuf);
1809	kfree(objp: lsop->rqstbuf);
1810	kfree(objp: lsop);
1811	out_put:
1812	nvme_fc_rport_put(rport);
1813	return ret;
1814	}
1815	EXPORT_SYMBOL_GPL(nvme_fc_rcv_ls_req);
1816
1817
1818	/* *********************** NVME Ctrl Routines **************************** */
1819
1820	static void
1821	__nvme_fc_exit_request(struct nvme_fc_ctrl *ctrl,
1822	struct nvme_fc_fcp_op *op)
1823	{
1824	fc_dma_unmap_single(dev: ctrl->lport->dev, addr: op->fcp_req.rspdma,
1825	size: sizeof(op->rsp_iu), dir: DMA_FROM_DEVICE);
1826	fc_dma_unmap_single(dev: ctrl->lport->dev, addr: op->fcp_req.cmddma,
1827	size: sizeof(op->cmd_iu), dir: DMA_TO_DEVICE);
1828
1829	atomic_set(v: &op->state, i: FCPOP_STATE_UNINIT);
1830	}
1831
1832	static void
1833	nvme_fc_exit_request(struct blk_mq_tag_set *set, struct request *rq,
1834	unsigned int hctx_idx)
1835	{
1836	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
1837
1838	return __nvme_fc_exit_request(ctrl: to_fc_ctrl(ctrl: set->driver_data), op);
1839	}
1840
1841	static int
1842	__nvme_fc_abort_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_fcp_op *op)
1843	{
1844	unsigned long flags;
1845	int opstate;
1846
1847	spin_lock_irqsave(&ctrl->lock, flags);
1848	opstate = atomic_xchg(v: &op->state, new: FCPOP_STATE_ABORTED);
1849	if (opstate != FCPOP_STATE_ACTIVE)
1850	atomic_set(v: &op->state, i: opstate);
1851	else if (test_bit(FCCTRL_TERMIO, &ctrl->flags)) {
1852	op->flags |= FCOP_FLAGS_TERMIO;
1853	ctrl->iocnt++;
1854	}
1855	spin_unlock_irqrestore(lock: &ctrl->lock, flags);
1856
1857	if (opstate != FCPOP_STATE_ACTIVE)
1858	return -ECANCELED;
1859
1860	ctrl->lport->ops->fcp_abort(&ctrl->lport->localport,
1861	&ctrl->rport->remoteport,
1862	op->queue->lldd_handle,
1863	&op->fcp_req);
1864
1865	return 0;
1866	}
1867
1868	static void
1869	nvme_fc_abort_aen_ops(struct nvme_fc_ctrl *ctrl)
1870	{
1871	struct nvme_fc_fcp_op *aen_op = ctrl->aen_ops;
1872	int i;
1873
1874	/* ensure we've initialized the ops once */
1875	if (!(aen_op->flags & FCOP_FLAGS_AEN))
1876	return;
1877
1878	for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++)
1879	__nvme_fc_abort_op(ctrl, op: aen_op);
1880	}
1881
1882	static inline void
1883	__nvme_fc_fcpop_chk_teardowns(struct nvme_fc_ctrl *ctrl,
1884	struct nvme_fc_fcp_op *op, int opstate)
1885	{
1886	unsigned long flags;
1887
1888	if (opstate == FCPOP_STATE_ABORTED) {
1889	spin_lock_irqsave(&ctrl->lock, flags);
1890	if (test_bit(FCCTRL_TERMIO, &ctrl->flags) &&
1891	op->flags & FCOP_FLAGS_TERMIO) {
1892	if (!--ctrl->iocnt)
1893	wake_up(&ctrl->ioabort_wait);
1894	}
1895	spin_unlock_irqrestore(lock: &ctrl->lock, flags);
1896	}
1897	}
1898
1899	static void
1900	nvme_fc_ctrl_ioerr_work(struct work_struct *work)
1901	{
1902	struct nvme_fc_ctrl *ctrl =
1903	container_of(work, struct nvme_fc_ctrl, ioerr_work);
1904
1905	nvme_fc_error_recovery(ctrl, errmsg: "transport detected io error");
1906	}
1907
1908	/*
1909	* nvme_fc_io_getuuid - Routine called to get the appid field
1910	* associated with request by the lldd
1911	* @req:IO request from nvme fc to driver
1912	* Returns: UUID if there is an appid associated with VM or
1913	* NULL if the user/libvirt has not set the appid to VM
1914	*/
1915	char *nvme_fc_io_getuuid(struct nvmefc_fcp_req *req)
1916	{
1917	struct nvme_fc_fcp_op *op = fcp_req_to_fcp_op(fcpreq: req);
1918	struct request *rq = op->rq;
1919
1920	if (!IS_ENABLED(CONFIG_BLK_CGROUP_FC_APPID) || !rq || !rq->bio)
1921	return NULL;
1922	return blkcg_get_fc_appid(bio: rq->bio);
1923	}
1924	EXPORT_SYMBOL_GPL(nvme_fc_io_getuuid);
1925
1926	static void
1927	nvme_fc_fcpio_done(struct nvmefc_fcp_req *req)
1928	{
1929	struct nvme_fc_fcp_op *op = fcp_req_to_fcp_op(fcpreq: req);
1930	struct request *rq = op->rq;
1931	struct nvmefc_fcp_req *freq = &op->fcp_req;
1932	struct nvme_fc_ctrl *ctrl = op->ctrl;
1933	struct nvme_fc_queue *queue = op->queue;
1934	struct nvme_completion *cqe = &op->rsp_iu.cqe;
1935	struct nvme_command *sqe = &op->cmd_iu.sqe;
1936	__le16 status = cpu_to_le16(NVME_SC_SUCCESS << 1);
1937	union nvme_result result;
1938	bool terminate_assoc = true;
1939	int opstate;
1940
1941	/*
1942	* WARNING:
1943	* The current linux implementation of a nvme controller
1944	* allocates a single tag set for all io queues and sizes
1945	* the io queues to fully hold all possible tags. Thus, the
1946	* implementation does not reference or care about the sqhd
1947	* value as it never needs to use the sqhd/sqtail pointers
1948	* for submission pacing.
1949	*
1950	* This affects the FC-NVME implementation in two ways:
1951	* 1) As the value doesn't matter, we don't need to waste
1952	* cycles extracting it from ERSPs and stamping it in the
1953	* cases where the transport fabricates CQEs on successful
1954	* completions.
1955	* 2) The FC-NVME implementation requires that delivery of
1956	* ERSP completions are to go back to the nvme layer in order
1957	* relative to the rsn, such that the sqhd value will always
1958	* be "in order" for the nvme layer. As the nvme layer in
1959	* linux doesn't care about sqhd, there's no need to return
1960	* them in order.
1961	*
1962	* Additionally:
1963	* As the core nvme layer in linux currently does not look at
1964	* every field in the cqe - in cases where the FC transport must
1965	* fabricate a CQE, the following fields will not be set as they
1966	* are not referenced:
1967	* cqe.sqid, cqe.sqhd, cqe.command_id
1968	*
1969	* Failure or error of an individual i/o, in a transport
1970	* detected fashion unrelated to the nvme completion status,
1971	* potentially cause the initiator and target sides to get out
1972	* of sync on SQ head/tail (aka outstanding io count allowed).
1973	* Per FC-NVME spec, failure of an individual command requires
1974	* the connection to be terminated, which in turn requires the
1975	* association to be terminated.
1976	*/
1977
1978	opstate = atomic_xchg(v: &op->state, new: FCPOP_STATE_COMPLETE);
1979
1980	fc_dma_sync_single_for_cpu(dev: ctrl->lport->dev, addr: op->fcp_req.rspdma,
1981	size: sizeof(op->rsp_iu), dir: DMA_FROM_DEVICE);
1982
1983	if (opstate == FCPOP_STATE_ABORTED)
1984	status = cpu_to_le16(NVME_SC_HOST_ABORTED_CMD << 1);
1985	else if (freq->status) {
1986	status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1);
1987	dev_info(ctrl->ctrl.device,
1988	"NVME-FC{%d}: io failed due to lldd error %d\n",
1989	ctrl->cnum, freq->status);
1990	}
1991
1992	/*
1993	* For the linux implementation, if we have an unsuccesful
1994	* status, they blk-mq layer can typically be called with the
1995	* non-zero status and the content of the cqe isn't important.
1996	*/
1997	if (status)
1998	goto done;
1999
2000	/*
2001	* command completed successfully relative to the wire
2002	* protocol. However, validate anything received and
2003	* extract the status and result from the cqe (create it
2004	* where necessary).
2005	*/
2006
2007	switch (freq->rcv_rsplen) {
2008
2009	case 0:
2010	case NVME_FC_SIZEOF_ZEROS_RSP:
2011	/*
2012	* No response payload or 12 bytes of payload (which
2013	* should all be zeros) are considered successful and
2014	* no payload in the CQE by the transport.
2015	*/
2016	if (freq->transferred_length !=
2017	be32_to_cpu(op->cmd_iu.data_len)) {
2018	status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1);
2019	dev_info(ctrl->ctrl.device,
2020	"NVME-FC{%d}: io failed due to bad transfer "
2021	"length: %d vs expected %d\n",
2022	ctrl->cnum, freq->transferred_length,
2023	be32_to_cpu(op->cmd_iu.data_len));
2024	goto done;
2025	}
2026	result.u64 = 0;
2027	break;
2028
2029	case sizeof(struct nvme_fc_ersp_iu):
2030	/*
2031	* The ERSP IU contains a full completion with CQE.
2032	* Validate ERSP IU and look at cqe.
2033	*/
2034	if (unlikely(be16_to_cpu(op->rsp_iu.iu_len) !=
2035	(freq->rcv_rsplen / 4) ||
2036	be32_to_cpu(op->rsp_iu.xfrd_len) !=
2037	freq->transferred_length ||
2038	op->rsp_iu.ersp_result ||
2039	sqe->common.command_id != cqe->command_id)) {
2040	status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1);
2041	dev_info(ctrl->ctrl.device,
2042	"NVME-FC{%d}: io failed due to bad NVMe_ERSP: "
2043	"iu len %d, xfr len %d vs %d, status code "
2044	"%d, cmdid %d vs %d\n",
2045	ctrl->cnum, be16_to_cpu(op->rsp_iu.iu_len),
2046	be32_to_cpu(op->rsp_iu.xfrd_len),
2047	freq->transferred_length,
2048	op->rsp_iu.ersp_result,
2049	sqe->common.command_id,
2050	cqe->command_id);
2051	goto done;
2052	}
2053	result = cqe->result;
2054	status = cqe->status;
2055	break;
2056
2057	default:
2058	status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1);
2059	dev_info(ctrl->ctrl.device,
2060	"NVME-FC{%d}: io failed due to odd NVMe_xRSP iu "
2061	"len %d\n",
2062	ctrl->cnum, freq->rcv_rsplen);
2063	goto done;
2064	}
2065
2066	terminate_assoc = false;
2067
2068	done:
2069	if (op->flags & FCOP_FLAGS_AEN) {
2070	nvme_complete_async_event(ctrl: &queue->ctrl->ctrl, status, res: &result);
2071	__nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate);
2072	atomic_set(v: &op->state, i: FCPOP_STATE_IDLE);
2073	op->flags = FCOP_FLAGS_AEN; /* clear other flags */
2074	nvme_fc_ctrl_put(ctrl);
2075	goto check_error;
2076	}
2077
2078	__nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate);
2079	if (!nvme_try_complete_req(req: rq, status, result))
2080	nvme_fc_complete_rq(rq);
2081
2082	check_error:
2083	if (terminate_assoc && ctrl->ctrl.state != NVME_CTRL_RESETTING)
2084	queue_work(wq: nvme_reset_wq, work: &ctrl->ioerr_work);
2085	}
2086
2087	static int
2088	__nvme_fc_init_request(struct nvme_fc_ctrl *ctrl,
2089	struct nvme_fc_queue *queue, struct nvme_fc_fcp_op *op,
2090	struct request *rq, u32 rqno)
2091	{
2092	struct nvme_fcp_op_w_sgl *op_w_sgl =
2093	container_of(op, typeof(*op_w_sgl), op);
2094	struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu;
2095	int ret = 0;
2096
2097	memset(op, 0, sizeof(*op));
2098	op->fcp_req.cmdaddr = &op->cmd_iu;
2099	op->fcp_req.cmdlen = sizeof(op->cmd_iu);
2100	op->fcp_req.rspaddr = &op->rsp_iu;
2101	op->fcp_req.rsplen = sizeof(op->rsp_iu);
2102	op->fcp_req.done = nvme_fc_fcpio_done;
2103	op->ctrl = ctrl;
2104	op->queue = queue;
2105	op->rq = rq;
2106	op->rqno = rqno;
2107
2108	cmdiu->format_id = NVME_CMD_FORMAT_ID;
2109	cmdiu->fc_id = NVME_CMD_FC_ID;
2110	cmdiu->iu_len = cpu_to_be16(sizeof(*cmdiu) / sizeof(u32));
2111	if (queue->qnum)
2112	cmdiu->rsv_cat = fccmnd_set_cat_css(rsv_cat: 0,
2113	css: (NVME_CC_CSS_NVM >> NVME_CC_CSS_SHIFT));
2114	else
2115	cmdiu->rsv_cat = fccmnd_set_cat_admin(rsv_cat: 0);
2116
2117	op->fcp_req.cmddma = fc_dma_map_single(dev: ctrl->lport->dev,
2118	ptr: &op->cmd_iu, size: sizeof(op->cmd_iu), dir: DMA_TO_DEVICE);
2119	if (fc_dma_mapping_error(dev: ctrl->lport->dev, dma_addr: op->fcp_req.cmddma)) {
2120	dev_err(ctrl->dev,
2121	"FCP Op failed - cmdiu dma mapping failed.\n");
2122	ret = -EFAULT;
2123	goto out_on_error;
2124	}
2125
2126	op->fcp_req.rspdma = fc_dma_map_single(dev: ctrl->lport->dev,
2127	ptr: &op->rsp_iu, size: sizeof(op->rsp_iu),
2128	dir: DMA_FROM_DEVICE);
2129	if (fc_dma_mapping_error(dev: ctrl->lport->dev, dma_addr: op->fcp_req.rspdma)) {
2130	dev_err(ctrl->dev,
2131	"FCP Op failed - rspiu dma mapping failed.\n");
2132	ret = -EFAULT;
2133	}
2134
2135	atomic_set(v: &op->state, i: FCPOP_STATE_IDLE);
2136	out_on_error:
2137	return ret;
2138	}
2139
2140	static int
2141	nvme_fc_init_request(struct blk_mq_tag_set *set, struct request *rq,
2142	unsigned int hctx_idx, unsigned int numa_node)
2143	{
2144	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(ctrl: set->driver_data);
2145	struct nvme_fcp_op_w_sgl *op = blk_mq_rq_to_pdu(rq);
2146	int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0;
2147	struct nvme_fc_queue *queue = &ctrl->queues[queue_idx];
2148	int res;
2149
2150	res = __nvme_fc_init_request(ctrl, queue, op: &op->op, rq, rqno: queue->rqcnt++);
2151	if (res)
2152	return res;
2153	op->op.fcp_req.first_sgl = op->sgl;
2154	op->op.fcp_req.private = &op->priv[0];
2155	nvme_req(req: rq)->ctrl = &ctrl->ctrl;
2156	nvme_req(req: rq)->cmd = &op->op.cmd_iu.sqe;
2157	return res;
2158	}
2159
2160	static int
2161	nvme_fc_init_aen_ops(struct nvme_fc_ctrl *ctrl)
2162	{
2163	struct nvme_fc_fcp_op *aen_op;
2164	struct nvme_fc_cmd_iu *cmdiu;
2165	struct nvme_command *sqe;
2166	void *private = NULL;
2167	int i, ret;
2168
2169	aen_op = ctrl->aen_ops;
2170	for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++) {
2171	if (ctrl->lport->ops->fcprqst_priv_sz) {
2172	private = kzalloc(size: ctrl->lport->ops->fcprqst_priv_sz,
2173	GFP_KERNEL);
2174	if (!private)
2175	return -ENOMEM;
2176	}
2177
2178	cmdiu = &aen_op->cmd_iu;
2179	sqe = &cmdiu->sqe;
2180	ret = __nvme_fc_init_request(ctrl, queue: &ctrl->queues[0],
2181	op: aen_op, rq: (struct request *)NULL,
2182	rqno: (NVME_AQ_BLK_MQ_DEPTH + i));
2183	if (ret) {
2184	kfree(objp: private);
2185	return ret;
2186	}
2187
2188	aen_op->flags = FCOP_FLAGS_AEN;
2189	aen_op->fcp_req.private = private;
2190
2191	memset(sqe, 0, sizeof(*sqe));
2192	sqe->common.opcode = nvme_admin_async_event;
2193	/* Note: core layer may overwrite the sqe.command_id value */
2194	sqe->common.command_id = NVME_AQ_BLK_MQ_DEPTH + i;
2195	}
2196	return 0;
2197	}
2198
2199	static void
2200	nvme_fc_term_aen_ops(struct nvme_fc_ctrl *ctrl)
2201	{
2202	struct nvme_fc_fcp_op *aen_op;
2203	int i;
2204
2205	cancel_work_sync(work: &ctrl->ctrl.async_event_work);
2206	aen_op = ctrl->aen_ops;
2207	for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++) {
2208	__nvme_fc_exit_request(ctrl, op: aen_op);
2209
2210	kfree(objp: aen_op->fcp_req.private);
2211	aen_op->fcp_req.private = NULL;
2212	}
2213	}
2214
2215	static inline int
2216	__nvme_fc_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, unsigned int qidx)
2217	{
2218	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(ctrl: data);
2219	struct nvme_fc_queue *queue = &ctrl->queues[qidx];
2220
2221	hctx->driver_data = queue;
2222	queue->hctx = hctx;
2223	return 0;
2224	}
2225
2226	static int
2227	nvme_fc_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, unsigned int hctx_idx)
2228	{
2229	return __nvme_fc_init_hctx(hctx, data, qidx: hctx_idx + 1);
2230	}
2231
2232	static int
2233	nvme_fc_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
2234	unsigned int hctx_idx)
2235	{
2236	return __nvme_fc_init_hctx(hctx, data, qidx: hctx_idx);
2237	}
2238
2239	static void
2240	nvme_fc_init_queue(struct nvme_fc_ctrl *ctrl, int idx)
2241	{
2242	struct nvme_fc_queue *queue;
2243
2244	queue = &ctrl->queues[idx];
2245	memset(queue, 0, sizeof(*queue));
2246	queue->ctrl = ctrl;
2247	queue->qnum = idx;
2248	atomic_set(v: &queue->csn, i: 0);
2249	queue->dev = ctrl->dev;
2250
2251	if (idx > 0)
2252	queue->cmnd_capsule_len = ctrl->ctrl.ioccsz * 16;
2253	else
2254	queue->cmnd_capsule_len = sizeof(struct nvme_command);
2255
2256	/*
2257	* Considered whether we should allocate buffers for all SQEs
2258	* and CQEs and dma map them - mapping their respective entries
2259	* into the request structures (kernel vm addr and dma address)
2260	* thus the driver could use the buffers/mappings directly.
2261	* It only makes sense if the LLDD would use them for its
2262	* messaging api. It's very unlikely most adapter api's would use
2263	* a native NVME sqe/cqe. More reasonable if FC-NVME IU payload
2264	* structures were used instead.
2265	*/
2266	}
2267
2268	/*
2269	* This routine terminates a queue at the transport level.
2270	* The transport has already ensured that all outstanding ios on
2271	* the queue have been terminated.
2272	* The transport will send a Disconnect LS request to terminate
2273	* the queue's connection. Termination of the admin queue will also
2274	* terminate the association at the target.
2275	*/
2276	static void
2277	nvme_fc_free_queue(struct nvme_fc_queue *queue)
2278	{
2279	if (!test_and_clear_bit(nr: NVME_FC_Q_CONNECTED, addr: &queue->flags))
2280	return;
2281
2282	clear_bit(nr: NVME_FC_Q_LIVE, addr: &queue->flags);
2283	/*
2284	* Current implementation never disconnects a single queue.
2285	* It always terminates a whole association. So there is never
2286	* a disconnect(queue) LS sent to the target.
2287	*/
2288
2289	queue->connection_id = 0;
2290	atomic_set(v: &queue->csn, i: 0);
2291	}
2292
2293	static void
2294	__nvme_fc_delete_hw_queue(struct nvme_fc_ctrl *ctrl,
2295	struct nvme_fc_queue *queue, unsigned int qidx)
2296	{
2297	if (ctrl->lport->ops->delete_queue)
2298	ctrl->lport->ops->delete_queue(&ctrl->lport->localport, qidx,
2299	queue->lldd_handle);
2300	queue->lldd_handle = NULL;
2301	}
2302
2303	static void
2304	nvme_fc_free_io_queues(struct nvme_fc_ctrl *ctrl)
2305	{
2306	int i;
2307
2308	for (i = 1; i < ctrl->ctrl.queue_count; i++)
2309	nvme_fc_free_queue(queue: &ctrl->queues[i]);
2310	}
2311
2312	static int
2313	__nvme_fc_create_hw_queue(struct nvme_fc_ctrl *ctrl,
2314	struct nvme_fc_queue *queue, unsigned int qidx, u16 qsize)
2315	{
2316	int ret = 0;
2317
2318	queue->lldd_handle = NULL;
2319	if (ctrl->lport->ops->create_queue)
2320	ret = ctrl->lport->ops->create_queue(&ctrl->lport->localport,
2321	qidx, qsize, &queue->lldd_handle);
2322
2323	return ret;
2324	}
2325
2326	static void
2327	nvme_fc_delete_hw_io_queues(struct nvme_fc_ctrl *ctrl)
2328	{
2329	struct nvme_fc_queue *queue = &ctrl->queues[ctrl->ctrl.queue_count - 1];
2330	int i;
2331
2332	for (i = ctrl->ctrl.queue_count - 1; i >= 1; i--, queue--)
2333	__nvme_fc_delete_hw_queue(ctrl, queue, qidx: i);
2334	}
2335
2336	static int
2337	nvme_fc_create_hw_io_queues(struct nvme_fc_ctrl *ctrl, u16 qsize)
2338	{
2339	struct nvme_fc_queue *queue = &ctrl->queues[1];
2340	int i, ret;
2341
2342	for (i = 1; i < ctrl->ctrl.queue_count; i++, queue++) {
2343	ret = __nvme_fc_create_hw_queue(ctrl, queue, qidx: i, qsize);
2344	if (ret)
2345	goto delete_queues;
2346	}
2347
2348	return 0;
2349
2350	delete_queues:
2351	for (; i > 0; i--)
2352	__nvme_fc_delete_hw_queue(ctrl, queue: &ctrl->queues[i], qidx: i);
2353	return ret;
2354	}
2355
2356	static int
2357	nvme_fc_connect_io_queues(struct nvme_fc_ctrl *ctrl, u16 qsize)
2358	{
2359	int i, ret = 0;
2360
2361	for (i = 1; i < ctrl->ctrl.queue_count; i++) {
2362	ret = nvme_fc_connect_queue(ctrl, queue: &ctrl->queues[i], qsize,
2363	ersp_ratio: (qsize / 5));
2364	if (ret)
2365	break;
2366	ret = nvmf_connect_io_queue(ctrl: &ctrl->ctrl, qid: i);
2367	if (ret)
2368	break;
2369
2370	set_bit(nr: NVME_FC_Q_LIVE, addr: &ctrl->queues[i].flags);
2371	}
2372
2373	return ret;
2374	}
2375
2376	static void
2377	nvme_fc_init_io_queues(struct nvme_fc_ctrl *ctrl)
2378	{
2379	int i;
2380
2381	for (i = 1; i < ctrl->ctrl.queue_count; i++)
2382	nvme_fc_init_queue(ctrl, idx: i);
2383	}
2384
2385	static void
2386	nvme_fc_ctrl_free(struct kref *ref)
2387	{
2388	struct nvme_fc_ctrl *ctrl =
2389	container_of(ref, struct nvme_fc_ctrl, ref);
2390	unsigned long flags;
2391
2392	if (ctrl->ctrl.tagset)
2393	nvme_remove_io_tag_set(ctrl: &ctrl->ctrl);
2394
2395	/* remove from rport list */
2396	spin_lock_irqsave(&ctrl->rport->lock, flags);
2397	list_del(entry: &ctrl->ctrl_list);
2398	spin_unlock_irqrestore(lock: &ctrl->rport->lock, flags);
2399
2400	nvme_unquiesce_admin_queue(ctrl: &ctrl->ctrl);
2401	nvme_remove_admin_tag_set(ctrl: &ctrl->ctrl);
2402
2403	kfree(objp: ctrl->queues);
2404
2405	put_device(dev: ctrl->dev);
2406	nvme_fc_rport_put(rport: ctrl->rport);
2407
2408	ida_free(&nvme_fc_ctrl_cnt, id: ctrl->cnum);
2409	if (ctrl->ctrl.opts)
2410	nvmf_free_options(opts: ctrl->ctrl.opts);
2411	kfree(objp: ctrl);
2412	}
2413
2414	static void
2415	nvme_fc_ctrl_put(struct nvme_fc_ctrl *ctrl)
2416	{
2417	kref_put(kref: &ctrl->ref, release: nvme_fc_ctrl_free);
2418	}
2419
2420	static int
2421	nvme_fc_ctrl_get(struct nvme_fc_ctrl *ctrl)
2422	{
2423	return kref_get_unless_zero(kref: &ctrl->ref);
2424	}
2425
2426	/*
2427	* All accesses from nvme core layer done - can now free the
2428	* controller. Called after last nvme_put_ctrl() call
2429	*/
2430	static void
2431	nvme_fc_free_ctrl(struct nvme_ctrl *nctrl)
2432	{
2433	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(ctrl: nctrl);
2434
2435	WARN_ON(nctrl != &ctrl->ctrl);
2436
2437	nvme_fc_ctrl_put(ctrl);
2438	}
2439
2440	/*
2441	* This routine is used by the transport when it needs to find active
2442	* io on a queue that is to be terminated. The transport uses
2443	* blk_mq_tagset_busy_itr() to find the busy requests, which then invoke
2444	* this routine to kill them on a 1 by 1 basis.
2445	*
2446	* As FC allocates FC exchange for each io, the transport must contact
2447	* the LLDD to terminate the exchange, thus releasing the FC exchange.
2448	* After terminating the exchange the LLDD will call the transport's
2449	* normal io done path for the request, but it will have an aborted
2450	* status. The done path will return the io request back to the block
2451	* layer with an error status.
2452	*/
2453	static bool nvme_fc_terminate_exchange(struct request *req, void *data)
2454	{
2455	struct nvme_ctrl *nctrl = data;
2456	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(ctrl: nctrl);
2457	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq: req);
2458
2459	op->nreq.flags |= NVME_REQ_CANCELLED;
2460	__nvme_fc_abort_op(ctrl, op);
2461	return true;
2462	}
2463
2464	/*
2465	* This routine runs through all outstanding commands on the association
2466	* and aborts them. This routine is typically be called by the
2467	* delete_association routine. It is also called due to an error during
2468	* reconnect. In that scenario, it is most likely a command that initializes
2469	* the controller, including fabric Connect commands on io queues, that
2470	* may have timed out or failed thus the io must be killed for the connect
2471	* thread to see the error.
2472	*/
2473	static void
2474	__nvme_fc_abort_outstanding_ios(struct nvme_fc_ctrl *ctrl, bool start_queues)
2475	{
2476	int q;
2477
2478	/*
2479	* if aborting io, the queues are no longer good, mark them
2480	* all as not live.
2481	*/
2482	if (ctrl->ctrl.queue_count > 1) {
2483	for (q = 1; q < ctrl->ctrl.queue_count; q++)
2484	clear_bit(nr: NVME_FC_Q_LIVE, addr: &ctrl->queues[q].flags);
2485	}
2486	clear_bit(nr: NVME_FC_Q_LIVE, addr: &ctrl->queues[0].flags);
2487
2488	/*
2489	* If io queues are present, stop them and terminate all outstanding
2490	* ios on them. As FC allocates FC exchange for each io, the
2491	* transport must contact the LLDD to terminate the exchange,
2492	* thus releasing the FC exchange. We use blk_mq_tagset_busy_itr()
2493	* to tell us what io's are busy and invoke a transport routine
2494	* to kill them with the LLDD. After terminating the exchange
2495	* the LLDD will call the transport's normal io done path, but it
2496	* will have an aborted status. The done path will return the
2497	* io requests back to the block layer as part of normal completions
2498	* (but with error status).
2499	*/
2500	if (ctrl->ctrl.queue_count > 1) {
2501	nvme_quiesce_io_queues(ctrl: &ctrl->ctrl);
2502	nvme_sync_io_queues(ctrl: &ctrl->ctrl);
2503	blk_mq_tagset_busy_iter(tagset: &ctrl->tag_set,
2504	fn: nvme_fc_terminate_exchange, priv: &ctrl->ctrl);
2505	blk_mq_tagset_wait_completed_request(tagset: &ctrl->tag_set);
2506	if (start_queues)
2507	nvme_unquiesce_io_queues(ctrl: &ctrl->ctrl);
2508	}
2509
2510	/*
2511	* Other transports, which don't have link-level contexts bound
2512	* to sqe's, would try to gracefully shutdown the controller by
2513	* writing the registers for shutdown and polling (call
2514	* nvme_disable_ctrl()). Given a bunch of i/o was potentially
2515	* just aborted and we will wait on those contexts, and given
2516	* there was no indication of how live the controlelr is on the
2517	* link, don't send more io to create more contexts for the
2518	* shutdown. Let the controller fail via keepalive failure if
2519	* its still present.
2520	*/
2521
2522	/*
2523	* clean up the admin queue. Same thing as above.
2524	*/
2525	nvme_quiesce_admin_queue(ctrl: &ctrl->ctrl);
2526	blk_sync_queue(q: ctrl->ctrl.admin_q);
2527	blk_mq_tagset_busy_iter(tagset: &ctrl->admin_tag_set,
2528	fn: nvme_fc_terminate_exchange, priv: &ctrl->ctrl);
2529	blk_mq_tagset_wait_completed_request(tagset: &ctrl->admin_tag_set);
2530	if (start_queues)
2531	nvme_unquiesce_admin_queue(ctrl: &ctrl->ctrl);
2532	}
2533
2534	static void
2535	nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl, char *errmsg)
2536	{
2537	/*
2538	* if an error (io timeout, etc) while (re)connecting, the remote
2539	* port requested terminating of the association (disconnect_ls)
2540	* or an error (timeout or abort) occurred on an io while creating
2541	* the controller. Abort any ios on the association and let the
2542	* create_association error path resolve things.
2543	*/
2544	if (ctrl->ctrl.state == NVME_CTRL_CONNECTING) {
2545	__nvme_fc_abort_outstanding_ios(ctrl, start_queues: true);
2546	set_bit(ASSOC_FAILED, addr: &ctrl->flags);
2547	dev_warn(ctrl->ctrl.device,
2548	"NVME-FC{%d}: transport error during (re)connect\n",
2549	ctrl->cnum);
2550	return;
2551	}
2552
2553	/* Otherwise, only proceed if in LIVE state - e.g. on first error */
2554	if (ctrl->ctrl.state != NVME_CTRL_LIVE)
2555	return;
2556
2557	dev_warn(ctrl->ctrl.device,
2558	"NVME-FC{%d}: transport association event: %s\n",
2559	ctrl->cnum, errmsg);
2560	dev_warn(ctrl->ctrl.device,
2561	"NVME-FC{%d}: resetting controller\n", ctrl->cnum);
2562
2563	nvme_reset_ctrl(ctrl: &ctrl->ctrl);
2564	}
2565
2566	static enum blk_eh_timer_return nvme_fc_timeout(struct request *rq)
2567	{
2568	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
2569	struct nvme_fc_ctrl *ctrl = op->ctrl;
2570	u16 qnum = op->queue->qnum;
2571	struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu;
2572	struct nvme_command *sqe = &cmdiu->sqe;
2573
2574	/*
2575	* Attempt to abort the offending command. Command completion
2576	* will detect the aborted io and will fail the connection.
2577	*/
2578	dev_info(ctrl->ctrl.device,
2579	"NVME-FC{%d.%d}: io timeout: opcode %d fctype %d (%s) w10/11: "
2580	"x%08x/x%08x\n",
2581	ctrl->cnum, qnum, sqe->common.opcode, sqe->fabrics.fctype,
2582	nvme_fabrics_opcode_str(qnum, sqe),
2583	sqe->common.cdw10, sqe->common.cdw11);
2584	if (__nvme_fc_abort_op(ctrl, op))
2585	nvme_fc_error_recovery(ctrl, errmsg: "io timeout abort failed");
2586
2587	/*
2588	* the io abort has been initiated. Have the reset timer
2589	* restarted and the abort completion will complete the io
2590	* shortly. Avoids a synchronous wait while the abort finishes.
2591	*/
2592	return BLK_EH_RESET_TIMER;
2593	}
2594
2595	static int
2596	nvme_fc_map_data(struct nvme_fc_ctrl *ctrl, struct request *rq,
2597	struct nvme_fc_fcp_op *op)
2598	{
2599	struct nvmefc_fcp_req *freq = &op->fcp_req;
2600	int ret;
2601
2602	freq->sg_cnt = 0;
2603
2604	if (!blk_rq_nr_phys_segments(rq))
2605	return 0;
2606
2607	freq->sg_table.sgl = freq->first_sgl;
2608	ret = sg_alloc_table_chained(table: &freq->sg_table,
2609	nents: blk_rq_nr_phys_segments(rq), first_chunk: freq->sg_table.sgl,
2610	NVME_INLINE_SG_CNT);
2611	if (ret)
2612	return -ENOMEM;
2613
2614	op->nents = blk_rq_map_sg(q: rq->q, rq, sglist: freq->sg_table.sgl);
2615	WARN_ON(op->nents > blk_rq_nr_phys_segments(rq));
2616	freq->sg_cnt = fc_dma_map_sg(dev: ctrl->lport->dev, sg: freq->sg_table.sgl,
2617	nents: op->nents, rq_dma_dir(rq));
2618	if (unlikely(freq->sg_cnt <= 0)) {
2619	sg_free_table_chained(table: &freq->sg_table, NVME_INLINE_SG_CNT);
2620	freq->sg_cnt = 0;
2621	return -EFAULT;
2622	}
2623
2624	/*
2625	* TODO: blk_integrity_rq(rq) for DIF
2626	*/
2627	return 0;
2628	}
2629
2630	static void
2631	nvme_fc_unmap_data(struct nvme_fc_ctrl *ctrl, struct request *rq,
2632	struct nvme_fc_fcp_op *op)
2633	{
2634	struct nvmefc_fcp_req *freq = &op->fcp_req;
2635
2636	if (!freq->sg_cnt)
2637	return;
2638
2639	fc_dma_unmap_sg(dev: ctrl->lport->dev, sg: freq->sg_table.sgl, nents: op->nents,
2640	rq_dma_dir(rq));
2641
2642	sg_free_table_chained(table: &freq->sg_table, NVME_INLINE_SG_CNT);
2643
2644	freq->sg_cnt = 0;
2645	}
2646
2647	/*
2648	* In FC, the queue is a logical thing. At transport connect, the target
2649	* creates its "queue" and returns a handle that is to be given to the
2650	* target whenever it posts something to the corresponding SQ. When an
2651	* SQE is sent on a SQ, FC effectively considers the SQE, or rather the
2652	* command contained within the SQE, an io, and assigns a FC exchange
2653	* to it. The SQE and the associated SQ handle are sent in the initial
2654	* CMD IU sents on the exchange. All transfers relative to the io occur
2655	* as part of the exchange. The CQE is the last thing for the io,
2656	* which is transferred (explicitly or implicitly) with the RSP IU
2657	* sent on the exchange. After the CQE is received, the FC exchange is
2658	* terminaed and the Exchange may be used on a different io.
2659	*
2660	* The transport to LLDD api has the transport making a request for a
2661	* new fcp io request to the LLDD. The LLDD then allocates a FC exchange
2662	* resource and transfers the command. The LLDD will then process all
2663	* steps to complete the io. Upon completion, the transport done routine
2664	* is called.
2665	*
2666	* So - while the operation is outstanding to the LLDD, there is a link
2667	* level FC exchange resource that is also outstanding. This must be
2668	* considered in all cleanup operations.
2669	*/
2670	static blk_status_t
2671	nvme_fc_start_fcp_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue,
2672	struct nvme_fc_fcp_op *op, u32 data_len,
2673	enum nvmefc_fcp_datadir io_dir)
2674	{
2675	struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu;
2676	struct nvme_command *sqe = &cmdiu->sqe;
2677	int ret, opstate;
2678
2679	/*
2680	* before attempting to send the io, check to see if we believe
2681	* the target device is present
2682	*/
2683	if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
2684	return BLK_STS_RESOURCE;
2685
2686	if (!nvme_fc_ctrl_get(ctrl))
2687	return BLK_STS_IOERR;
2688
2689	/* format the FC-NVME CMD IU and fcp_req */
2690	cmdiu->connection_id = cpu_to_be64(queue->connection_id);
2691	cmdiu->data_len = cpu_to_be32(data_len);
2692	switch (io_dir) {
2693	case NVMEFC_FCP_WRITE:
2694	cmdiu->flags = FCNVME_CMD_FLAGS_WRITE;
2695	break;
2696	case NVMEFC_FCP_READ:
2697	cmdiu->flags = FCNVME_CMD_FLAGS_READ;
2698	break;
2699	case NVMEFC_FCP_NODATA:
2700	cmdiu->flags = 0;
2701	break;
2702	}
2703	op->fcp_req.payload_length = data_len;
2704	op->fcp_req.io_dir = io_dir;
2705	op->fcp_req.transferred_length = 0;
2706	op->fcp_req.rcv_rsplen = 0;
2707	op->fcp_req.status = NVME_SC_SUCCESS;
2708	op->fcp_req.sqid = cpu_to_le16(queue->qnum);
2709
2710	/*
2711	* validate per fabric rules, set fields mandated by fabric spec
2712	* as well as those by FC-NVME spec.
2713	*/
2714	WARN_ON_ONCE(sqe->common.metadata);
2715	sqe->common.flags |= NVME_CMD_SGL_METABUF;
2716
2717	/*
2718	* format SQE DPTR field per FC-NVME rules:
2719	* type=0x5 Transport SGL Data Block Descriptor
2720	* subtype=0xA Transport-specific value
2721	* address=0
2722	* length=length of the data series
2723	*/
2724	sqe->rw.dptr.sgl.type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) |
2725	NVME_SGL_FMT_TRANSPORT_A;
2726	sqe->rw.dptr.sgl.length = cpu_to_le32(data_len);
2727	sqe->rw.dptr.sgl.addr = 0;
2728
2729	if (!(op->flags & FCOP_FLAGS_AEN)) {
2730	ret = nvme_fc_map_data(ctrl, rq: op->rq, op);
2731	if (ret < 0) {
2732	nvme_cleanup_cmd(req: op->rq);
2733	nvme_fc_ctrl_put(ctrl);
2734	if (ret == -ENOMEM || ret == -EAGAIN)
2735	return BLK_STS_RESOURCE;
2736	return BLK_STS_IOERR;
2737	}
2738	}
2739
2740	fc_dma_sync_single_for_device(dev: ctrl->lport->dev, addr: op->fcp_req.cmddma,
2741	size: sizeof(op->cmd_iu), dir: DMA_TO_DEVICE);
2742
2743	atomic_set(v: &op->state, i: FCPOP_STATE_ACTIVE);
2744
2745	if (!(op->flags & FCOP_FLAGS_AEN))
2746	nvme_start_request(rq: op->rq);
2747
2748	cmdiu->csn = cpu_to_be32(atomic_inc_return(&queue->csn));
2749	ret = ctrl->lport->ops->fcp_io(&ctrl->lport->localport,
2750	&ctrl->rport->remoteport,
2751	queue->lldd_handle, &op->fcp_req);
2752
2753	if (ret) {
2754	/*
2755	* If the lld fails to send the command is there an issue with
2756	* the csn value? If the command that fails is the Connect,
2757	* no - as the connection won't be live. If it is a command
2758	* post-connect, it's possible a gap in csn may be created.
2759	* Does this matter? As Linux initiators don't send fused
2760	* commands, no. The gap would exist, but as there's nothing
2761	* that depends on csn order to be delivered on the target
2762	* side, it shouldn't hurt. It would be difficult for a
2763	* target to even detect the csn gap as it has no idea when the
2764	* cmd with the csn was supposed to arrive.
2765	*/
2766	opstate = atomic_xchg(v: &op->state, new: FCPOP_STATE_COMPLETE);
2767	__nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate);
2768
2769	if (!(op->flags & FCOP_FLAGS_AEN)) {
2770	nvme_fc_unmap_data(ctrl, rq: op->rq, op);
2771	nvme_cleanup_cmd(req: op->rq);
2772	}
2773
2774	nvme_fc_ctrl_put(ctrl);
2775
2776	if (ctrl->rport->remoteport.port_state == FC_OBJSTATE_ONLINE &&
2777	ret != -EBUSY)
2778	return BLK_STS_IOERR;
2779
2780	return BLK_STS_RESOURCE;
2781	}
2782
2783	return BLK_STS_OK;
2784	}
2785
2786	static blk_status_t
2787	nvme_fc_queue_rq(struct blk_mq_hw_ctx *hctx,
2788	const struct blk_mq_queue_data *bd)
2789	{
2790	struct nvme_ns *ns = hctx->queue->queuedata;
2791	struct nvme_fc_queue *queue = hctx->driver_data;
2792	struct nvme_fc_ctrl *ctrl = queue->ctrl;
2793	struct request *rq = bd->rq;
2794	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
2795	enum nvmefc_fcp_datadir io_dir;
2796	bool queue_ready = test_bit(NVME_FC_Q_LIVE, &queue->flags);
2797	u32 data_len;
2798	blk_status_t ret;
2799
2800	if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE ||
2801	!nvme_check_ready(ctrl: &queue->ctrl->ctrl, rq, queue_live: queue_ready))
2802	return nvme_fail_nonready_command(ctrl: &queue->ctrl->ctrl, req: rq);
2803
2804	ret = nvme_setup_cmd(ns, req: rq);
2805	if (ret)
2806	return ret;
2807
2808	/*
2809	* nvme core doesn't quite treat the rq opaquely. Commands such
2810	* as WRITE ZEROES will return a non-zero rq payload_bytes yet
2811	* there is no actual payload to be transferred.
2812	* To get it right, key data transmission on there being 1 or
2813	* more physical segments in the sg list. If there is no
2814	* physical segments, there is no payload.
2815	*/
2816	if (blk_rq_nr_phys_segments(rq)) {
2817	data_len = blk_rq_payload_bytes(rq);
2818	io_dir = ((rq_data_dir(rq) == WRITE) ?
2819	NVMEFC_FCP_WRITE : NVMEFC_FCP_READ);
2820	} else {
2821	data_len = 0;
2822	io_dir = NVMEFC_FCP_NODATA;
2823	}
2824
2825
2826	return nvme_fc_start_fcp_op(ctrl, queue, op, data_len, io_dir);
2827	}
2828
2829	static void
2830	nvme_fc_submit_async_event(struct nvme_ctrl *arg)
2831	{
2832	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(ctrl: arg);
2833	struct nvme_fc_fcp_op *aen_op;
2834	blk_status_t ret;
2835
2836	if (test_bit(FCCTRL_TERMIO, &ctrl->flags))
2837	return;
2838
2839	aen_op = &ctrl->aen_ops[0];
2840
2841	ret = nvme_fc_start_fcp_op(ctrl, queue: aen_op->queue, op: aen_op, data_len: 0,
2842	io_dir: NVMEFC_FCP_NODATA);
2843	if (ret)
2844	dev_err(ctrl->ctrl.device,
2845	"failed async event work\n");
2846	}
2847
2848	static void
2849	nvme_fc_complete_rq(struct request *rq)
2850	{
2851	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
2852	struct nvme_fc_ctrl *ctrl = op->ctrl;
2853
2854	atomic_set(v: &op->state, i: FCPOP_STATE_IDLE);
2855	op->flags &= ~FCOP_FLAGS_TERMIO;
2856
2857	nvme_fc_unmap_data(ctrl, rq, op);
2858	nvme_complete_rq(req: rq);
2859	nvme_fc_ctrl_put(ctrl);
2860	}
2861
2862	static void nvme_fc_map_queues(struct blk_mq_tag_set *set)
2863	{
2864	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(ctrl: set->driver_data);
2865	int i;
2866
2867	for (i = 0; i < set->nr_maps; i++) {
2868	struct blk_mq_queue_map *map = &set->map[i];
2869
2870	if (!map->nr_queues) {
2871	WARN_ON(i == HCTX_TYPE_DEFAULT);
2872	continue;
2873	}
2874
2875	/* Call LLDD map queue functionality if defined */
2876	if (ctrl->lport->ops->map_queues)
2877	ctrl->lport->ops->map_queues(&ctrl->lport->localport,
2878	map);
2879	else
2880	blk_mq_map_queues(qmap: map);
2881	}
2882	}
2883
2884	static const struct blk_mq_ops nvme_fc_mq_ops = {
2885	.queue_rq = nvme_fc_queue_rq,
2886	.complete = nvme_fc_complete_rq,
2887	.init_request = nvme_fc_init_request,
2888	.exit_request = nvme_fc_exit_request,
2889	.init_hctx = nvme_fc_init_hctx,
2890	.timeout = nvme_fc_timeout,
2891	.map_queues = nvme_fc_map_queues,
2892	};
2893
2894	static int
2895	nvme_fc_create_io_queues(struct nvme_fc_ctrl *ctrl)
2896	{
2897	struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
2898	unsigned int nr_io_queues;
2899	int ret;
2900
2901	nr_io_queues = min(min(opts->nr_io_queues, num_online_cpus()),
2902	ctrl->lport->ops->max_hw_queues);
2903	ret = nvme_set_queue_count(ctrl: &ctrl->ctrl, count: &nr_io_queues);
2904	if (ret) {
2905	dev_info(ctrl->ctrl.device,
2906	"set_queue_count failed: %d\n", ret);
2907	return ret;
2908	}
2909
2910	ctrl->ctrl.queue_count = nr_io_queues + 1;
2911	if (!nr_io_queues)
2912	return 0;
2913
2914	nvme_fc_init_io_queues(ctrl);
2915
2916	ret = nvme_alloc_io_tag_set(ctrl: &ctrl->ctrl, set: &ctrl->tag_set,
2917	ops: &nvme_fc_mq_ops, nr_maps: 1,
2918	struct_size_t(struct nvme_fcp_op_w_sgl, priv,
2919	ctrl->lport->ops->fcprqst_priv_sz));
2920	if (ret)
2921	return ret;
2922
2923	ret = nvme_fc_create_hw_io_queues(ctrl, qsize: ctrl->ctrl.sqsize + 1);
2924	if (ret)
2925	goto out_cleanup_tagset;
2926
2927	ret = nvme_fc_connect_io_queues(ctrl, qsize: ctrl->ctrl.sqsize + 1);
2928	if (ret)
2929	goto out_delete_hw_queues;
2930
2931	ctrl->ioq_live = true;
2932
2933	return 0;
2934
2935	out_delete_hw_queues:
2936	nvme_fc_delete_hw_io_queues(ctrl);
2937	out_cleanup_tagset:
2938	nvme_remove_io_tag_set(ctrl: &ctrl->ctrl);
2939	nvme_fc_free_io_queues(ctrl);
2940
2941	/* force put free routine to ignore io queues */
2942	ctrl->ctrl.tagset = NULL;
2943
2944	return ret;
2945	}
2946
2947	static int
2948	nvme_fc_recreate_io_queues(struct nvme_fc_ctrl *ctrl)
2949	{
2950	struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
2951	u32 prior_ioq_cnt = ctrl->ctrl.queue_count - 1;
2952	unsigned int nr_io_queues;
2953	int ret;
2954
2955	nr_io_queues = min(min(opts->nr_io_queues, num_online_cpus()),
2956	ctrl->lport->ops->max_hw_queues);
2957	ret = nvme_set_queue_count(ctrl: &ctrl->ctrl, count: &nr_io_queues);
2958	if (ret) {
2959	dev_info(ctrl->ctrl.device,
2960	"set_queue_count failed: %d\n", ret);
2961	return ret;
2962	}
2963
2964	if (!nr_io_queues && prior_ioq_cnt) {
2965	dev_info(ctrl->ctrl.device,
2966	"Fail Reconnect: At least 1 io queue "
2967	"required (was %d)\n", prior_ioq_cnt);
2968	return -ENOSPC;
2969	}
2970
2971	ctrl->ctrl.queue_count = nr_io_queues + 1;
2972	/* check for io queues existing */
2973	if (ctrl->ctrl.queue_count == 1)
2974	return 0;
2975
2976	if (prior_ioq_cnt != nr_io_queues) {
2977	dev_info(ctrl->ctrl.device,
2978	"reconnect: revising io queue count from %d to %d\n",
2979	prior_ioq_cnt, nr_io_queues);
2980	blk_mq_update_nr_hw_queues(set: &ctrl->tag_set, nr_hw_queues: nr_io_queues);
2981	}
2982
2983	ret = nvme_fc_create_hw_io_queues(ctrl, qsize: ctrl->ctrl.sqsize + 1);
2984	if (ret)
2985	goto out_free_io_queues;
2986
2987	ret = nvme_fc_connect_io_queues(ctrl, qsize: ctrl->ctrl.sqsize + 1);
2988	if (ret)
2989	goto out_delete_hw_queues;
2990
2991	return 0;
2992
2993	out_delete_hw_queues:
2994	nvme_fc_delete_hw_io_queues(ctrl);
2995	out_free_io_queues:
2996	nvme_fc_free_io_queues(ctrl);
2997	return ret;
2998	}
2999
3000	static void
3001	nvme_fc_rport_active_on_lport(struct nvme_fc_rport *rport)
3002	{
3003	struct nvme_fc_lport *lport = rport->lport;
3004
3005	atomic_inc(v: &lport->act_rport_cnt);
3006	}
3007
3008	static void
3009	nvme_fc_rport_inactive_on_lport(struct nvme_fc_rport *rport)
3010	{
3011	struct nvme_fc_lport *lport = rport->lport;
3012	u32 cnt;
3013
3014	cnt = atomic_dec_return(v: &lport->act_rport_cnt);
3015	if (cnt == 0 && lport->localport.port_state == FC_OBJSTATE_DELETED)
3016	lport->ops->localport_delete(&lport->localport);
3017	}
3018
3019	static int
3020	nvme_fc_ctlr_active_on_rport(struct nvme_fc_ctrl *ctrl)
3021	{
3022	struct nvme_fc_rport *rport = ctrl->rport;
3023	u32 cnt;
3024
3025	if (test_and_set_bit(ASSOC_ACTIVE, addr: &ctrl->flags))
3026	return 1;
3027
3028	cnt = atomic_inc_return(v: &rport->act_ctrl_cnt);
3029	if (cnt == 1)
3030	nvme_fc_rport_active_on_lport(rport);
3031
3032	return 0;
3033	}
3034
3035	static int
3036	nvme_fc_ctlr_inactive_on_rport(struct nvme_fc_ctrl *ctrl)
3037	{
3038	struct nvme_fc_rport *rport = ctrl->rport;
3039	struct nvme_fc_lport *lport = rport->lport;
3040	u32 cnt;
3041
3042	/* clearing of ctrl->flags ASSOC_ACTIVE bit is in association delete */
3043
3044	cnt = atomic_dec_return(v: &rport->act_ctrl_cnt);
3045	if (cnt == 0) {
3046	if (rport->remoteport.port_state == FC_OBJSTATE_DELETED)
3047	lport->ops->remoteport_delete(&rport->remoteport);
3048	nvme_fc_rport_inactive_on_lport(rport);
3049	}
3050
3051	return 0;
3052	}
3053
3054	/*
3055	* This routine restarts the controller on the host side, and
3056	* on the link side, recreates the controller association.
3057	*/
3058	static int
3059	nvme_fc_create_association(struct nvme_fc_ctrl *ctrl)
3060	{
3061	struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
3062	struct nvmefc_ls_rcv_op *disls = NULL;
3063	unsigned long flags;
3064	int ret;
3065	bool changed;
3066
3067	++ctrl->ctrl.nr_reconnects;
3068
3069	if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
3070	return -ENODEV;
3071
3072	if (nvme_fc_ctlr_active_on_rport(ctrl))
3073	return -ENOTUNIQ;
3074
3075	dev_info(ctrl->ctrl.device,
3076	"NVME-FC{%d}: create association : host wwpn 0x%016llx "
3077	" rport wwpn 0x%016llx: NQN \"%s\"\n",
3078	ctrl->cnum, ctrl->lport->localport.port_name,
3079	ctrl->rport->remoteport.port_name, ctrl->ctrl.opts->subsysnqn);
3080
3081	clear_bit(ASSOC_FAILED, addr: &ctrl->flags);
3082
3083	/*
3084	* Create the admin queue
3085	*/
3086
3087	ret = __nvme_fc_create_hw_queue(ctrl, queue: &ctrl->queues[0], qidx: 0,
3088	NVME_AQ_DEPTH);
3089	if (ret)
3090	goto out_free_queue;
3091
3092	ret = nvme_fc_connect_admin_queue(ctrl, queue: &ctrl->queues[0],
3093	NVME_AQ_DEPTH, ersp_ratio: (NVME_AQ_DEPTH / 4));
3094	if (ret)
3095	goto out_delete_hw_queue;
3096
3097	ret = nvmf_connect_admin_queue(ctrl: &ctrl->ctrl);
3098	if (ret)
3099	goto out_disconnect_admin_queue;
3100
3101	set_bit(nr: NVME_FC_Q_LIVE, addr: &ctrl->queues[0].flags);
3102
3103	/*
3104	* Check controller capabilities
3105	*
3106	* todo:- add code to check if ctrl attributes changed from
3107	* prior connection values
3108	*/
3109
3110	ret = nvme_enable_ctrl(ctrl: &ctrl->ctrl);
3111	if (!ret && test_bit(ASSOC_FAILED, &ctrl->flags))
3112	ret = -EIO;
3113	if (ret)
3114	goto out_disconnect_admin_queue;
3115
3116	ctrl->ctrl.max_segments = ctrl->lport->ops->max_sgl_segments;
3117	ctrl->ctrl.max_hw_sectors = ctrl->ctrl.max_segments <<
3118	(ilog2(SZ_4K) - 9);
3119
3120	nvme_unquiesce_admin_queue(ctrl: &ctrl->ctrl);
3121
3122	ret = nvme_init_ctrl_finish(ctrl: &ctrl->ctrl, was_suspended: false);
3123	if (ret)
3124	goto out_disconnect_admin_queue;
3125	if (test_bit(ASSOC_FAILED, &ctrl->flags)) {
3126	ret = -EIO;
3127	goto out_stop_keep_alive;
3128	}
3129	/* sanity checks */
3130
3131	/* FC-NVME does not have other data in the capsule */
3132	if (ctrl->ctrl.icdoff) {
3133	dev_err(ctrl->ctrl.device, "icdoff %d is not supported!\n",
3134	ctrl->ctrl.icdoff);
3135	ret = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
3136	goto out_stop_keep_alive;
3137	}
3138
3139	/* FC-NVME supports normal SGL Data Block Descriptors */
3140	if (!nvme_ctrl_sgl_supported(ctrl: &ctrl->ctrl)) {
3141	dev_err(ctrl->ctrl.device,
3142	"Mandatory sgls are not supported!\n");
3143	ret = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
3144	goto out_stop_keep_alive;
3145	}
3146
3147	if (opts->queue_size > ctrl->ctrl.maxcmd) {
3148	/* warn if maxcmd is lower than queue_size */
3149	dev_warn(ctrl->ctrl.device,
3150	"queue_size %zu > ctrl maxcmd %u, reducing "
3151	"to maxcmd\n",
3152	opts->queue_size, ctrl->ctrl.maxcmd);
3153	opts->queue_size = ctrl->ctrl.maxcmd;
3154	ctrl->ctrl.sqsize = opts->queue_size - 1;
3155	}
3156
3157	ret = nvme_fc_init_aen_ops(ctrl);
3158	if (ret)
3159	goto out_term_aen_ops;
3160
3161	/*
3162	* Create the io queues
3163	*/
3164
3165	if (ctrl->ctrl.queue_count > 1) {
3166	if (!ctrl->ioq_live)
3167	ret = nvme_fc_create_io_queues(ctrl);
3168	else
3169	ret = nvme_fc_recreate_io_queues(ctrl);
3170	}
3171	if (!ret && test_bit(ASSOC_FAILED, &ctrl->flags))
3172	ret = -EIO;
3173	if (ret)
3174	goto out_term_aen_ops;
3175
3176	changed = nvme_change_ctrl_state(ctrl: &ctrl->ctrl, new_state: NVME_CTRL_LIVE);
3177
3178	ctrl->ctrl.nr_reconnects = 0;
3179
3180	if (changed)
3181	nvme_start_ctrl(ctrl: &ctrl->ctrl);
3182
3183	return 0; /* Success */
3184
3185	out_term_aen_ops:
3186	nvme_fc_term_aen_ops(ctrl);
3187	out_stop_keep_alive:
3188	nvme_stop_keep_alive(ctrl: &ctrl->ctrl);
3189	out_disconnect_admin_queue:
3190	dev_warn(ctrl->ctrl.device,
3191	"NVME-FC{%d}: create_assoc failed, assoc_id %llx ret %d\n",
3192	ctrl->cnum, ctrl->association_id, ret);
3193	/* send a Disconnect(association) LS to fc-nvme target */
3194	nvme_fc_xmt_disconnect_assoc(ctrl);
3195	spin_lock_irqsave(&ctrl->lock, flags);
3196	ctrl->association_id = 0;
3197	disls = ctrl->rcv_disconn;
3198	ctrl->rcv_disconn = NULL;
3199	spin_unlock_irqrestore(lock: &ctrl->lock, flags);
3200	if (disls)
3201	nvme_fc_xmt_ls_rsp(lsop: disls);
3202	out_delete_hw_queue:
3203	__nvme_fc_delete_hw_queue(ctrl, queue: &ctrl->queues[0], qidx: 0);
3204	out_free_queue:
3205	nvme_fc_free_queue(queue: &ctrl->queues[0]);
3206	clear_bit(ASSOC_ACTIVE, addr: &ctrl->flags);
3207	nvme_fc_ctlr_inactive_on_rport(ctrl);
3208
3209	return ret;
3210	}
3211
3212
3213	/*
3214	* This routine stops operation of the controller on the host side.
3215	* On the host os stack side: Admin and IO queues are stopped,
3216	* outstanding ios on them terminated via FC ABTS.
3217	* On the link side: the association is terminated.
3218	*/
3219	static void
3220	nvme_fc_delete_association(struct nvme_fc_ctrl *ctrl)
3221	{
3222	struct nvmefc_ls_rcv_op *disls = NULL;
3223	unsigned long flags;
3224
3225	if (!test_and_clear_bit(ASSOC_ACTIVE, addr: &ctrl->flags))
3226	return;
3227
3228	spin_lock_irqsave(&ctrl->lock, flags);
3229	set_bit(FCCTRL_TERMIO, addr: &ctrl->flags);
3230	ctrl->iocnt = 0;
3231	spin_unlock_irqrestore(lock: &ctrl->lock, flags);
3232
3233	__nvme_fc_abort_outstanding_ios(ctrl, start_queues: false);
3234
3235	/* kill the aens as they are a separate path */
3236	nvme_fc_abort_aen_ops(ctrl);
3237
3238	/* wait for all io that had to be aborted */
3239	spin_lock_irq(lock: &ctrl->lock);
3240	wait_event_lock_irq(ctrl->ioabort_wait, ctrl->iocnt == 0, ctrl->lock);
3241	clear_bit(FCCTRL_TERMIO, addr: &ctrl->flags);
3242	spin_unlock_irq(lock: &ctrl->lock);
3243
3244	nvme_fc_term_aen_ops(ctrl);
3245
3246	/*
3247	* send a Disconnect(association) LS to fc-nvme target
3248	* Note: could have been sent at top of process, but
3249	* cleaner on link traffic if after the aborts complete.
3250	* Note: if association doesn't exist, association_id will be 0
3251	*/
3252	if (ctrl->association_id)
3253	nvme_fc_xmt_disconnect_assoc(ctrl);
3254
3255	spin_lock_irqsave(&ctrl->lock, flags);
3256	ctrl->association_id = 0;
3257	disls = ctrl->rcv_disconn;
3258	ctrl->rcv_disconn = NULL;
3259	spin_unlock_irqrestore(lock: &ctrl->lock, flags);
3260	if (disls)
3261	/*
3262	* if a Disconnect Request was waiting for a response, send
3263	* now that all ABTS's have been issued (and are complete).
3264	*/
3265	nvme_fc_xmt_ls_rsp(lsop: disls);
3266
3267	if (ctrl->ctrl.tagset) {
3268	nvme_fc_delete_hw_io_queues(ctrl);
3269	nvme_fc_free_io_queues(ctrl);
3270	}
3271
3272	__nvme_fc_delete_hw_queue(ctrl, queue: &ctrl->queues[0], qidx: 0);
3273	nvme_fc_free_queue(queue: &ctrl->queues[0]);
3274
3275	/* re-enable the admin_q so anything new can fast fail */
3276	nvme_unquiesce_admin_queue(ctrl: &ctrl->ctrl);
3277
3278	/* resume the io queues so that things will fast fail */
3279	nvme_unquiesce_io_queues(ctrl: &ctrl->ctrl);
3280
3281	nvme_fc_ctlr_inactive_on_rport(ctrl);
3282	}
3283
3284	static void
3285	nvme_fc_delete_ctrl(struct nvme_ctrl *nctrl)
3286	{
3287	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(ctrl: nctrl);
3288
3289	cancel_work_sync(work: &ctrl->ioerr_work);
3290	cancel_delayed_work_sync(dwork: &ctrl->connect_work);
3291	/*
3292	* kill the association on the link side. this will block
3293	* waiting for io to terminate
3294	*/
3295	nvme_fc_delete_association(ctrl);
3296	}
3297
3298	static void
3299	nvme_fc_reconnect_or_delete(struct nvme_fc_ctrl *ctrl, int status)
3300	{
3301	struct nvme_fc_rport *rport = ctrl->rport;
3302	struct nvme_fc_remote_port *portptr = &rport->remoteport;
3303	unsigned long recon_delay = ctrl->ctrl.opts->reconnect_delay * HZ;
3304	bool recon = true;
3305
3306	if (nvme_ctrl_state(ctrl: &ctrl->ctrl) != NVME_CTRL_CONNECTING)
3307	return;
3308
3309	if (portptr->port_state == FC_OBJSTATE_ONLINE) {
3310	dev_info(ctrl->ctrl.device,
3311	"NVME-FC{%d}: reset: Reconnect attempt failed (%d)\n",
3312	ctrl->cnum, status);
3313	if (status > 0 && (status & NVME_SC_DNR))
3314	recon = false;
3315	} else if (time_after_eq(jiffies, rport->dev_loss_end))
3316	recon = false;
3317
3318	if (recon && nvmf_should_reconnect(ctrl: &ctrl->ctrl)) {
3319	if (portptr->port_state == FC_OBJSTATE_ONLINE)
3320	dev_info(ctrl->ctrl.device,
3321	"NVME-FC{%d}: Reconnect attempt in %ld "
3322	"seconds\n",
3323	ctrl->cnum, recon_delay / HZ);
3324	else if (time_after(jiffies + recon_delay, rport->dev_loss_end))
3325	recon_delay = rport->dev_loss_end - jiffies;
3326
3327	queue_delayed_work(wq: nvme_wq, dwork: &ctrl->connect_work, delay: recon_delay);
3328	} else {
3329	if (portptr->port_state == FC_OBJSTATE_ONLINE) {
3330	if (status > 0 && (status & NVME_SC_DNR))
3331	dev_warn(ctrl->ctrl.device,
3332	"NVME-FC{%d}: reconnect failure\n",
3333	ctrl->cnum);
3334	else
3335	dev_warn(ctrl->ctrl.device,
3336	"NVME-FC{%d}: Max reconnect attempts "
3337	"(%d) reached.\n",
3338	ctrl->cnum, ctrl->ctrl.nr_reconnects);
3339	} else
3340	dev_warn(ctrl->ctrl.device,
3341	"NVME-FC{%d}: dev_loss_tmo (%d) expired "
3342	"while waiting for remoteport connectivity.\n",
3343	ctrl->cnum, min_t(int, portptr->dev_loss_tmo,
3344	(ctrl->ctrl.opts->max_reconnects *
3345	ctrl->ctrl.opts->reconnect_delay)));
3346	WARN_ON(nvme_delete_ctrl(&ctrl->ctrl));
3347	}
3348	}
3349
3350	static void
3351	nvme_fc_reset_ctrl_work(struct work_struct *work)
3352	{
3353	struct nvme_fc_ctrl *ctrl =
3354	container_of(work, struct nvme_fc_ctrl, ctrl.reset_work);
3355
3356	nvme_stop_ctrl(ctrl: &ctrl->ctrl);
3357
3358	/* will block will waiting for io to terminate */
3359	nvme_fc_delete_association(ctrl);
3360
3361	if (!nvme_change_ctrl_state(ctrl: &ctrl->ctrl, new_state: NVME_CTRL_CONNECTING))
3362	dev_err(ctrl->ctrl.device,
3363	"NVME-FC{%d}: error_recovery: Couldn't change state "
3364	"to CONNECTING\n", ctrl->cnum);
3365
3366	if (ctrl->rport->remoteport.port_state == FC_OBJSTATE_ONLINE) {
3367	if (!queue_delayed_work(wq: nvme_wq, dwork: &ctrl->connect_work, delay: 0)) {
3368	dev_err(ctrl->ctrl.device,
3369	"NVME-FC{%d}: failed to schedule connect "
3370	"after reset\n", ctrl->cnum);
3371	} else {
3372	flush_delayed_work(dwork: &ctrl->connect_work);
3373	}
3374	} else {
3375	nvme_fc_reconnect_or_delete(ctrl, status: -ENOTCONN);
3376	}
3377	}
3378
3379
3380	static const struct nvme_ctrl_ops nvme_fc_ctrl_ops = {
3381	.name = "fc",
3382	.module = THIS_MODULE,
3383	.flags = NVME_F_FABRICS,
3384	.reg_read32 = nvmf_reg_read32,
3385	.reg_read64 = nvmf_reg_read64,
3386	.reg_write32 = nvmf_reg_write32,
3387	.free_ctrl = nvme_fc_free_ctrl,
3388	.submit_async_event = nvme_fc_submit_async_event,
3389	.delete_ctrl = nvme_fc_delete_ctrl,
3390	.get_address = nvmf_get_address,
3391	};
3392
3393	static void
3394	nvme_fc_connect_ctrl_work(struct work_struct *work)
3395	{
3396	int ret;
3397
3398	struct nvme_fc_ctrl *ctrl =
3399	container_of(to_delayed_work(work),
3400	struct nvme_fc_ctrl, connect_work);
3401
3402	ret = nvme_fc_create_association(ctrl);
3403	if (ret)
3404	nvme_fc_reconnect_or_delete(ctrl, status: ret);
3405	else
3406	dev_info(ctrl->ctrl.device,
3407	"NVME-FC{%d}: controller connect complete\n",
3408	ctrl->cnum);
3409	}
3410
3411
3412	static const struct blk_mq_ops nvme_fc_admin_mq_ops = {
3413	.queue_rq = nvme_fc_queue_rq,
3414	.complete = nvme_fc_complete_rq,
3415	.init_request = nvme_fc_init_request,
3416	.exit_request = nvme_fc_exit_request,
3417	.init_hctx = nvme_fc_init_admin_hctx,
3418	.timeout = nvme_fc_timeout,
3419	};
3420
3421
3422	/*
3423	* Fails a controller request if it matches an existing controller
3424	* (association) with the same tuple:
3425	* <Host NQN, Host ID, local FC port, remote FC port, SUBSYS NQN>
3426	*
3427	* The ports don't need to be compared as they are intrinsically
3428	* already matched by the port pointers supplied.
3429	*/
3430	static bool
3431	nvme_fc_existing_controller(struct nvme_fc_rport *rport,
3432	struct nvmf_ctrl_options *opts)
3433	{
3434	struct nvme_fc_ctrl *ctrl;
3435	unsigned long flags;
3436	bool found = false;
3437
3438	spin_lock_irqsave(&rport->lock, flags);
3439	list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
3440	found = nvmf_ctlr_matches_baseopts(ctrl: &ctrl->ctrl, opts);
3441	if (found)
3442	break;
3443	}
3444	spin_unlock_irqrestore(lock: &rport->lock, flags);
3445
3446	return found;
3447	}
3448
3449	static struct nvme_ctrl *
3450	nvme_fc_init_ctrl(struct device *dev, struct nvmf_ctrl_options *opts,
3451	struct nvme_fc_lport *lport, struct nvme_fc_rport *rport)
3452	{
3453	struct nvme_fc_ctrl *ctrl;
3454	unsigned long flags;
3455	int ret, idx, ctrl_loss_tmo;
3456
3457	if (!(rport->remoteport.port_role &
3458	(FC_PORT_ROLE_NVME_DISCOVERY | FC_PORT_ROLE_NVME_TARGET))) {
3459	ret = -EBADR;
3460	goto out_fail;
3461	}
3462
3463	if (!opts->duplicate_connect &&
3464	nvme_fc_existing_controller(rport, opts)) {
3465	ret = -EALREADY;
3466	goto out_fail;
3467	}
3468
3469	ctrl = kzalloc(size: sizeof(*ctrl), GFP_KERNEL);
3470	if (!ctrl) {
3471	ret = -ENOMEM;
3472	goto out_fail;
3473	}
3474
3475	idx = ida_alloc(ida: &nvme_fc_ctrl_cnt, GFP_KERNEL);
3476	if (idx < 0) {
3477	ret = -ENOSPC;
3478	goto out_free_ctrl;
3479	}
3480
3481	/*
3482	* if ctrl_loss_tmo is being enforced and the default reconnect delay
3483	* is being used, change to a shorter reconnect delay for FC.
3484	*/
3485	if (opts->max_reconnects != -1 &&
3486	opts->reconnect_delay == NVMF_DEF_RECONNECT_DELAY &&
3487	opts->reconnect_delay > NVME_FC_DEFAULT_RECONNECT_TMO) {
3488	ctrl_loss_tmo = opts->max_reconnects * opts->reconnect_delay;
3489	opts->reconnect_delay = NVME_FC_DEFAULT_RECONNECT_TMO;
3490	opts->max_reconnects = DIV_ROUND_UP(ctrl_loss_tmo,
3491	opts->reconnect_delay);
3492	}
3493
3494	ctrl->ctrl.opts = opts;
3495	ctrl->ctrl.nr_reconnects = 0;
3496	INIT_LIST_HEAD(list: &ctrl->ctrl_list);
3497	ctrl->lport = lport;
3498	ctrl->rport = rport;
3499	ctrl->dev = lport->dev;
3500	ctrl->cnum = idx;
3501	ctrl->ioq_live = false;
3502	init_waitqueue_head(&ctrl->ioabort_wait);
3503
3504	get_device(dev: ctrl->dev);
3505	kref_init(kref: &ctrl->ref);
3506
3507	INIT_WORK(&ctrl->ctrl.reset_work, nvme_fc_reset_ctrl_work);
3508	INIT_DELAYED_WORK(&ctrl->connect_work, nvme_fc_connect_ctrl_work);
3509	INIT_WORK(&ctrl->ioerr_work, nvme_fc_ctrl_ioerr_work);
3510	spin_lock_init(&ctrl->lock);
3511
3512	/* io queue count */
3513	ctrl->ctrl.queue_count = min_t(unsigned int,
3514	opts->nr_io_queues,
3515	lport->ops->max_hw_queues);
3516	ctrl->ctrl.queue_count++; /* +1 for admin queue */
3517
3518	ctrl->ctrl.sqsize = opts->queue_size - 1;
3519	ctrl->ctrl.kato = opts->kato;
3520	ctrl->ctrl.cntlid = 0xffff;
3521
3522	ret = -ENOMEM;
3523	ctrl->queues = kcalloc(n: ctrl->ctrl.queue_count,
3524	size: sizeof(struct nvme_fc_queue), GFP_KERNEL);
3525	if (!ctrl->queues)
3526	goto out_free_ida;
3527
3528	nvme_fc_init_queue(ctrl, idx: 0);
3529
3530	/*
3531	* Would have been nice to init io queues tag set as well.
3532	* However, we require interaction from the controller
3533	* for max io queue count before we can do so.
3534	* Defer this to the connect path.
3535	*/
3536
3537	ret = nvme_init_ctrl(ctrl: &ctrl->ctrl, dev, ops: &nvme_fc_ctrl_ops, quirks: 0);
3538	if (ret)
3539	goto out_free_queues;
3540	if (lport->dev)
3541	ctrl->ctrl.numa_node = dev_to_node(dev: lport->dev);
3542
3543	/* at this point, teardown path changes to ref counting on nvme ctrl */
3544
3545	ret = nvme_alloc_admin_tag_set(ctrl: &ctrl->ctrl, set: &ctrl->admin_tag_set,
3546	ops: &nvme_fc_admin_mq_ops,
3547	struct_size_t(struct nvme_fcp_op_w_sgl, priv,
3548	ctrl->lport->ops->fcprqst_priv_sz));
3549	if (ret)
3550	goto fail_ctrl;
3551
3552	spin_lock_irqsave(&rport->lock, flags);
3553	list_add_tail(new: &ctrl->ctrl_list, head: &rport->ctrl_list);
3554	spin_unlock_irqrestore(lock: &rport->lock, flags);
3555
3556	if (!nvme_change_ctrl_state(ctrl: &ctrl->ctrl, new_state: NVME_CTRL_RESETTING) ||
3557	!nvme_change_ctrl_state(ctrl: &ctrl->ctrl, new_state: NVME_CTRL_CONNECTING)) {
3558	dev_err(ctrl->ctrl.device,
3559	"NVME-FC{%d}: failed to init ctrl state\n", ctrl->cnum);
3560	goto fail_ctrl;
3561	}
3562
3563	if (!queue_delayed_work(wq: nvme_wq, dwork: &ctrl->connect_work, delay: 0)) {
3564	dev_err(ctrl->ctrl.device,
3565	"NVME-FC{%d}: failed to schedule initial connect\n",
3566	ctrl->cnum);
3567	goto fail_ctrl;
3568	}
3569
3570	flush_delayed_work(dwork: &ctrl->connect_work);
3571
3572	dev_info(ctrl->ctrl.device,
3573	"NVME-FC{%d}: new ctrl: NQN \"%s\", hostnqn: %s\n",
3574	ctrl->cnum, nvmf_ctrl_subsysnqn(&ctrl->ctrl), opts->host->nqn);
3575
3576	return &ctrl->ctrl;
3577
3578	fail_ctrl:
3579	nvme_change_ctrl_state(ctrl: &ctrl->ctrl, new_state: NVME_CTRL_DELETING);
3580	cancel_work_sync(work: &ctrl->ioerr_work);
3581	cancel_work_sync(work: &ctrl->ctrl.reset_work);
3582	cancel_delayed_work_sync(dwork: &ctrl->connect_work);
3583
3584	ctrl->ctrl.opts = NULL;
3585
3586	/* initiate nvme ctrl ref counting teardown */
3587	nvme_uninit_ctrl(ctrl: &ctrl->ctrl);
3588
3589	/* Remove core ctrl ref. */
3590	nvme_put_ctrl(ctrl: &ctrl->ctrl);
3591
3592	/* as we're past the point where we transition to the ref
3593	* counting teardown path, if we return a bad pointer here,
3594	* the calling routine, thinking it's prior to the
3595	* transition, will do an rport put. Since the teardown
3596	* path also does a rport put, we do an extra get here to
3597	* so proper order/teardown happens.
3598	*/
3599	nvme_fc_rport_get(rport);
3600
3601	return ERR_PTR(error: -EIO);
3602
3603	out_free_queues:
3604	kfree(objp: ctrl->queues);
3605	out_free_ida:
3606	put_device(dev: ctrl->dev);
3607	ida_free(&nvme_fc_ctrl_cnt, id: ctrl->cnum);
3608	out_free_ctrl:
3609	kfree(objp: ctrl);
3610	out_fail:
3611	/* exit via here doesn't follow ctlr ref points */
3612	return ERR_PTR(error: ret);
3613	}
3614
3615
3616	struct nvmet_fc_traddr {
3617	u64 nn;
3618	u64 pn;
3619	};
3620
3621	static int
3622	__nvme_fc_parse_u64(substring_t *sstr, u64 *val)
3623	{
3624	u64 token64;
3625
3626	if (match_u64(sstr, result: &token64))
3627	return -EINVAL;
3628	*val = token64;
3629
3630	return 0;
3631	}
3632
3633	/*
3634	* This routine validates and extracts the WWN's from the TRADDR string.
3635	* As kernel parsers need the 0x to determine number base, universally
3636	* build string to parse with 0x prefix before parsing name strings.
3637	*/
3638	static int
3639	nvme_fc_parse_traddr(struct nvmet_fc_traddr *traddr, char *buf, size_t blen)
3640	{
3641	char name[2 + NVME_FC_TRADDR_HEXNAMELEN + 1];
3642	substring_t wwn = { name, &name[sizeof(name)-1] };
3643	int nnoffset, pnoffset;
3644
3645	/* validate if string is one of the 2 allowed formats */
3646	if (strnlen(p: buf, maxlen: blen) == NVME_FC_TRADDR_MAXLENGTH &&
3647	!strncmp(buf, "nn-0x", NVME_FC_TRADDR_OXNNLEN) &&
3648	!strncmp(&buf[NVME_FC_TRADDR_MAX_PN_OFFSET],
3649	"pn-0x", NVME_FC_TRADDR_OXNNLEN)) {
3650	nnoffset = NVME_FC_TRADDR_OXNNLEN;
3651	pnoffset = NVME_FC_TRADDR_MAX_PN_OFFSET +
3652	NVME_FC_TRADDR_OXNNLEN;
3653	} else if ((strnlen(p: buf, maxlen: blen) == NVME_FC_TRADDR_MINLENGTH &&
3654	!strncmp(buf, "nn-", NVME_FC_TRADDR_NNLEN) &&
3655	!strncmp(&buf[NVME_FC_TRADDR_MIN_PN_OFFSET],
3656	"pn-", NVME_FC_TRADDR_NNLEN))) {
3657	nnoffset = NVME_FC_TRADDR_NNLEN;
3658	pnoffset = NVME_FC_TRADDR_MIN_PN_OFFSET + NVME_FC_TRADDR_NNLEN;
3659	} else
3660	goto out_einval;
3661
3662	name[0] = '0';
3663	name[1] = 'x';
3664	name[2 + NVME_FC_TRADDR_HEXNAMELEN] = 0;
3665
3666	memcpy(&name[2], &buf[nnoffset], NVME_FC_TRADDR_HEXNAMELEN);
3667	if (__nvme_fc_parse_u64(sstr: &wwn, val: &traddr->nn))
3668	goto out_einval;
3669
3670	memcpy(&name[2], &buf[pnoffset], NVME_FC_TRADDR_HEXNAMELEN);
3671	if (__nvme_fc_parse_u64(sstr: &wwn, val: &traddr->pn))
3672	goto out_einval;
3673
3674	return 0;
3675
3676	out_einval:
3677	pr_warn("%s: bad traddr string\n", __func__);
3678	return -EINVAL;
3679	}
3680
3681	static struct nvme_ctrl *
3682	nvme_fc_create_ctrl(struct device *dev, struct nvmf_ctrl_options *opts)
3683	{
3684	struct nvme_fc_lport *lport;
3685	struct nvme_fc_rport *rport;
3686	struct nvme_ctrl *ctrl;
3687	struct nvmet_fc_traddr laddr = { 0L, 0L };
3688	struct nvmet_fc_traddr raddr = { 0L, 0L };
3689	unsigned long flags;
3690	int ret;
3691
3692	ret = nvme_fc_parse_traddr(traddr: &raddr, buf: opts->traddr, NVMF_TRADDR_SIZE);
3693	if (ret || !raddr.nn || !raddr.pn)
3694	return ERR_PTR(error: -EINVAL);
3695
3696	ret = nvme_fc_parse_traddr(traddr: &laddr, buf: opts->host_traddr, NVMF_TRADDR_SIZE);
3697	if (ret || !laddr.nn || !laddr.pn)
3698	return ERR_PTR(error: -EINVAL);
3699
3700	/* find the host and remote ports to connect together */
3701	spin_lock_irqsave(&nvme_fc_lock, flags);
3702	list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
3703	if (lport->localport.node_name != laddr.nn ||
3704	lport->localport.port_name != laddr.pn ||
3705	lport->localport.port_state != FC_OBJSTATE_ONLINE)
3706	continue;
3707
3708	list_for_each_entry(rport, &lport->endp_list, endp_list) {
3709	if (rport->remoteport.node_name != raddr.nn ||
3710	rport->remoteport.port_name != raddr.pn ||
3711	rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
3712	continue;
3713
3714	/* if fail to get reference fall through. Will error */
3715	if (!nvme_fc_rport_get(rport))
3716	break;
3717
3718	spin_unlock_irqrestore(lock: &nvme_fc_lock, flags);
3719
3720	ctrl = nvme_fc_init_ctrl(dev, opts, lport, rport);
3721	if (IS_ERR(ptr: ctrl))
3722	nvme_fc_rport_put(rport);
3723	return ctrl;
3724	}
3725	}
3726	spin_unlock_irqrestore(lock: &nvme_fc_lock, flags);
3727
3728	pr_warn("%s: %s - %s combination not found\n",
3729	__func__, opts->traddr, opts->host_traddr);
3730	return ERR_PTR(error: -ENOENT);
3731	}
3732
3733
3734	static struct nvmf_transport_ops nvme_fc_transport = {
3735	.name = "fc",
3736	.module = THIS_MODULE,
3737	.required_opts = NVMF_OPT_TRADDR | NVMF_OPT_HOST_TRADDR,
3738	.allowed_opts = NVMF_OPT_RECONNECT_DELAY | NVMF_OPT_CTRL_LOSS_TMO,
3739	.create_ctrl = nvme_fc_create_ctrl,
3740	};
3741
3742	/* Arbitrary successive failures max. With lots of subsystems could be high */
3743	#define DISCOVERY_MAX_FAIL 20
3744
3745	static ssize_t nvme_fc_nvme_discovery_store(struct device *dev,
3746	struct device_attribute *attr, const char *buf, size_t count)
3747	{
3748	unsigned long flags;
3749	LIST_HEAD(local_disc_list);
3750	struct nvme_fc_lport *lport;
3751	struct nvme_fc_rport *rport;
3752	int failcnt = 0;
3753
3754	spin_lock_irqsave(&nvme_fc_lock, flags);
3755	restart:
3756	list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
3757	list_for_each_entry(rport, &lport->endp_list, endp_list) {
3758	if (!nvme_fc_lport_get(lport))
3759	continue;
3760	if (!nvme_fc_rport_get(rport)) {
3761	/*
3762	* This is a temporary condition. Upon restart
3763	* this rport will be gone from the list.
3764	*
3765	* Revert the lport put and retry. Anything
3766	* added to the list already will be skipped (as
3767	* they are no longer list_empty). Loops should
3768	* resume at rports that were not yet seen.
3769	*/
3770	nvme_fc_lport_put(lport);
3771
3772	if (failcnt++ < DISCOVERY_MAX_FAIL)
3773	goto restart;
3774
3775	pr_err("nvme_discovery: too many reference "
3776	"failures\n");
3777	goto process_local_list;
3778	}
3779	if (list_empty(head: &rport->disc_list))
3780	list_add_tail(new: &rport->disc_list,
3781	head: &local_disc_list);
3782	}
3783	}
3784
3785	process_local_list:
3786	while (!list_empty(head: &local_disc_list)) {
3787	rport = list_first_entry(&local_disc_list,
3788	struct nvme_fc_rport, disc_list);
3789	list_del_init(entry: &rport->disc_list);
3790	spin_unlock_irqrestore(lock: &nvme_fc_lock, flags);
3791
3792	lport = rport->lport;
3793	/* signal discovery. Won't hurt if it repeats */
3794	nvme_fc_signal_discovery_scan(lport, rport);
3795	nvme_fc_rport_put(rport);
3796	nvme_fc_lport_put(lport);
3797
3798	spin_lock_irqsave(&nvme_fc_lock, flags);
3799	}
3800	spin_unlock_irqrestore(lock: &nvme_fc_lock, flags);
3801
3802	return count;
3803	}
3804
3805	static DEVICE_ATTR(nvme_discovery, 0200, NULL, nvme_fc_nvme_discovery_store);
3806
3807	#ifdef CONFIG_BLK_CGROUP_FC_APPID
3808	/* Parse the cgroup id from a buf and return the length of cgrpid */
3809	static int fc_parse_cgrpid(const char *buf, u64 *id)
3810	{
3811	char cgrp_id[16+1];
3812	int cgrpid_len, j;
3813
3814	memset(cgrp_id, 0x0, sizeof(cgrp_id));
3815	for (cgrpid_len = 0, j = 0; cgrpid_len < 17; cgrpid_len++) {
3816	if (buf[cgrpid_len] != ':')
3817	cgrp_id[cgrpid_len] = buf[cgrpid_len];
3818	else {
3819	j = 1;
3820	break;
3821	}
3822	}
3823	if (!j)
3824	return -EINVAL;
3825	if (kstrtou64(s: cgrp_id, base: 16, res: id) < 0)
3826	return -EINVAL;
3827	return cgrpid_len;
3828	}
3829
3830	/*
3831	* Parse and update the appid in the blkcg associated with the cgroupid.
3832	*/
3833	static ssize_t fc_appid_store(struct device *dev,
3834	struct device_attribute *attr, const char *buf, size_t count)
3835	{
3836	size_t orig_count = count;
3837	u64 cgrp_id;
3838	int appid_len = 0;
3839	int cgrpid_len = 0;
3840	char app_id[FC_APPID_LEN];
3841	int ret = 0;
3842
3843	if (buf[count-1] == '\n')
3844	count--;
3845
3846	if ((count > (16+1+FC_APPID_LEN)) || (!strchr(buf, ':')))
3847	return -EINVAL;
3848
3849	cgrpid_len = fc_parse_cgrpid(buf, id: &cgrp_id);
3850	if (cgrpid_len < 0)
3851	return -EINVAL;
3852	appid_len = count - cgrpid_len - 1;
3853	if (appid_len > FC_APPID_LEN)
3854	return -EINVAL;
3855
3856	memset(app_id, 0x0, sizeof(app_id));
3857	memcpy(app_id, &buf[cgrpid_len+1], appid_len);
3858	ret = blkcg_set_fc_appid(app_id, cgrp_id, app_id_len: sizeof(app_id));
3859	if (ret < 0)
3860	return ret;
3861	return orig_count;
3862	}
3863	static DEVICE_ATTR(appid_store, 0200, NULL, fc_appid_store);
3864	#endif /* CONFIG_BLK_CGROUP_FC_APPID */
3865
3866	static struct attribute *nvme_fc_attrs[] = {
3867	&dev_attr_nvme_discovery.attr,
3868	#ifdef CONFIG_BLK_CGROUP_FC_APPID
3869	&dev_attr_appid_store.attr,
3870	#endif
3871	NULL
3872	};
3873
3874	static const struct attribute_group nvme_fc_attr_group = {
3875	.attrs = nvme_fc_attrs,
3876	};
3877
3878	static const struct attribute_group *nvme_fc_attr_groups[] = {
3879	&nvme_fc_attr_group,
3880	NULL
3881	};
3882
3883	static struct class fc_class = {
3884	.name = "fc",
3885	.dev_groups = nvme_fc_attr_groups,
3886	};
3887
3888	static int __init nvme_fc_init_module(void)
3889	{
3890	int ret;
3891
3892	/*
3893	* NOTE:
3894	* It is expected that in the future the kernel will combine
3895	* the FC-isms that are currently under scsi and now being
3896	* added to by NVME into a new standalone FC class. The SCSI
3897	* and NVME protocols and their devices would be under this
3898	* new FC class.
3899	*
3900	* As we need something to post FC-specific udev events to,
3901	* specifically for nvme probe events, start by creating the
3902	* new device class. When the new standalone FC class is
3903	* put in place, this code will move to a more generic
3904	* location for the class.
3905	*/
3906	ret = class_register(class: &fc_class);
3907	if (ret) {
3908	pr_err("couldn't register class fc\n");
3909	return ret;
3910	}
3911
3912	/*
3913	* Create a device for the FC-centric udev events
3914	*/
3915	fc_udev_device = device_create(cls: &fc_class, NULL, MKDEV(0, 0), NULL,
3916	fmt: "fc_udev_device");
3917	if (IS_ERR(ptr: fc_udev_device)) {
3918	pr_err("couldn't create fc_udev device!\n");
3919	ret = PTR_ERR(ptr: fc_udev_device);
3920	goto out_destroy_class;
3921	}
3922
3923	ret = nvmf_register_transport(ops: &nvme_fc_transport);
3924	if (ret)
3925	goto out_destroy_device;
3926
3927	return 0;
3928
3929	out_destroy_device:
3930	device_destroy(cls: &fc_class, MKDEV(0, 0));
3931	out_destroy_class:
3932	class_unregister(class: &fc_class);
3933
3934	return ret;
3935	}
3936
3937	static void
3938	nvme_fc_delete_controllers(struct nvme_fc_rport *rport)
3939	{
3940	struct nvme_fc_ctrl *ctrl;
3941
3942	spin_lock(lock: &rport->lock);
3943	list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
3944	dev_warn(ctrl->ctrl.device,
3945	"NVME-FC{%d}: transport unloading: deleting ctrl\n",
3946	ctrl->cnum);
3947	nvme_delete_ctrl(ctrl: &ctrl->ctrl);
3948	}
3949	spin_unlock(lock: &rport->lock);
3950	}
3951
3952	static void __exit nvme_fc_exit_module(void)
3953	{
3954	struct nvme_fc_lport *lport;
3955	struct nvme_fc_rport *rport;
3956	unsigned long flags;
3957
3958	spin_lock_irqsave(&nvme_fc_lock, flags);
3959	list_for_each_entry(lport, &nvme_fc_lport_list, port_list)
3960	list_for_each_entry(rport, &lport->endp_list, endp_list)
3961	nvme_fc_delete_controllers(rport);
3962	spin_unlock_irqrestore(lock: &nvme_fc_lock, flags);
3963	flush_workqueue(nvme_delete_wq);
3964
3965	nvmf_unregister_transport(ops: &nvme_fc_transport);
3966
3967	device_destroy(cls: &fc_class, MKDEV(0, 0));
3968	class_unregister(class: &fc_class);
3969	}
3970
3971	module_init(nvme_fc_init_module);
3972	module_exit(nvme_fc_exit_module);
3973
3974	MODULE_DESCRIPTION("NVMe host FC transport driver");
3975	MODULE_LICENSE("GPL v2");
3976