1	/*******************************************************************
2	* This file is part of the Emulex Linux Device Driver for *
3	* Fibre Channel Host Bus Adapters. *
4	* Copyright (C) 2017-2024 Broadcom. All Rights Reserved. The term *
5	* “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. *
6	* Copyright (C) 2004-2016 Emulex. All rights reserved. *
7	* EMULEX and SLI are trademarks of Emulex. *
8	* www.broadcom.com *
9	* Portions Copyright (C) 2004-2005 Christoph Hellwig *
10	* *
11	* This program is free software; you can redistribute it and/or *
12	* modify it under the terms of version 2 of the GNU General *
13	* Public License as published by the Free Software Foundation. *
14	* This program is distributed in the hope that it will be useful. *
15	* ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND *
16	* WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, *
17	* FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE *
18	* DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
19	* TO BE LEGALLY INVALID. See the GNU General Public License for *
20	* more details, a copy of which can be found in the file COPYING *
21	* included with this package. *
22	********************************************************************/
23	#include <linux/pci.h>
24	#include <linux/slab.h>
25	#include <linux/interrupt.h>
26	#include <linux/delay.h>
27	#include <linux/unaligned.h>
28	#include <linux/crc-t10dif.h>
29	#include <net/checksum.h>
30
31	#include <scsi/scsi.h>
32	#include <scsi/scsi_device.h>
33	#include <scsi/scsi_eh.h>
34	#include <scsi/scsi_host.h>
35	#include <scsi/scsi_tcq.h>
36	#include <scsi/scsi_transport_fc.h>
37	#include <scsi/fc/fc_fs.h>
38
39	#include "lpfc_version.h"
40	#include "lpfc_hw4.h"
41	#include "lpfc_hw.h"
42	#include "lpfc_sli.h"
43	#include "lpfc_sli4.h"
44	#include "lpfc_nl.h"
45	#include "lpfc_disc.h"
46	#include "lpfc.h"
47	#include "lpfc_scsi.h"
48	#include "lpfc_nvme.h"
49	#include "lpfc_logmsg.h"
50	#include "lpfc_crtn.h"
51	#include "lpfc_vport.h"
52	#include "lpfc_debugfs.h"
53
54	static struct lpfc_iocbq *lpfc_nvmet_prep_ls_wqe(struct lpfc_hba *,
55	struct lpfc_async_xchg_ctx *,
56	dma_addr_t rspbuf,
57	uint16_t rspsize);
58	static struct lpfc_iocbq *lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *,
59	struct lpfc_async_xchg_ctx *);
60	static int lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *,
61	struct lpfc_async_xchg_ctx *,
62	uint32_t, uint16_t);
63	static int lpfc_nvmet_unsol_fcp_issue_abort(struct lpfc_hba *,
64	struct lpfc_async_xchg_ctx *,
65	uint32_t, uint16_t);
66	static void lpfc_nvmet_wqfull_flush(struct lpfc_hba *, struct lpfc_queue *,
67	struct lpfc_async_xchg_ctx *);
68	static void lpfc_nvmet_fcp_rqst_defer_work(struct work_struct *);
69
70	static void lpfc_nvmet_process_rcv_fcp_req(struct lpfc_nvmet_ctxbuf *ctx_buf);
71
72	static union lpfc_wqe128 lpfc_tsend_cmd_template;
73	static union lpfc_wqe128 lpfc_treceive_cmd_template;
74	static union lpfc_wqe128 lpfc_trsp_cmd_template;
75
76	/* Setup WQE templates for NVME IOs */
77	void
78	lpfc_nvmet_cmd_template(void)
79	{
80	union lpfc_wqe128 *wqe;
81
82	/* TSEND template */
83	wqe = &lpfc_tsend_cmd_template;
84	memset(wqe, 0, sizeof(union lpfc_wqe128));
85
86	/* Word 0, 1, 2 - BDE is variable */
87
88	/* Word 3 - payload_offset_len is zero */
89
90	/* Word 4 - relative_offset is variable */
91
92	/* Word 5 - is zero */
93
94	/* Word 6 - ctxt_tag, xri_tag is variable */
95
96	/* Word 7 - wqe_ar is variable */
97	bf_set(wqe_cmnd, &wqe->fcp_tsend.wqe_com, CMD_FCP_TSEND64_WQE);
98	bf_set(wqe_pu, &wqe->fcp_tsend.wqe_com, PARM_REL_OFF);
99	bf_set(wqe_class, &wqe->fcp_tsend.wqe_com, CLASS3);
100	bf_set(wqe_ct, &wqe->fcp_tsend.wqe_com, SLI4_CT_RPI);
101	bf_set(wqe_ar, &wqe->fcp_tsend.wqe_com, 1);
102
103	/* Word 8 - abort_tag is variable */
104
105	/* Word 9 - reqtag, rcvoxid is variable */
106
107	/* Word 10 - wqes, xc is variable */
108	bf_set(wqe_xchg, &wqe->fcp_tsend.wqe_com, LPFC_NVME_XCHG);
109	bf_set(wqe_dbde, &wqe->fcp_tsend.wqe_com, 1);
110	bf_set(wqe_wqes, &wqe->fcp_tsend.wqe_com, 0);
111	bf_set(wqe_xc, &wqe->fcp_tsend.wqe_com, 1);
112	bf_set(wqe_iod, &wqe->fcp_tsend.wqe_com, LPFC_WQE_IOD_WRITE);
113	bf_set(wqe_lenloc, &wqe->fcp_tsend.wqe_com, LPFC_WQE_LENLOC_WORD12);
114
115	/* Word 11 - sup, irsp, irsplen is variable */
116	bf_set(wqe_cmd_type, &wqe->fcp_tsend.wqe_com, FCP_COMMAND_TSEND);
117	bf_set(wqe_cqid, &wqe->fcp_tsend.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
118	bf_set(wqe_sup, &wqe->fcp_tsend.wqe_com, 0);
119	bf_set(wqe_irsp, &wqe->fcp_tsend.wqe_com, 0);
120	bf_set(wqe_irsplen, &wqe->fcp_tsend.wqe_com, 0);
121	bf_set(wqe_pbde, &wqe->fcp_tsend.wqe_com, 0);
122
123	/* Word 12 - fcp_data_len is variable */
124
125	/* Word 13, 14, 15 - PBDE is zero */
126
127	/* TRECEIVE template */
128	wqe = &lpfc_treceive_cmd_template;
129	memset(wqe, 0, sizeof(union lpfc_wqe128));
130
131	/* Word 0, 1, 2 - BDE is variable */
132
133	/* Word 3 */
134	wqe->fcp_treceive.payload_offset_len = TXRDY_PAYLOAD_LEN;
135
136	/* Word 4 - relative_offset is variable */
137
138	/* Word 5 - is zero */
139
140	/* Word 6 - ctxt_tag, xri_tag is variable */
141
142	/* Word 7 */
143	bf_set(wqe_cmnd, &wqe->fcp_treceive.wqe_com, CMD_FCP_TRECEIVE64_WQE);
144	bf_set(wqe_pu, &wqe->fcp_treceive.wqe_com, PARM_REL_OFF);
145	bf_set(wqe_class, &wqe->fcp_treceive.wqe_com, CLASS3);
146	bf_set(wqe_ct, &wqe->fcp_treceive.wqe_com, SLI4_CT_RPI);
147	bf_set(wqe_ar, &wqe->fcp_treceive.wqe_com, 0);
148
149	/* Word 8 - abort_tag is variable */
150
151	/* Word 9 - reqtag, rcvoxid is variable */
152
153	/* Word 10 - xc is variable */
154	bf_set(wqe_dbde, &wqe->fcp_treceive.wqe_com, 1);
155	bf_set(wqe_wqes, &wqe->fcp_treceive.wqe_com, 0);
156	bf_set(wqe_xchg, &wqe->fcp_treceive.wqe_com, LPFC_NVME_XCHG);
157	bf_set(wqe_iod, &wqe->fcp_treceive.wqe_com, LPFC_WQE_IOD_READ);
158	bf_set(wqe_lenloc, &wqe->fcp_treceive.wqe_com, LPFC_WQE_LENLOC_WORD12);
159	bf_set(wqe_xc, &wqe->fcp_tsend.wqe_com, 1);
160
161	/* Word 11 - pbde is variable */
162	bf_set(wqe_cmd_type, &wqe->fcp_treceive.wqe_com, FCP_COMMAND_TRECEIVE);
163	bf_set(wqe_cqid, &wqe->fcp_treceive.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
164	bf_set(wqe_sup, &wqe->fcp_treceive.wqe_com, 0);
165	bf_set(wqe_irsp, &wqe->fcp_treceive.wqe_com, 0);
166	bf_set(wqe_irsplen, &wqe->fcp_treceive.wqe_com, 0);
167	bf_set(wqe_pbde, &wqe->fcp_treceive.wqe_com, 1);
168
169	/* Word 12 - fcp_data_len is variable */
170
171	/* Word 13, 14, 15 - PBDE is variable */
172
173	/* TRSP template */
174	wqe = &lpfc_trsp_cmd_template;
175	memset(wqe, 0, sizeof(union lpfc_wqe128));
176
177	/* Word 0, 1, 2 - BDE is variable */
178
179	/* Word 3 - response_len is variable */
180
181	/* Word 4, 5 - is zero */
182
183	/* Word 6 - ctxt_tag, xri_tag is variable */
184
185	/* Word 7 */
186	bf_set(wqe_cmnd, &wqe->fcp_trsp.wqe_com, CMD_FCP_TRSP64_WQE);
187	bf_set(wqe_pu, &wqe->fcp_trsp.wqe_com, PARM_UNUSED);
188	bf_set(wqe_class, &wqe->fcp_trsp.wqe_com, CLASS3);
189	bf_set(wqe_ct, &wqe->fcp_trsp.wqe_com, SLI4_CT_RPI);
190	bf_set(wqe_ag, &wqe->fcp_trsp.wqe_com, 1); /* wqe_ar */
191
192	/* Word 8 - abort_tag is variable */
193
194	/* Word 9 - reqtag is variable */
195
196	/* Word 10 wqes, xc is variable */
197	bf_set(wqe_dbde, &wqe->fcp_trsp.wqe_com, 1);
198	bf_set(wqe_xchg, &wqe->fcp_trsp.wqe_com, LPFC_NVME_XCHG);
199	bf_set(wqe_wqes, &wqe->fcp_trsp.wqe_com, 0);
200	bf_set(wqe_xc, &wqe->fcp_trsp.wqe_com, 0);
201	bf_set(wqe_iod, &wqe->fcp_trsp.wqe_com, LPFC_WQE_IOD_NONE);
202	bf_set(wqe_lenloc, &wqe->fcp_trsp.wqe_com, LPFC_WQE_LENLOC_WORD3);
203
204	/* Word 11 irsp, irsplen is variable */
205	bf_set(wqe_cmd_type, &wqe->fcp_trsp.wqe_com, FCP_COMMAND_TRSP);
206	bf_set(wqe_cqid, &wqe->fcp_trsp.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
207	bf_set(wqe_sup, &wqe->fcp_trsp.wqe_com, 0);
208	bf_set(wqe_irsp, &wqe->fcp_trsp.wqe_com, 0);
209	bf_set(wqe_irsplen, &wqe->fcp_trsp.wqe_com, 0);
210	bf_set(wqe_pbde, &wqe->fcp_trsp.wqe_com, 0);
211
212	/* Word 12, 13, 14, 15 - is zero */
213	}
214
215	#if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
216	static struct lpfc_async_xchg_ctx *
217	lpfc_nvmet_get_ctx_for_xri(struct lpfc_hba *phba, u16 xri)
218	{
219	struct lpfc_async_xchg_ctx *ctxp;
220	unsigned long iflag;
221	bool found = false;
222
223	spin_lock_irqsave(&phba->sli4_hba.t_active_list_lock, iflag);
224	list_for_each_entry(ctxp, &phba->sli4_hba.t_active_ctx_list, list) {
225	if (ctxp->ctxbuf->sglq->sli4_xritag != xri)
226	continue;
227
228	found = true;
229	break;
230	}
231	spin_unlock_irqrestore(lock: &phba->sli4_hba.t_active_list_lock, flags: iflag);
232	if (found)
233	return ctxp;
234
235	return NULL;
236	}
237
238	static struct lpfc_async_xchg_ctx *
239	lpfc_nvmet_get_ctx_for_oxid(struct lpfc_hba *phba, u16 oxid, u32 sid)
240	{
241	struct lpfc_async_xchg_ctx *ctxp;
242	unsigned long iflag;
243	bool found = false;
244
245	spin_lock_irqsave(&phba->sli4_hba.t_active_list_lock, iflag);
246	list_for_each_entry(ctxp, &phba->sli4_hba.t_active_ctx_list, list) {
247	if (ctxp->oxid != oxid || ctxp->sid != sid)
248	continue;
249
250	found = true;
251	break;
252	}
253	spin_unlock_irqrestore(lock: &phba->sli4_hba.t_active_list_lock, flags: iflag);
254	if (found)
255	return ctxp;
256
257	return NULL;
258	}
259	#endif
260
261	static void
262	lpfc_nvmet_defer_release(struct lpfc_hba *phba,
263	struct lpfc_async_xchg_ctx *ctxp)
264	{
265	lockdep_assert_held(&ctxp->ctxlock);
266
267	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
268	"6313 NVMET Defer ctx release oxid x%x flg x%x\n",
269	ctxp->oxid, ctxp->flag);
270
271	if (ctxp->flag & LPFC_NVME_CTX_RLS)
272	return;
273
274	ctxp->flag |= LPFC_NVME_CTX_RLS;
275	spin_lock(lock: &phba->sli4_hba.t_active_list_lock);
276	list_del(entry: &ctxp->list);
277	spin_unlock(lock: &phba->sli4_hba.t_active_list_lock);
278	spin_lock(lock: &phba->sli4_hba.abts_nvmet_buf_list_lock);
279	list_add_tail(new: &ctxp->list, head: &phba->sli4_hba.lpfc_abts_nvmet_ctx_list);
280	spin_unlock(lock: &phba->sli4_hba.abts_nvmet_buf_list_lock);
281	}
282
283	/**
284	* __lpfc_nvme_xmt_ls_rsp_cmp - Generic completion handler for the
285	* transmission of an NVME LS response.
286	* @phba: Pointer to HBA context object.
287	* @cmdwqe: Pointer to driver command WQE object.
288	* @rspwqe: Pointer to driver response WQE object.
289	*
290	* The function is called from SLI ring event handler with no
291	* lock held. The function frees memory resources used for the command
292	* used to send the NVME LS RSP.
293	**/
294	void
295	__lpfc_nvme_xmt_ls_rsp_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
296	struct lpfc_iocbq *rspwqe)
297	{
298	struct lpfc_async_xchg_ctx *axchg = cmdwqe->context_un.axchg;
299	struct lpfc_wcqe_complete *wcqe = &rspwqe->wcqe_cmpl;
300	struct nvmefc_ls_rsp *ls_rsp = &axchg->ls_rsp;
301	uint32_t status, result;
302
303	status = bf_get(lpfc_wcqe_c_status, wcqe);
304	result = wcqe->parameter;
305
306	if (axchg->state != LPFC_NVME_STE_LS_RSP || axchg->entry_cnt != 2) {
307	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
308	"6410 NVMEx LS cmpl state mismatch IO x%x: "
309	"%d %d\n",
310	axchg->oxid, axchg->state, axchg->entry_cnt);
311	}
312
313	lpfc_nvmeio_data(phba, "NVMEx LS CMPL: xri x%x stat x%x result x%x\n",
314	axchg->oxid, status, result);
315
316	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
317	"6038 NVMEx LS rsp cmpl: %d %d oxid x%x\n",
318	status, result, axchg->oxid);
319
320	lpfc_nlp_put(cmdwqe->ndlp);
321	cmdwqe->context_un.axchg = NULL;
322	cmdwqe->bpl_dmabuf = NULL;
323	lpfc_sli_release_iocbq(phba, cmdwqe);
324	ls_rsp->done(ls_rsp);
325	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
326	"6200 NVMEx LS rsp cmpl done status %d oxid x%x\n",
327	status, axchg->oxid);
328	kfree(objp: axchg);
329	}
330
331	/**
332	* lpfc_nvmet_xmt_ls_rsp_cmp - Completion handler for LS Response
333	* @phba: Pointer to HBA context object.
334	* @cmdwqe: Pointer to driver command WQE object.
335	* @rspwqe: Pointer to driver response WQE object.
336	*
337	* The function is called from SLI ring event handler with no
338	* lock held. This function is the completion handler for NVME LS commands
339	* The function updates any states and statistics, then calls the
340	* generic completion handler to free resources.
341	**/
342	static void
343	lpfc_nvmet_xmt_ls_rsp_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
344	struct lpfc_iocbq *rspwqe)
345	{
346	struct lpfc_nvmet_tgtport *tgtp;
347	uint32_t status, result;
348	struct lpfc_wcqe_complete *wcqe = &rspwqe->wcqe_cmpl;
349
350	if (!phba->targetport)
351	goto finish;
352
353	status = bf_get(lpfc_wcqe_c_status, wcqe);
354	result = wcqe->parameter;
355
356	tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
357	if (tgtp) {
358	if (status) {
359	atomic_inc(v: &tgtp->xmt_ls_rsp_error);
360	if (result == IOERR_ABORT_REQUESTED)
361	atomic_inc(v: &tgtp->xmt_ls_rsp_aborted);
362	if (bf_get(lpfc_wcqe_c_xb, wcqe))
363	atomic_inc(v: &tgtp->xmt_ls_rsp_xb_set);
364	} else {
365	atomic_inc(v: &tgtp->xmt_ls_rsp_cmpl);
366	}
367	}
368
369	finish:
370	__lpfc_nvme_xmt_ls_rsp_cmp(phba, cmdwqe, rspwqe);
371	}
372
373	/**
374	* lpfc_nvmet_ctxbuf_post - Repost a NVMET RQ DMA buffer and clean up context
375	* @phba: HBA buffer is associated with
376	* @ctx_buf: ctx buffer context
377	*
378	* Description: Frees the given DMA buffer in the appropriate way given by
379	* reposting it to its associated RQ so it can be reused.
380	*
381	* Notes: Takes phba->hbalock. Can be called with or without other locks held.
382	*
383	* Returns: None
384	**/
385	void
386	lpfc_nvmet_ctxbuf_post(struct lpfc_hba *phba, struct lpfc_nvmet_ctxbuf *ctx_buf)
387	{
388	#if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
389	struct lpfc_async_xchg_ctx *ctxp = ctx_buf->context;
390	struct lpfc_nvmet_tgtport *tgtp;
391	struct fc_frame_header *fc_hdr;
392	struct rqb_dmabuf *nvmebuf;
393	struct lpfc_nvmet_ctx_info *infop;
394	uint32_t size, oxid, sid;
395	int cpu;
396	unsigned long iflag;
397
398	if (ctxp->state == LPFC_NVME_STE_FREE) {
399	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
400	"6411 NVMET free, already free IO x%x: %d %d\n",
401	ctxp->oxid, ctxp->state, ctxp->entry_cnt);
402	}
403
404	if (ctxp->rqb_buffer) {
405	spin_lock_irqsave(&ctxp->ctxlock, iflag);
406	nvmebuf = ctxp->rqb_buffer;
407	/* check if freed in another path whilst acquiring lock */
408	if (nvmebuf) {
409	ctxp->rqb_buffer = NULL;
410	if (ctxp->flag & LPFC_NVME_CTX_REUSE_WQ) {
411	ctxp->flag &= ~LPFC_NVME_CTX_REUSE_WQ;
412	spin_unlock_irqrestore(lock: &ctxp->ctxlock, flags: iflag);
413	nvmebuf->hrq->rqbp->rqb_free_buffer(phba,
414	nvmebuf);
415	} else {
416	spin_unlock_irqrestore(lock: &ctxp->ctxlock, flags: iflag);
417	/* repost */
418	lpfc_rq_buf_free(phba, mp: &nvmebuf->hbuf);
419	}
420	} else {
421	spin_unlock_irqrestore(lock: &ctxp->ctxlock, flags: iflag);
422	}
423	}
424	ctxp->state = LPFC_NVME_STE_FREE;
425
426	spin_lock_irqsave(&phba->sli4_hba.nvmet_io_wait_lock, iflag);
427	if (phba->sli4_hba.nvmet_io_wait_cnt) {
428	list_remove_head(&phba->sli4_hba.lpfc_nvmet_io_wait_list,
429	nvmebuf, struct rqb_dmabuf,
430	hbuf.list);
431	phba->sli4_hba.nvmet_io_wait_cnt--;
432	spin_unlock_irqrestore(lock: &phba->sli4_hba.nvmet_io_wait_lock,
433	flags: iflag);
434
435	fc_hdr = (struct fc_frame_header *)(nvmebuf->hbuf.virt);
436	oxid = be16_to_cpu(fc_hdr->fh_ox_id);
437	tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
438	size = nvmebuf->bytes_recv;
439	sid = sli4_sid_from_fc_hdr(fc_hdr);
440
441	ctxp = (struct lpfc_async_xchg_ctx *)ctx_buf->context;
442	ctxp->wqeq = NULL;
443	ctxp->offset = 0;
444	ctxp->phba = phba;
445	ctxp->size = size;
446	ctxp->oxid = oxid;
447	ctxp->sid = sid;
448	ctxp->state = LPFC_NVME_STE_RCV;
449	ctxp->entry_cnt = 1;
450	ctxp->flag = 0;
451	ctxp->ctxbuf = ctx_buf;
452	ctxp->rqb_buffer = (void *)nvmebuf;
453	spin_lock_init(&ctxp->ctxlock);
454
455	#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
456	/* NOTE: isr time stamp is stale when context is re-assigned*/
457	if (ctxp->ts_isr_cmd) {
458	ctxp->ts_cmd_nvme = 0;
459	ctxp->ts_nvme_data = 0;
460	ctxp->ts_data_wqput = 0;
461	ctxp->ts_isr_data = 0;
462	ctxp->ts_data_nvme = 0;
463	ctxp->ts_nvme_status = 0;
464	ctxp->ts_status_wqput = 0;
465	ctxp->ts_isr_status = 0;
466	ctxp->ts_status_nvme = 0;
467	}
468	#endif
469	atomic_inc(v: &tgtp->rcv_fcp_cmd_in);
470
471	/* Indicate that a replacement buffer has been posted */
472	spin_lock_irqsave(&ctxp->ctxlock, iflag);
473	ctxp->flag |= LPFC_NVME_CTX_REUSE_WQ;
474	spin_unlock_irqrestore(lock: &ctxp->ctxlock, flags: iflag);
475
476	if (!queue_work(wq: phba->wq, work: &ctx_buf->defer_work)) {
477	atomic_inc(v: &tgtp->rcv_fcp_cmd_drop);
478	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
479	"6181 Unable to queue deferred work "
480	"for oxid x%x. "
481	"FCP Drop IO [x%x x%x x%x]\n",
482	ctxp->oxid,
483	atomic_read(&tgtp->rcv_fcp_cmd_in),
484	atomic_read(&tgtp->rcv_fcp_cmd_out),
485	atomic_read(&tgtp->xmt_fcp_release));
486
487	spin_lock_irqsave(&ctxp->ctxlock, iflag);
488	lpfc_nvmet_defer_release(phba, ctxp);
489	spin_unlock_irqrestore(lock: &ctxp->ctxlock, flags: iflag);
490	lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, sid, oxid);
491	}
492	return;
493	}
494	spin_unlock_irqrestore(lock: &phba->sli4_hba.nvmet_io_wait_lock, flags: iflag);
495
496	/*
497	* Use the CPU context list, from the MRQ the IO was received on
498	* (ctxp->idx), to save context structure.
499	*/
500	spin_lock_irqsave(&phba->sli4_hba.t_active_list_lock, iflag);
501	list_del_init(entry: &ctxp->list);
502	spin_unlock_irqrestore(lock: &phba->sli4_hba.t_active_list_lock, flags: iflag);
503	cpu = raw_smp_processor_id();
504	infop = lpfc_get_ctx_list(phba, cpu, ctxp->idx);
505	spin_lock_irqsave(&infop->nvmet_ctx_list_lock, iflag);
506	list_add_tail(new: &ctx_buf->list, head: &infop->nvmet_ctx_list);
507	infop->nvmet_ctx_list_cnt++;
508	spin_unlock_irqrestore(lock: &infop->nvmet_ctx_list_lock, flags: iflag);
509	#endif
510	}
511
512	#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
513	static void
514	lpfc_nvmet_ktime(struct lpfc_hba *phba,
515	struct lpfc_async_xchg_ctx *ctxp)
516	{
517	uint64_t seg1, seg2, seg3, seg4, seg5;
518	uint64_t seg6, seg7, seg8, seg9, seg10;
519	uint64_t segsum;
520
521	if (!ctxp->ts_isr_cmd || !ctxp->ts_cmd_nvme ||
522	!ctxp->ts_nvme_data || !ctxp->ts_data_wqput ||
523	!ctxp->ts_isr_data || !ctxp->ts_data_nvme ||
524	!ctxp->ts_nvme_status || !ctxp->ts_status_wqput ||
525	!ctxp->ts_isr_status || !ctxp->ts_status_nvme)
526	return;
527
528	if (ctxp->ts_status_nvme < ctxp->ts_isr_cmd)
529	return;
530	if (ctxp->ts_isr_cmd > ctxp->ts_cmd_nvme)
531	return;
532	if (ctxp->ts_cmd_nvme > ctxp->ts_nvme_data)
533	return;
534	if (ctxp->ts_nvme_data > ctxp->ts_data_wqput)
535	return;
536	if (ctxp->ts_data_wqput > ctxp->ts_isr_data)
537	return;
538	if (ctxp->ts_isr_data > ctxp->ts_data_nvme)
539	return;
540	if (ctxp->ts_data_nvme > ctxp->ts_nvme_status)
541	return;
542	if (ctxp->ts_nvme_status > ctxp->ts_status_wqput)
543	return;
544	if (ctxp->ts_status_wqput > ctxp->ts_isr_status)
545	return;
546	if (ctxp->ts_isr_status > ctxp->ts_status_nvme)
547	return;
548	/*
549	* Segment 1 - Time from FCP command received by MSI-X ISR
550	* to FCP command is passed to NVME Layer.
551	* Segment 2 - Time from FCP command payload handed
552	* off to NVME Layer to Driver receives a Command op
553	* from NVME Layer.
554	* Segment 3 - Time from Driver receives a Command op
555	* from NVME Layer to Command is put on WQ.
556	* Segment 4 - Time from Driver WQ put is done
557	* to MSI-X ISR for Command cmpl.
558	* Segment 5 - Time from MSI-X ISR for Command cmpl to
559	* Command cmpl is passed to NVME Layer.
560	* Segment 6 - Time from Command cmpl is passed to NVME
561	* Layer to Driver receives a RSP op from NVME Layer.
562	* Segment 7 - Time from Driver receives a RSP op from
563	* NVME Layer to WQ put is done on TRSP FCP Status.
564	* Segment 8 - Time from Driver WQ put is done on TRSP
565	* FCP Status to MSI-X ISR for TRSP cmpl.
566	* Segment 9 - Time from MSI-X ISR for TRSP cmpl to
567	* TRSP cmpl is passed to NVME Layer.
568	* Segment 10 - Time from FCP command received by
569	* MSI-X ISR to command is completed on wire.
570	* (Segments 1 thru 8) for READDATA / WRITEDATA
571	* (Segments 1 thru 4) for READDATA_RSP
572	*/
573	seg1 = ctxp->ts_cmd_nvme - ctxp->ts_isr_cmd;
574	segsum = seg1;
575
576	seg2 = ctxp->ts_nvme_data - ctxp->ts_isr_cmd;
577	if (segsum > seg2)
578	return;
579	seg2 -= segsum;
580	segsum += seg2;
581
582	seg3 = ctxp->ts_data_wqput - ctxp->ts_isr_cmd;
583	if (segsum > seg3)
584	return;
585	seg3 -= segsum;
586	segsum += seg3;
587
588	seg4 = ctxp->ts_isr_data - ctxp->ts_isr_cmd;
589	if (segsum > seg4)
590	return;
591	seg4 -= segsum;
592	segsum += seg4;
593
594	seg5 = ctxp->ts_data_nvme - ctxp->ts_isr_cmd;
595	if (segsum > seg5)
596	return;
597	seg5 -= segsum;
598	segsum += seg5;
599
600
601	/* For auto rsp commands seg6 thru seg10 will be 0 */
602	if (ctxp->ts_nvme_status > ctxp->ts_data_nvme) {
603	seg6 = ctxp->ts_nvme_status - ctxp->ts_isr_cmd;
604	if (segsum > seg6)
605	return;
606	seg6 -= segsum;
607	segsum += seg6;
608
609	seg7 = ctxp->ts_status_wqput - ctxp->ts_isr_cmd;
610	if (segsum > seg7)
611	return;
612	seg7 -= segsum;
613	segsum += seg7;
614
615	seg8 = ctxp->ts_isr_status - ctxp->ts_isr_cmd;
616	if (segsum > seg8)
617	return;
618	seg8 -= segsum;
619	segsum += seg8;
620
621	seg9 = ctxp->ts_status_nvme - ctxp->ts_isr_cmd;
622	if (segsum > seg9)
623	return;
624	seg9 -= segsum;
625	segsum += seg9;
626
627	if (ctxp->ts_isr_status < ctxp->ts_isr_cmd)
628	return;
629	seg10 = (ctxp->ts_isr_status -
630	ctxp->ts_isr_cmd);
631	} else {
632	if (ctxp->ts_isr_data < ctxp->ts_isr_cmd)
633	return;
634	seg6 = 0;
635	seg7 = 0;
636	seg8 = 0;
637	seg9 = 0;
638	seg10 = (ctxp->ts_isr_data - ctxp->ts_isr_cmd);
639	}
640
641	phba->ktime_seg1_total += seg1;
642	if (seg1 < phba->ktime_seg1_min)
643	phba->ktime_seg1_min = seg1;
644	else if (seg1 > phba->ktime_seg1_max)
645	phba->ktime_seg1_max = seg1;
646
647	phba->ktime_seg2_total += seg2;
648	if (seg2 < phba->ktime_seg2_min)
649	phba->ktime_seg2_min = seg2;
650	else if (seg2 > phba->ktime_seg2_max)
651	phba->ktime_seg2_max = seg2;
652
653	phba->ktime_seg3_total += seg3;
654	if (seg3 < phba->ktime_seg3_min)
655	phba->ktime_seg3_min = seg3;
656	else if (seg3 > phba->ktime_seg3_max)
657	phba->ktime_seg3_max = seg3;
658
659	phba->ktime_seg4_total += seg4;
660	if (seg4 < phba->ktime_seg4_min)
661	phba->ktime_seg4_min = seg4;
662	else if (seg4 > phba->ktime_seg4_max)
663	phba->ktime_seg4_max = seg4;
664
665	phba->ktime_seg5_total += seg5;
666	if (seg5 < phba->ktime_seg5_min)
667	phba->ktime_seg5_min = seg5;
668	else if (seg5 > phba->ktime_seg5_max)
669	phba->ktime_seg5_max = seg5;
670
671	phba->ktime_data_samples++;
672	if (!seg6)
673	goto out;
674
675	phba->ktime_seg6_total += seg6;
676	if (seg6 < phba->ktime_seg6_min)
677	phba->ktime_seg6_min = seg6;
678	else if (seg6 > phba->ktime_seg6_max)
679	phba->ktime_seg6_max = seg6;
680
681	phba->ktime_seg7_total += seg7;
682	if (seg7 < phba->ktime_seg7_min)
683	phba->ktime_seg7_min = seg7;
684	else if (seg7 > phba->ktime_seg7_max)
685	phba->ktime_seg7_max = seg7;
686
687	phba->ktime_seg8_total += seg8;
688	if (seg8 < phba->ktime_seg8_min)
689	phba->ktime_seg8_min = seg8;
690	else if (seg8 > phba->ktime_seg8_max)
691	phba->ktime_seg8_max = seg8;
692
693	phba->ktime_seg9_total += seg9;
694	if (seg9 < phba->ktime_seg9_min)
695	phba->ktime_seg9_min = seg9;
696	else if (seg9 > phba->ktime_seg9_max)
697	phba->ktime_seg9_max = seg9;
698	out:
699	phba->ktime_seg10_total += seg10;
700	if (seg10 < phba->ktime_seg10_min)
701	phba->ktime_seg10_min = seg10;
702	else if (seg10 > phba->ktime_seg10_max)
703	phba->ktime_seg10_max = seg10;
704	phba->ktime_status_samples++;
705	}
706	#endif
707
708	/**
709	* lpfc_nvmet_xmt_fcp_op_cmp - Completion handler for FCP Response
710	* @phba: Pointer to HBA context object.
711	* @cmdwqe: Pointer to driver command WQE object.
712	* @rspwqe: Pointer to driver response WQE object.
713	*
714	* The function is called from SLI ring event handler with no
715	* lock held. This function is the completion handler for NVME FCP commands
716	* The function frees memory resources used for the NVME commands.
717	**/
718	static void
719	lpfc_nvmet_xmt_fcp_op_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
720	struct lpfc_iocbq *rspwqe)
721	{
722	struct lpfc_nvmet_tgtport *tgtp;
723	struct nvmefc_tgt_fcp_req *rsp;
724	struct lpfc_async_xchg_ctx *ctxp;
725	uint32_t status, result, op, logerr;
726	struct lpfc_wcqe_complete *wcqe = &rspwqe->wcqe_cmpl;
727	#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
728	int id;
729	#endif
730
731	ctxp = cmdwqe->context_un.axchg;
732	ctxp->flag &= ~LPFC_NVME_IO_INP;
733
734	rsp = &ctxp->hdlrctx.fcp_req;
735	op = rsp->op;
736
737	status = bf_get(lpfc_wcqe_c_status, wcqe);
738	result = wcqe->parameter;
739
740	if (phba->targetport)
741	tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
742	else
743	tgtp = NULL;
744
745	lpfc_nvmeio_data(phba, "NVMET FCP CMPL: xri x%x op x%x status x%x\n",
746	ctxp->oxid, op, status);
747
748	if (status) {
749	rsp->fcp_error = NVME_SC_DATA_XFER_ERROR;
750	rsp->transferred_length = 0;
751	if (tgtp) {
752	atomic_inc(v: &tgtp->xmt_fcp_rsp_error);
753	if (result == IOERR_ABORT_REQUESTED)
754	atomic_inc(v: &tgtp->xmt_fcp_rsp_aborted);
755	}
756
757	logerr = LOG_NVME_IOERR;
758
759	/* pick up SLI4 exhange busy condition */
760	if (bf_get(lpfc_wcqe_c_xb, wcqe)) {
761	ctxp->flag |= LPFC_NVME_XBUSY;
762	logerr |= LOG_NVME_ABTS;
763	if (tgtp)
764	atomic_inc(v: &tgtp->xmt_fcp_rsp_xb_set);
765
766	} else {
767	ctxp->flag &= ~LPFC_NVME_XBUSY;
768	}
769
770	lpfc_printf_log(phba, KERN_INFO, logerr,
771	"6315 IO Error Cmpl oxid: x%x xri: x%x %x/%x "
772	"XBUSY:x%x\n",
773	ctxp->oxid, ctxp->ctxbuf->sglq->sli4_xritag,
774	status, result, ctxp->flag);
775
776	} else {
777	rsp->fcp_error = NVME_SC_SUCCESS;
778	if (op == NVMET_FCOP_RSP)
779	rsp->transferred_length = rsp->rsplen;
780	else
781	rsp->transferred_length = rsp->transfer_length;
782	if (tgtp)
783	atomic_inc(v: &tgtp->xmt_fcp_rsp_cmpl);
784	}
785
786	if ((op == NVMET_FCOP_READDATA_RSP) ||
787	(op == NVMET_FCOP_RSP)) {
788	/* Sanity check */
789	ctxp->state = LPFC_NVME_STE_DONE;
790	ctxp->entry_cnt++;
791
792	#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
793	if (ctxp->ts_cmd_nvme) {
794	if (rsp->op == NVMET_FCOP_READDATA_RSP) {
795	ctxp->ts_isr_data =
796	cmdwqe->isr_timestamp;
797	ctxp->ts_data_nvme =
798	ktime_get_ns();
799	ctxp->ts_nvme_status =
800	ctxp->ts_data_nvme;
801	ctxp->ts_status_wqput =
802	ctxp->ts_data_nvme;
803	ctxp->ts_isr_status =
804	ctxp->ts_data_nvme;
805	ctxp->ts_status_nvme =
806	ctxp->ts_data_nvme;
807	} else {
808	ctxp->ts_isr_status =
809	cmdwqe->isr_timestamp;
810	ctxp->ts_status_nvme =
811	ktime_get_ns();
812	}
813	}
814	#endif
815	rsp->done(rsp);
816	#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
817	if (ctxp->ts_cmd_nvme)
818	lpfc_nvmet_ktime(phba, ctxp);
819	#endif
820	/* lpfc_nvmet_xmt_fcp_release() will recycle the context */
821	} else {
822	ctxp->entry_cnt++;
823	memset_startat(cmdwqe, 0, cmd_flag);
824	#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
825	if (ctxp->ts_cmd_nvme) {
826	ctxp->ts_isr_data = cmdwqe->isr_timestamp;
827	ctxp->ts_data_nvme = ktime_get_ns();
828	}
829	#endif
830	rsp->done(rsp);
831	}
832	#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
833	if (phba->hdwqstat_on & LPFC_CHECK_NVMET_IO) {
834	id = raw_smp_processor_id();
835	this_cpu_inc(phba->sli4_hba.c_stat->cmpl_io);
836	if (ctxp->cpu != id)
837	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,
838	"6704 CPU Check cmdcmpl: "
839	"cpu %d expect %d\n",
840	id, ctxp->cpu);
841	}
842	#endif
843	}
844
845	/**
846	* __lpfc_nvme_xmt_ls_rsp - Generic service routine to issue transmit
847	* an NVME LS rsp for a prior NVME LS request that was received.
848	* @axchg: pointer to exchange context for the NVME LS request the response
849	* is for.
850	* @ls_rsp: pointer to the transport LS RSP that is to be sent
851	* @xmt_ls_rsp_cmp: completion routine to call upon RSP transmit done
852	*
853	* This routine is used to format and send a WQE to transmit a NVME LS
854	* Response. The response is for a prior NVME LS request that was
855	* received and posted to the transport.
856	*
857	* Returns:
858	* 0 : if response successfully transmit
859	* non-zero : if response failed to transmit, of the form -Exxx.
860	**/
861	int
862	__lpfc_nvme_xmt_ls_rsp(struct lpfc_async_xchg_ctx *axchg,
863	struct nvmefc_ls_rsp *ls_rsp,
864	void (*xmt_ls_rsp_cmp)(struct lpfc_hba *phba,
865	struct lpfc_iocbq *cmdwqe,
866	struct lpfc_iocbq *rspwqe))
867	{
868	struct lpfc_hba *phba = axchg->phba;
869	struct hbq_dmabuf *nvmebuf = (struct hbq_dmabuf *)axchg->rqb_buffer;
870	struct lpfc_iocbq *nvmewqeq;
871	struct lpfc_dmabuf dmabuf;
872	struct ulp_bde64 bpl;
873	int rc;
874
875	if (test_bit(FC_UNLOADING, &phba->pport->load_flag))
876	return -ENODEV;
877
878	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
879	"6023 NVMEx LS rsp oxid x%x\n", axchg->oxid);
880
881	if (axchg->state != LPFC_NVME_STE_LS_RCV || axchg->entry_cnt != 1) {
882	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
883	"6412 NVMEx LS rsp state mismatch "
884	"oxid x%x: %d %d\n",
885	axchg->oxid, axchg->state, axchg->entry_cnt);
886	return -EALREADY;
887	}
888	axchg->state = LPFC_NVME_STE_LS_RSP;
889	axchg->entry_cnt++;
890
891	nvmewqeq = lpfc_nvmet_prep_ls_wqe(phba, axchg, rspbuf: ls_rsp->rspdma,
892	rspsize: ls_rsp->rsplen);
893	if (nvmewqeq == NULL) {
894	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
895	"6150 NVMEx LS Drop Rsp x%x: Prep\n",
896	axchg->oxid);
897	rc = -ENOMEM;
898	goto out_free_buf;
899	}
900
901	/* Save numBdes for bpl2sgl */
902	nvmewqeq->num_bdes = 1;
903	nvmewqeq->hba_wqidx = 0;
904	nvmewqeq->bpl_dmabuf = &dmabuf;
905	dmabuf.virt = &bpl;
906	bpl.addrLow = nvmewqeq->wqe.xmit_sequence.bde.addrLow;
907	bpl.addrHigh = nvmewqeq->wqe.xmit_sequence.bde.addrHigh;
908	bpl.tus.f.bdeSize = ls_rsp->rsplen;
909	bpl.tus.f.bdeFlags = 0;
910	bpl.tus.w = le32_to_cpu(bpl.tus.w);
911	/*
912	* Note: although we're using stack space for the dmabuf, the
913	* call to lpfc_sli4_issue_wqe is synchronous, so it will not
914	* be referenced after it returns back to this routine.
915	*/
916
917	nvmewqeq->cmd_cmpl = xmt_ls_rsp_cmp;
918	nvmewqeq->context_un.axchg = axchg;
919
920	lpfc_nvmeio_data(phba, "NVMEx LS RSP: xri x%x wqidx x%x len x%x\n",
921	axchg->oxid, nvmewqeq->hba_wqidx, ls_rsp->rsplen);
922
923	rc = lpfc_sli4_issue_wqe(phba, qp: axchg->hdwq, pwqe: nvmewqeq);
924
925	/* clear to be sure there's no reference */
926	nvmewqeq->bpl_dmabuf = NULL;
927
928	if (rc == WQE_SUCCESS) {
929	/*
930	* Okay to repost buffer here, but wait till cmpl
931	* before freeing ctxp and iocbq.
932	*/
933	lpfc_in_buf_free(phba, &nvmebuf->dbuf);
934	return 0;
935	}
936
937	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
938	"6151 NVMEx LS RSP x%x: failed to transmit %d\n",
939	axchg->oxid, rc);
940
941	rc = -ENXIO;
942
943	lpfc_nlp_put(nvmewqeq->ndlp);
944
945	out_free_buf:
946	/* Give back resources */
947	lpfc_in_buf_free(phba, &nvmebuf->dbuf);
948
949	/*
950	* As transport doesn't track completions of responses, if the rsp
951	* fails to send, the transport will effectively ignore the rsp
952	* and consider the LS done. However, the driver has an active
953	* exchange open for the LS - so be sure to abort the exchange
954	* if the response isn't sent.
955	*/
956	lpfc_nvme_unsol_ls_issue_abort(phba, ctxp: axchg, sid: axchg->sid, xri: axchg->oxid);
957	return rc;
958	}
959
960	/**
961	* lpfc_nvmet_xmt_ls_rsp - Transmit NVME LS response
962	* @tgtport: pointer to target port that NVME LS is to be transmit from.
963	* @ls_rsp: pointer to the transport LS RSP that is to be sent
964	*
965	* Driver registers this routine to transmit responses for received NVME
966	* LS requests.
967	*
968	* This routine is used to format and send a WQE to transmit a NVME LS
969	* Response. The ls_rsp is used to reverse-map the LS to the original
970	* NVME LS request sequence, which provides addressing information for
971	* the remote port the LS to be sent to, as well as the exchange id
972	* that is the LS is bound to.
973	*
974	* Returns:
975	* 0 : if response successfully transmit
976	* non-zero : if response failed to transmit, of the form -Exxx.
977	**/
978	static int
979	lpfc_nvmet_xmt_ls_rsp(struct nvmet_fc_target_port *tgtport,
980	struct nvmefc_ls_rsp *ls_rsp)
981	{
982	struct lpfc_async_xchg_ctx *axchg =
983	container_of(ls_rsp, struct lpfc_async_xchg_ctx, ls_rsp);
984	struct lpfc_nvmet_tgtport *nvmep = tgtport->private;
985	int rc;
986
987	if (test_bit(FC_UNLOADING, &axchg->phba->pport->load_flag))
988	return -ENODEV;
989
990	rc = __lpfc_nvme_xmt_ls_rsp(axchg, ls_rsp, xmt_ls_rsp_cmp: lpfc_nvmet_xmt_ls_rsp_cmp);
991
992	if (rc) {
993	atomic_inc(v: &nvmep->xmt_ls_drop);
994	/*
995	* unless the failure is due to having already sent
996	* the response, an abort will be generated for the
997	* exchange if the rsp can't be sent.
998	*/
999	if (rc != -EALREADY)
1000	atomic_inc(v: &nvmep->xmt_ls_abort);
1001	return rc;
1002	}
1003
1004	atomic_inc(v: &nvmep->xmt_ls_rsp);
1005	return 0;
1006	}
1007
1008	static int
1009	lpfc_nvmet_xmt_fcp_op(struct nvmet_fc_target_port *tgtport,
1010	struct nvmefc_tgt_fcp_req *rsp)
1011	{
1012	struct lpfc_nvmet_tgtport *lpfc_nvmep = tgtport->private;
1013	struct lpfc_async_xchg_ctx *ctxp =
1014	container_of(rsp, struct lpfc_async_xchg_ctx, hdlrctx.fcp_req);
1015	struct lpfc_hba *phba = ctxp->phba;
1016	struct lpfc_queue *wq;
1017	struct lpfc_iocbq *nvmewqeq;
1018	struct lpfc_sli_ring *pring;
1019	unsigned long iflags;
1020	int rc;
1021	#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1022	int id;
1023	#endif
1024
1025	if (test_bit(FC_UNLOADING, &phba->pport->load_flag)) {
1026	rc = -ENODEV;
1027	goto aerr;
1028	}
1029
1030	#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1031	if (ctxp->ts_cmd_nvme) {
1032	if (rsp->op == NVMET_FCOP_RSP)
1033	ctxp->ts_nvme_status = ktime_get_ns();
1034	else
1035	ctxp->ts_nvme_data = ktime_get_ns();
1036	}
1037
1038	/* Setup the hdw queue if not already set */
1039	if (!ctxp->hdwq)
1040	ctxp->hdwq = &phba->sli4_hba.hdwq[rsp->hwqid];
1041
1042	if (phba->hdwqstat_on & LPFC_CHECK_NVMET_IO) {
1043	id = raw_smp_processor_id();
1044	this_cpu_inc(phba->sli4_hba.c_stat->xmt_io);
1045	if (rsp->hwqid != id)
1046	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,
1047	"6705 CPU Check OP: "
1048	"cpu %d expect %d\n",
1049	id, rsp->hwqid);
1050	ctxp->cpu = id; /* Setup cpu for cmpl check */
1051	}
1052	#endif
1053
1054	/* Sanity check */
1055	if ((ctxp->flag & LPFC_NVME_ABTS_RCV) ||
1056	(ctxp->state == LPFC_NVME_STE_ABORT)) {
1057	atomic_inc(v: &lpfc_nvmep->xmt_fcp_drop);
1058	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1059	"6102 IO oxid x%x aborted\n",
1060	ctxp->oxid);
1061	rc = -ENXIO;
1062	goto aerr;
1063	}
1064
1065	nvmewqeq = lpfc_nvmet_prep_fcp_wqe(phba, ctxp);
1066	if (nvmewqeq == NULL) {
1067	atomic_inc(v: &lpfc_nvmep->xmt_fcp_drop);
1068	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1069	"6152 FCP Drop IO x%x: Prep\n",
1070	ctxp->oxid);
1071	rc = -ENXIO;
1072	goto aerr;
1073	}
1074
1075	nvmewqeq->cmd_cmpl = lpfc_nvmet_xmt_fcp_op_cmp;
1076	nvmewqeq->context_un.axchg = ctxp;
1077	nvmewqeq->cmd_flag |= LPFC_IO_NVMET;
1078	ctxp->wqeq->hba_wqidx = rsp->hwqid;
1079
1080	lpfc_nvmeio_data(phba, "NVMET FCP CMND: xri x%x op x%x len x%x\n",
1081	ctxp->oxid, rsp->op, rsp->rsplen);
1082
1083	ctxp->flag |= LPFC_NVME_IO_INP;
1084	rc = lpfc_sli4_issue_wqe(phba, qp: ctxp->hdwq, pwqe: nvmewqeq);
1085	if (rc == WQE_SUCCESS) {
1086	#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1087	if (!ctxp->ts_cmd_nvme)
1088	return 0;
1089	if (rsp->op == NVMET_FCOP_RSP)
1090	ctxp->ts_status_wqput = ktime_get_ns();
1091	else
1092	ctxp->ts_data_wqput = ktime_get_ns();
1093	#endif
1094	return 0;
1095	}
1096
1097	if (rc == -EBUSY) {
1098	/*
1099	* WQ was full, so queue nvmewqeq to be sent after
1100	* WQE release CQE
1101	*/
1102	ctxp->flag |= LPFC_NVME_DEFER_WQFULL;
1103	wq = ctxp->hdwq->io_wq;
1104	pring = wq->pring;
1105	spin_lock_irqsave(&pring->ring_lock, iflags);
1106	list_add_tail(new: &nvmewqeq->list, head: &wq->wqfull_list);
1107	wq->q_flag |= HBA_NVMET_WQFULL;
1108	spin_unlock_irqrestore(lock: &pring->ring_lock, flags: iflags);
1109	atomic_inc(v: &lpfc_nvmep->defer_wqfull);
1110	return 0;
1111	}
1112
1113	/* Give back resources */
1114	atomic_inc(v: &lpfc_nvmep->xmt_fcp_drop);
1115	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1116	"6153 FCP Drop IO x%x: Issue: %d\n",
1117	ctxp->oxid, rc);
1118
1119	ctxp->wqeq->hba_wqidx = 0;
1120	nvmewqeq->context_un.axchg = NULL;
1121	nvmewqeq->bpl_dmabuf = NULL;
1122	rc = -EBUSY;
1123	aerr:
1124	return rc;
1125	}
1126
1127	static void
1128	lpfc_nvmet_targetport_delete(struct nvmet_fc_target_port *targetport)
1129	{
1130	struct lpfc_nvmet_tgtport *tport = targetport->private;
1131
1132	/* release any threads waiting for the unreg to complete */
1133	if (tport->phba->targetport)
1134	complete(tport->tport_unreg_cmp);
1135	}
1136
1137	static void
1138	lpfc_nvmet_xmt_fcp_abort(struct nvmet_fc_target_port *tgtport,
1139	struct nvmefc_tgt_fcp_req *req)
1140	{
1141	struct lpfc_nvmet_tgtport *lpfc_nvmep = tgtport->private;
1142	struct lpfc_async_xchg_ctx *ctxp =
1143	container_of(req, struct lpfc_async_xchg_ctx, hdlrctx.fcp_req);
1144	struct lpfc_hba *phba = ctxp->phba;
1145	struct lpfc_queue *wq;
1146	unsigned long flags;
1147
1148	if (test_bit(FC_UNLOADING, &phba->pport->load_flag))
1149	return;
1150
1151	if (!ctxp->hdwq)
1152	ctxp->hdwq = &phba->sli4_hba.hdwq[0];
1153
1154	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
1155	"6103 NVMET Abort op: oxid x%x flg x%x ste %d\n",
1156	ctxp->oxid, ctxp->flag, ctxp->state);
1157
1158	lpfc_nvmeio_data(phba, "NVMET FCP ABRT: xri x%x flg x%x ste x%x\n",
1159	ctxp->oxid, ctxp->flag, ctxp->state);
1160
1161	atomic_inc(v: &lpfc_nvmep->xmt_fcp_abort);
1162
1163	spin_lock_irqsave(&ctxp->ctxlock, flags);
1164
1165	/* Since iaab/iaar are NOT set, we need to check
1166	* if the firmware is in process of aborting IO
1167	*/
1168	if (ctxp->flag & (LPFC_NVME_XBUSY | LPFC_NVME_ABORT_OP)) {
1169	spin_unlock_irqrestore(lock: &ctxp->ctxlock, flags);
1170	return;
1171	}
1172	ctxp->flag |= LPFC_NVME_ABORT_OP;
1173
1174	if (ctxp->flag & LPFC_NVME_DEFER_WQFULL) {
1175	spin_unlock_irqrestore(lock: &ctxp->ctxlock, flags);
1176	lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, ctxp->sid,
1177	ctxp->oxid);
1178	wq = ctxp->hdwq->io_wq;
1179	lpfc_nvmet_wqfull_flush(phba, wq, ctxp);
1180	return;
1181	}
1182	spin_unlock_irqrestore(lock: &ctxp->ctxlock, flags);
1183
1184	/* A state of LPFC_NVME_STE_RCV means we have just received
1185	* the NVME command and have not started processing it.
1186	* (by issuing any IO WQEs on this exchange yet)
1187	*/
1188	if (ctxp->state == LPFC_NVME_STE_RCV)
1189	lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, ctxp->sid,
1190	ctxp->oxid);
1191	else
1192	lpfc_nvmet_sol_fcp_issue_abort(phba, ctxp, ctxp->sid,
1193	ctxp->oxid);
1194	}
1195
1196	static void
1197	lpfc_nvmet_xmt_fcp_release(struct nvmet_fc_target_port *tgtport,
1198	struct nvmefc_tgt_fcp_req *rsp)
1199	{
1200	struct lpfc_nvmet_tgtport *lpfc_nvmep = tgtport->private;
1201	struct lpfc_async_xchg_ctx *ctxp =
1202	container_of(rsp, struct lpfc_async_xchg_ctx, hdlrctx.fcp_req);
1203	struct lpfc_hba *phba = ctxp->phba;
1204	unsigned long flags;
1205	bool aborting = false;
1206
1207	spin_lock_irqsave(&ctxp->ctxlock, flags);
1208	if (ctxp->flag & LPFC_NVME_XBUSY)
1209	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,
1210	"6027 NVMET release with XBUSY flag x%x"
1211	" oxid x%x\n",
1212	ctxp->flag, ctxp->oxid);
1213	else if (ctxp->state != LPFC_NVME_STE_DONE &&
1214	ctxp->state != LPFC_NVME_STE_ABORT)
1215	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1216	"6413 NVMET release bad state %d %d oxid x%x\n",
1217	ctxp->state, ctxp->entry_cnt, ctxp->oxid);
1218
1219	if ((ctxp->flag & LPFC_NVME_ABORT_OP) ||
1220	(ctxp->flag & LPFC_NVME_XBUSY)) {
1221	aborting = true;
1222	/* let the abort path do the real release */
1223	lpfc_nvmet_defer_release(phba, ctxp);
1224	}
1225	spin_unlock_irqrestore(lock: &ctxp->ctxlock, flags);
1226
1227	lpfc_nvmeio_data(phba, "NVMET FCP FREE: xri x%x ste %d abt %d\n", ctxp->oxid,
1228	ctxp->state, aborting);
1229
1230	atomic_inc(v: &lpfc_nvmep->xmt_fcp_release);
1231	ctxp->flag &= ~LPFC_NVME_TNOTIFY;
1232
1233	if (aborting)
1234	return;
1235
1236	lpfc_nvmet_ctxbuf_post(phba, ctx_buf: ctxp->ctxbuf);
1237	}
1238
1239	static void
1240	lpfc_nvmet_defer_rcv(struct nvmet_fc_target_port *tgtport,
1241	struct nvmefc_tgt_fcp_req *rsp)
1242	{
1243	struct lpfc_nvmet_tgtport *tgtp;
1244	struct lpfc_async_xchg_ctx *ctxp =
1245	container_of(rsp, struct lpfc_async_xchg_ctx, hdlrctx.fcp_req);
1246	struct rqb_dmabuf *nvmebuf;
1247	struct lpfc_hba *phba = ctxp->phba;
1248	unsigned long iflag;
1249
1250
1251	lpfc_nvmeio_data(phba, "NVMET DEFERRCV: xri x%x sz %d CPU %02x\n",
1252	ctxp->oxid, ctxp->size, raw_smp_processor_id());
1253
1254	spin_lock_irqsave(&ctxp->ctxlock, iflag);
1255	nvmebuf = ctxp->rqb_buffer;
1256	if (!nvmebuf) {
1257	spin_unlock_irqrestore(lock: &ctxp->ctxlock, flags: iflag);
1258	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,
1259	"6425 Defer rcv: no buffer oxid x%x: "
1260	"flg %x ste %x\n",
1261	ctxp->oxid, ctxp->flag, ctxp->state);
1262	return;
1263	}
1264	ctxp->rqb_buffer = NULL;
1265	spin_unlock_irqrestore(lock: &ctxp->ctxlock, flags: iflag);
1266
1267	tgtp = phba->targetport->private;
1268	if (tgtp)
1269	atomic_inc(v: &tgtp->rcv_fcp_cmd_defer);
1270
1271	/* Free the nvmebuf since a new buffer already replaced it */
1272	nvmebuf->hrq->rqbp->rqb_free_buffer(phba, nvmebuf);
1273	}
1274
1275	/**
1276	* lpfc_nvmet_ls_req_cmp - completion handler for a nvme ls request
1277	* @phba: Pointer to HBA context object
1278	* @cmdwqe: Pointer to driver command WQE object.
1279	* @rspwqe: Pointer to driver response WQE object.
1280	*
1281	* This function is the completion handler for NVME LS requests.
1282	* The function updates any states and statistics, then calls the
1283	* generic completion handler to finish completion of the request.
1284	**/
1285	static void
1286	lpfc_nvmet_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
1287	struct lpfc_iocbq *rspwqe)
1288	{
1289	struct lpfc_wcqe_complete *wcqe = &rspwqe->wcqe_cmpl;
1290	__lpfc_nvme_ls_req_cmp(phba, vport: cmdwqe->vport, cmdwqe, wcqe);
1291	}
1292
1293	/**
1294	* lpfc_nvmet_ls_req - Issue an Link Service request
1295	* @targetport: pointer to target instance registered with nvmet transport.
1296	* @hosthandle: hosthandle set by the driver in a prior ls_rqst_rcv.
1297	* Driver sets this value to the ndlp pointer.
1298	* @pnvme_lsreq: the transport nvme_ls_req structure for the LS
1299	*
1300	* Driver registers this routine to handle any link service request
1301	* from the nvme_fc transport to a remote nvme-aware port.
1302	*
1303	* Return value :
1304	* 0 - Success
1305	* non-zero: various error codes, in form of -Exxx
1306	**/
1307	static int
1308	lpfc_nvmet_ls_req(struct nvmet_fc_target_port *targetport,
1309	void *hosthandle,
1310	struct nvmefc_ls_req *pnvme_lsreq)
1311	{
1312	struct lpfc_nvmet_tgtport *lpfc_nvmet = targetport->private;
1313	struct lpfc_hba *phba;
1314	struct lpfc_nodelist *ndlp;
1315	int ret;
1316	u32 hstate;
1317
1318	if (!lpfc_nvmet)
1319	return -EINVAL;
1320
1321	phba = lpfc_nvmet->phba;
1322	if (test_bit(FC_UNLOADING, &phba->pport->load_flag))
1323	return -EINVAL;
1324
1325	hstate = atomic_read(v: &lpfc_nvmet->state);
1326	if (hstate == LPFC_NVMET_INV_HOST_ACTIVE)
1327	return -EACCES;
1328
1329	ndlp = (struct lpfc_nodelist *)hosthandle;
1330
1331	ret = __lpfc_nvme_ls_req(vport: phba->pport, ndlp, pnvme_lsreq,
1332	gen_req_cmp: lpfc_nvmet_ls_req_cmp);
1333
1334	return ret;
1335	}
1336
1337	/**
1338	* lpfc_nvmet_ls_abort - Abort a prior NVME LS request
1339	* @targetport: Transport targetport, that LS was issued from.
1340	* @hosthandle: hosthandle set by the driver in a prior ls_rqst_rcv.
1341	* Driver sets this value to the ndlp pointer.
1342	* @pnvme_lsreq: the transport nvme_ls_req structure for LS to be aborted
1343	*
1344	* Driver registers this routine to abort an NVME LS request that is
1345	* in progress (from the transports perspective).
1346	**/
1347	static void
1348	lpfc_nvmet_ls_abort(struct nvmet_fc_target_port *targetport,
1349	void *hosthandle,
1350	struct nvmefc_ls_req *pnvme_lsreq)
1351	{
1352	struct lpfc_nvmet_tgtport *lpfc_nvmet = targetport->private;
1353	struct lpfc_hba *phba;
1354	struct lpfc_nodelist *ndlp;
1355	int ret;
1356
1357	phba = lpfc_nvmet->phba;
1358	if (test_bit(FC_UNLOADING, &phba->pport->load_flag))
1359	return;
1360
1361	ndlp = (struct lpfc_nodelist *)hosthandle;
1362
1363	ret = __lpfc_nvme_ls_abort(vport: phba->pport, ndlp, pnvme_lsreq);
1364	if (!ret)
1365	atomic_inc(v: &lpfc_nvmet->xmt_ls_abort);
1366	}
1367
1368	static int
1369	lpfc_nvmet_host_traddr(void *hosthandle, u64 *wwnn, u64 *wwpn)
1370	{
1371	struct lpfc_nodelist *ndlp = hosthandle;
1372
1373	*wwnn = wwn_to_u64(wwn: ndlp->nlp_nodename.u.wwn);
1374	*wwpn = wwn_to_u64(wwn: ndlp->nlp_portname.u.wwn);
1375	return 0;
1376	}
1377
1378	static void
1379	lpfc_nvmet_host_release(void *hosthandle)
1380	{
1381	struct lpfc_nodelist *ndlp = hosthandle;
1382	struct lpfc_hba *phba = ndlp->phba;
1383	struct lpfc_nvmet_tgtport *tgtp;
1384
1385	if (!phba->targetport || !phba->targetport->private)
1386	return;
1387
1388	lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
1389	"6202 NVMET XPT releasing hosthandle x%px "
1390	"DID x%x xflags x%x refcnt %d\n",
1391	hosthandle, ndlp->nlp_DID, ndlp->fc4_xpt_flags,
1392	kref_read(&ndlp->kref));
1393	tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
1394	spin_lock_irq(lock: &ndlp->lock);
1395	ndlp->fc4_xpt_flags &= ~NLP_XPT_HAS_HH;
1396	spin_unlock_irq(lock: &ndlp->lock);
1397	lpfc_nlp_put(ndlp);
1398	atomic_set(v: &tgtp->state, i: 0);
1399	}
1400
1401	static void
1402	lpfc_nvmet_discovery_event(struct nvmet_fc_target_port *tgtport)
1403	{
1404	struct lpfc_nvmet_tgtport *tgtp;
1405	struct lpfc_hba *phba;
1406	uint32_t rc;
1407
1408	tgtp = tgtport->private;
1409	phba = tgtp->phba;
1410
1411	rc = lpfc_issue_els_rscn(vport: phba->pport, retry: 0);
1412	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1413	"6420 NVMET subsystem change: Notification %s\n",
1414	(rc) ? "Failed" : "Sent");
1415	}
1416
1417	static struct nvmet_fc_target_template lpfc_tgttemplate = {
1418	.targetport_delete = lpfc_nvmet_targetport_delete,
1419	.xmt_ls_rsp = lpfc_nvmet_xmt_ls_rsp,
1420	.fcp_op = lpfc_nvmet_xmt_fcp_op,
1421	.fcp_abort = lpfc_nvmet_xmt_fcp_abort,
1422	.fcp_req_release = lpfc_nvmet_xmt_fcp_release,
1423	.defer_rcv = lpfc_nvmet_defer_rcv,
1424	.discovery_event = lpfc_nvmet_discovery_event,
1425	.ls_req = lpfc_nvmet_ls_req,
1426	.ls_abort = lpfc_nvmet_ls_abort,
1427	.host_release = lpfc_nvmet_host_release,
1428	.host_traddr = lpfc_nvmet_host_traddr,
1429
1430	.max_hw_queues = 1,
1431	.max_sgl_segments = LPFC_NVMET_DEFAULT_SEGS,
1432	.max_dif_sgl_segments = LPFC_NVMET_DEFAULT_SEGS,
1433	.dma_boundary = 0xFFFFFFFF,
1434
1435	/* optional features */
1436	.target_features = 0,
1437	/* sizes of additional private data for data structures */
1438	.target_priv_sz = sizeof(struct lpfc_nvmet_tgtport),
1439	.lsrqst_priv_sz = 0,
1440	};
1441
1442	static void
1443	__lpfc_nvmet_clean_io_for_cpu(struct lpfc_hba *phba,
1444	struct lpfc_nvmet_ctx_info *infop)
1445	{
1446	struct lpfc_nvmet_ctxbuf *ctx_buf, *next_ctx_buf;
1447	unsigned long flags;
1448
1449	spin_lock_irqsave(&infop->nvmet_ctx_list_lock, flags);
1450	list_for_each_entry_safe(ctx_buf, next_ctx_buf,
1451	&infop->nvmet_ctx_list, list) {
1452	spin_lock(lock: &phba->sli4_hba.abts_nvmet_buf_list_lock);
1453	list_del_init(entry: &ctx_buf->list);
1454	spin_unlock(lock: &phba->sli4_hba.abts_nvmet_buf_list_lock);
1455
1456	spin_lock(lock: &phba->hbalock);
1457	__lpfc_clear_active_sglq(phba, xri: ctx_buf->sglq->sli4_lxritag);
1458	spin_unlock(lock: &phba->hbalock);
1459
1460	ctx_buf->sglq->state = SGL_FREED;
1461	ctx_buf->sglq->ndlp = NULL;
1462
1463	spin_lock(lock: &phba->sli4_hba.sgl_list_lock);
1464	list_add_tail(new: &ctx_buf->sglq->list,
1465	head: &phba->sli4_hba.lpfc_nvmet_sgl_list);
1466	spin_unlock(lock: &phba->sli4_hba.sgl_list_lock);
1467
1468	lpfc_sli_release_iocbq(phba, ctx_buf->iocbq);
1469	kfree(objp: ctx_buf->context);
1470	}
1471	spin_unlock_irqrestore(lock: &infop->nvmet_ctx_list_lock, flags);
1472	}
1473
1474	static void
1475	lpfc_nvmet_cleanup_io_context(struct lpfc_hba *phba)
1476	{
1477	struct lpfc_nvmet_ctx_info *infop;
1478	int i, j;
1479
1480	/* The first context list, MRQ 0 CPU 0 */
1481	infop = phba->sli4_hba.nvmet_ctx_info;
1482	if (!infop)
1483	return;
1484
1485	/* Cycle the entire CPU context list for every MRQ */
1486	for (i = 0; i < phba->cfg_nvmet_mrq; i++) {
1487	for_each_present_cpu(j) {
1488	infop = lpfc_get_ctx_list(phba, j, i);
1489	__lpfc_nvmet_clean_io_for_cpu(phba, infop);
1490	}
1491	}
1492	kfree(objp: phba->sli4_hba.nvmet_ctx_info);
1493	phba->sli4_hba.nvmet_ctx_info = NULL;
1494	}
1495
1496	static int
1497	lpfc_nvmet_setup_io_context(struct lpfc_hba *phba)
1498	{
1499	struct lpfc_nvmet_ctxbuf *ctx_buf;
1500	struct lpfc_iocbq *nvmewqe;
1501	union lpfc_wqe128 *wqe;
1502	struct lpfc_nvmet_ctx_info *last_infop;
1503	struct lpfc_nvmet_ctx_info *infop;
1504	int i, j, idx, cpu;
1505
1506	lpfc_printf_log(phba, KERN_INFO, LOG_NVME,
1507	"6403 Allocate NVMET resources for %d XRIs\n",
1508	phba->sli4_hba.nvmet_xri_cnt);
1509
1510	phba->sli4_hba.nvmet_ctx_info = kcalloc(
1511	phba->sli4_hba.num_possible_cpu * phba->cfg_nvmet_mrq,
1512	sizeof(struct lpfc_nvmet_ctx_info), GFP_KERNEL);
1513	if (!phba->sli4_hba.nvmet_ctx_info) {
1514	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1515	"6419 Failed allocate memory for "
1516	"nvmet context lists\n");
1517	return -ENOMEM;
1518	}
1519
1520	/*
1521	* Assuming X CPUs in the system, and Y MRQs, allocate some
1522	* lpfc_nvmet_ctx_info structures as follows:
1523	*
1524	* cpu0/mrq0 cpu1/mrq0 ... cpuX/mrq0
1525	* cpu0/mrq1 cpu1/mrq1 ... cpuX/mrq1
1526	* ...
1527	* cpuX/mrqY cpuX/mrqY ... cpuX/mrqY
1528	*
1529	* Each line represents a MRQ "silo" containing an entry for
1530	* every CPU.
1531	*
1532	* MRQ X is initially assumed to be associated with CPU X, thus
1533	* contexts are initially distributed across all MRQs using
1534	* the MRQ index (N) as follows cpuN/mrqN. When contexts are
1535	* freed, the are freed to the MRQ silo based on the CPU number
1536	* of the IO completion. Thus a context that was allocated for MRQ A
1537	* whose IO completed on CPU B will be freed to cpuB/mrqA.
1538	*/
1539	for_each_possible_cpu(i) {
1540	for (j = 0; j < phba->cfg_nvmet_mrq; j++) {
1541	infop = lpfc_get_ctx_list(phba, i, j);
1542	INIT_LIST_HEAD(list: &infop->nvmet_ctx_list);
1543	spin_lock_init(&infop->nvmet_ctx_list_lock);
1544	infop->nvmet_ctx_list_cnt = 0;
1545	}
1546	}
1547
1548	/*
1549	* Setup the next CPU context info ptr for each MRQ.
1550	* MRQ 0 will cycle thru CPUs 0 - X separately from
1551	* MRQ 1 cycling thru CPUs 0 - X, and so on.
1552	*/
1553	for (j = 0; j < phba->cfg_nvmet_mrq; j++) {
1554	last_infop = lpfc_get_ctx_list(phba,
1555	cpumask_first(cpu_present_mask),
1556	j);
1557	for (i = phba->sli4_hba.num_possible_cpu - 1; i >= 0; i--) {
1558	infop = lpfc_get_ctx_list(phba, i, j);
1559	infop->nvmet_ctx_next_cpu = last_infop;
1560	last_infop = infop;
1561	}
1562	}
1563
1564	/* For all nvmet xris, allocate resources needed to process a
1565	* received command on a per xri basis.
1566	*/
1567	idx = 0;
1568	cpu = cpumask_first(cpu_present_mask);
1569	for (i = 0; i < phba->sli4_hba.nvmet_xri_cnt; i++) {
1570	ctx_buf = kzalloc(sizeof(*ctx_buf), GFP_KERNEL);
1571	if (!ctx_buf) {
1572	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1573	"6404 Ran out of memory for NVMET\n");
1574	return -ENOMEM;
1575	}
1576
1577	ctx_buf->context = kzalloc(sizeof(*ctx_buf->context),
1578	GFP_KERNEL);
1579	if (!ctx_buf->context) {
1580	kfree(objp: ctx_buf);
1581	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1582	"6405 Ran out of NVMET "
1583	"context memory\n");
1584	return -ENOMEM;
1585	}
1586	ctx_buf->context->ctxbuf = ctx_buf;
1587	ctx_buf->context->state = LPFC_NVME_STE_FREE;
1588
1589	ctx_buf->iocbq = lpfc_sli_get_iocbq(phba);
1590	if (!ctx_buf->iocbq) {
1591	kfree(objp: ctx_buf->context);
1592	kfree(objp: ctx_buf);
1593	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1594	"6406 Ran out of NVMET iocb/WQEs\n");
1595	return -ENOMEM;
1596	}
1597	ctx_buf->iocbq->cmd_flag = LPFC_IO_NVMET;
1598	nvmewqe = ctx_buf->iocbq;
1599	wqe = &nvmewqe->wqe;
1600
1601	/* Initialize WQE */
1602	memset(wqe, 0, sizeof(*wqe));
1603
1604	ctx_buf->iocbq->cmd_dmabuf = NULL;
1605	spin_lock(lock: &phba->sli4_hba.sgl_list_lock);
1606	ctx_buf->sglq = __lpfc_sli_get_nvmet_sglq(phba, piocbq: ctx_buf->iocbq);
1607	spin_unlock(lock: &phba->sli4_hba.sgl_list_lock);
1608	if (!ctx_buf->sglq) {
1609	lpfc_sli_release_iocbq(phba, ctx_buf->iocbq);
1610	kfree(objp: ctx_buf->context);
1611	kfree(objp: ctx_buf);
1612	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1613	"6407 Ran out of NVMET XRIs\n");
1614	return -ENOMEM;
1615	}
1616	INIT_WORK(&ctx_buf->defer_work, lpfc_nvmet_fcp_rqst_defer_work);
1617
1618	/*
1619	* Add ctx to MRQidx context list. Our initial assumption
1620	* is MRQidx will be associated with CPUidx. This association
1621	* can change on the fly.
1622	*/
1623	infop = lpfc_get_ctx_list(phba, cpu, idx);
1624	spin_lock(lock: &infop->nvmet_ctx_list_lock);
1625	list_add_tail(new: &ctx_buf->list, head: &infop->nvmet_ctx_list);
1626	infop->nvmet_ctx_list_cnt++;
1627	spin_unlock(lock: &infop->nvmet_ctx_list_lock);
1628
1629	/* Spread ctx structures evenly across all MRQs */
1630	idx++;
1631	if (idx >= phba->cfg_nvmet_mrq) {
1632	idx = 0;
1633	cpu = cpumask_first(cpu_present_mask);
1634	continue;
1635	}
1636	cpu = lpfc_next_present_cpu(n: cpu);
1637	}
1638
1639	for_each_present_cpu(i) {
1640	for (j = 0; j < phba->cfg_nvmet_mrq; j++) {
1641	infop = lpfc_get_ctx_list(phba, i, j);
1642	lpfc_printf_log(phba, KERN_INFO, LOG_NVME | LOG_INIT,
1643	"6408 TOTAL NVMET ctx for CPU %d "
1644	"MRQ %d: cnt %d nextcpu x%px\n",
1645	i, j, infop->nvmet_ctx_list_cnt,
1646	infop->nvmet_ctx_next_cpu);
1647	}
1648	}
1649	return 0;
1650	}
1651
1652	int
1653	lpfc_nvmet_create_targetport(struct lpfc_hba *phba)
1654	{
1655	struct lpfc_vport *vport = phba->pport;
1656	struct lpfc_nvmet_tgtport *tgtp;
1657	struct nvmet_fc_port_info pinfo;
1658	int error;
1659
1660	if (phba->targetport)
1661	return 0;
1662
1663	error = lpfc_nvmet_setup_io_context(phba);
1664	if (error)
1665	return error;
1666
1667	memset(&pinfo, 0, sizeof(struct nvmet_fc_port_info));
1668	pinfo.node_name = wwn_to_u64(wwn: vport->fc_nodename.u.wwn);
1669	pinfo.port_name = wwn_to_u64(wwn: vport->fc_portname.u.wwn);
1670	pinfo.port_id = vport->fc_myDID;
1671
1672	/* We need to tell the transport layer + 1 because it takes page
1673	* alignment into account. When space for the SGL is allocated we
1674	* allocate + 3, one for cmd, one for rsp and one for this alignment
1675	*/
1676	lpfc_tgttemplate.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1;
1677	lpfc_tgttemplate.max_hw_queues = phba->cfg_hdw_queue;
1678	lpfc_tgttemplate.target_features = NVMET_FCTGTFEAT_READDATA_RSP;
1679
1680	#if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
1681	error = nvmet_fc_register_targetport(portinfo: &pinfo, template: &lpfc_tgttemplate,
1682	dev: &phba->pcidev->dev,
1683	tgtport_p: &phba->targetport);
1684	#else
1685	error = -ENOENT;
1686	#endif
1687	if (error) {
1688	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1689	"6025 Cannot register NVME targetport x%x: "
1690	"portnm %llx nodenm %llx segs %d qs %d\n",
1691	error,
1692	pinfo.port_name, pinfo.node_name,
1693	lpfc_tgttemplate.max_sgl_segments,
1694	lpfc_tgttemplate.max_hw_queues);
1695	phba->targetport = NULL;
1696	phba->nvmet_support = 0;
1697
1698	lpfc_nvmet_cleanup_io_context(phba);
1699
1700	} else {
1701	tgtp = (struct lpfc_nvmet_tgtport *)
1702	phba->targetport->private;
1703	tgtp->phba = phba;
1704
1705	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
1706	"6026 Registered NVME "
1707	"targetport: x%px, private x%px "
1708	"portnm %llx nodenm %llx segs %d qs %d\n",
1709	phba->targetport, tgtp,
1710	pinfo.port_name, pinfo.node_name,
1711	lpfc_tgttemplate.max_sgl_segments,
1712	lpfc_tgttemplate.max_hw_queues);
1713
1714	atomic_set(v: &tgtp->rcv_ls_req_in, i: 0);
1715	atomic_set(v: &tgtp->rcv_ls_req_out, i: 0);
1716	atomic_set(v: &tgtp->rcv_ls_req_drop, i: 0);
1717	atomic_set(v: &tgtp->xmt_ls_abort, i: 0);
1718	atomic_set(v: &tgtp->xmt_ls_abort_cmpl, i: 0);
1719	atomic_set(v: &tgtp->xmt_ls_rsp, i: 0);
1720	atomic_set(v: &tgtp->xmt_ls_drop, i: 0);
1721	atomic_set(v: &tgtp->xmt_ls_rsp_error, i: 0);
1722	atomic_set(v: &tgtp->xmt_ls_rsp_xb_set, i: 0);
1723	atomic_set(v: &tgtp->xmt_ls_rsp_aborted, i: 0);
1724	atomic_set(v: &tgtp->xmt_ls_rsp_cmpl, i: 0);
1725	atomic_set(v: &tgtp->rcv_fcp_cmd_in, i: 0);
1726	atomic_set(v: &tgtp->rcv_fcp_cmd_out, i: 0);
1727	atomic_set(v: &tgtp->rcv_fcp_cmd_drop, i: 0);
1728	atomic_set(v: &tgtp->xmt_fcp_drop, i: 0);
1729	atomic_set(v: &tgtp->xmt_fcp_read_rsp, i: 0);
1730	atomic_set(v: &tgtp->xmt_fcp_read, i: 0);
1731	atomic_set(v: &tgtp->xmt_fcp_write, i: 0);
1732	atomic_set(v: &tgtp->xmt_fcp_rsp, i: 0);
1733	atomic_set(v: &tgtp->xmt_fcp_release, i: 0);
1734	atomic_set(v: &tgtp->xmt_fcp_rsp_cmpl, i: 0);
1735	atomic_set(v: &tgtp->xmt_fcp_rsp_error, i: 0);
1736	atomic_set(v: &tgtp->xmt_fcp_rsp_xb_set, i: 0);
1737	atomic_set(v: &tgtp->xmt_fcp_rsp_aborted, i: 0);
1738	atomic_set(v: &tgtp->xmt_fcp_rsp_drop, i: 0);
1739	atomic_set(v: &tgtp->xmt_fcp_xri_abort_cqe, i: 0);
1740	atomic_set(v: &tgtp->xmt_fcp_abort, i: 0);
1741	atomic_set(v: &tgtp->xmt_fcp_abort_cmpl, i: 0);
1742	atomic_set(v: &tgtp->xmt_abort_unsol, i: 0);
1743	atomic_set(v: &tgtp->xmt_abort_sol, i: 0);
1744	atomic_set(v: &tgtp->xmt_abort_rsp, i: 0);
1745	atomic_set(v: &tgtp->xmt_abort_rsp_error, i: 0);
1746	atomic_set(v: &tgtp->defer_ctx, i: 0);
1747	atomic_set(v: &tgtp->defer_fod, i: 0);
1748	atomic_set(v: &tgtp->defer_wqfull, i: 0);
1749	}
1750	return error;
1751	}
1752
1753	int
1754	lpfc_nvmet_update_targetport(struct lpfc_hba *phba)
1755	{
1756	struct lpfc_vport *vport = phba->pport;
1757
1758	if (!phba->targetport)
1759	return 0;
1760
1761	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
1762	"6007 Update NVMET port x%px did x%x\n",
1763	phba->targetport, vport->fc_myDID);
1764
1765	phba->targetport->port_id = vport->fc_myDID;
1766	return 0;
1767	}
1768
1769	/**
1770	* lpfc_sli4_nvmet_xri_aborted - Fast-path process of nvmet xri abort
1771	* @phba: pointer to lpfc hba data structure.
1772	* @axri: pointer to the nvmet xri abort wcqe structure.
1773	*
1774	* This routine is invoked by the worker thread to process a SLI4 fast-path
1775	* NVMET aborted xri.
1776	**/
1777	void
1778	lpfc_sli4_nvmet_xri_aborted(struct lpfc_hba *phba,
1779	struct sli4_wcqe_xri_aborted *axri)
1780	{
1781	#if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
1782	uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri);
1783	uint16_t rxid = bf_get(lpfc_wcqe_xa_remote_xid, axri);
1784	struct lpfc_async_xchg_ctx *ctxp, *next_ctxp;
1785	struct lpfc_nvmet_tgtport *tgtp;
1786	struct nvmefc_tgt_fcp_req *req = NULL;
1787	struct lpfc_nodelist *ndlp;
1788	unsigned long iflag = 0;
1789	int rrq_empty = 0;
1790	bool released = false;
1791
1792	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
1793	"6317 XB aborted xri x%x rxid x%x\n", xri, rxid);
1794
1795	if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME))
1796	return;
1797
1798	if (phba->targetport) {
1799	tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
1800	atomic_inc(v: &tgtp->xmt_fcp_xri_abort_cqe);
1801	}
1802
1803	spin_lock_irqsave(&phba->sli4_hba.abts_nvmet_buf_list_lock, iflag);
1804	list_for_each_entry_safe(ctxp, next_ctxp,
1805	&phba->sli4_hba.lpfc_abts_nvmet_ctx_list,
1806	list) {
1807	if (ctxp->ctxbuf->sglq->sli4_xritag != xri)
1808	continue;
1809
1810	spin_unlock_irqrestore(lock: &phba->sli4_hba.abts_nvmet_buf_list_lock,
1811	flags: iflag);
1812
1813	spin_lock_irqsave(&ctxp->ctxlock, iflag);
1814	/* Check if we already received a free context call
1815	* and we have completed processing an abort situation.
1816	*/
1817	if (ctxp->flag & LPFC_NVME_CTX_RLS &&
1818	!(ctxp->flag & LPFC_NVME_ABORT_OP)) {
1819	spin_lock(lock: &phba->sli4_hba.abts_nvmet_buf_list_lock);
1820	list_del_init(entry: &ctxp->list);
1821	spin_unlock(lock: &phba->sli4_hba.abts_nvmet_buf_list_lock);
1822	released = true;
1823	}
1824	ctxp->flag &= ~LPFC_NVME_XBUSY;
1825	spin_unlock_irqrestore(lock: &ctxp->ctxlock, flags: iflag);
1826
1827	spin_lock_irqsave(&phba->rrq_list_lock, iflag);
1828	rrq_empty = list_empty(head: &phba->active_rrq_list);
1829	spin_unlock_irqrestore(lock: &phba->rrq_list_lock, flags: iflag);
1830	ndlp = lpfc_findnode_did(phba->pport, ctxp->sid);
1831	if (ndlp &&
1832	(ndlp->nlp_state == NLP_STE_UNMAPPED_NODE ||
1833	ndlp->nlp_state == NLP_STE_MAPPED_NODE)) {
1834	lpfc_set_rrq_active(phba, ndlp,
1835	ctxp->ctxbuf->sglq->sli4_lxritag,
1836	rxid, 1);
1837	lpfc_sli4_abts_err_handler(phba, ndlp, axri);
1838	}
1839
1840	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
1841	"6318 XB aborted oxid x%x flg x%x (%x)\n",
1842	ctxp->oxid, ctxp->flag, released);
1843	if (released)
1844	lpfc_nvmet_ctxbuf_post(phba, ctx_buf: ctxp->ctxbuf);
1845
1846	if (rrq_empty)
1847	lpfc_worker_wake_up(phba);
1848	return;
1849	}
1850	spin_unlock_irqrestore(lock: &phba->sli4_hba.abts_nvmet_buf_list_lock, flags: iflag);
1851	ctxp = lpfc_nvmet_get_ctx_for_xri(phba, xri);
1852	if (ctxp) {
1853	/*
1854	* Abort already done by FW, so BA_ACC sent.
1855	* However, the transport may be unaware.
1856	*/
1857	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
1858	"6323 NVMET Rcv ABTS xri x%x ctxp state x%x "
1859	"flag x%x oxid x%x rxid x%x\n",
1860	xri, ctxp->state, ctxp->flag, ctxp->oxid,
1861	rxid);
1862
1863	spin_lock_irqsave(&ctxp->ctxlock, iflag);
1864	ctxp->flag |= LPFC_NVME_ABTS_RCV;
1865	ctxp->state = LPFC_NVME_STE_ABORT;
1866	spin_unlock_irqrestore(lock: &ctxp->ctxlock, flags: iflag);
1867
1868	lpfc_nvmeio_data(phba,
1869	"NVMET ABTS RCV: xri x%x CPU %02x rjt %d\n",
1870	xri, raw_smp_processor_id(), 0);
1871
1872	req = &ctxp->hdlrctx.fcp_req;
1873	if (req)
1874	nvmet_fc_rcv_fcp_abort(tgtport: phba->targetport, fcpreq: req);
1875	}
1876	#endif
1877	}
1878
1879	int
1880	lpfc_nvmet_rcv_unsol_abort(struct lpfc_vport *vport,
1881	struct fc_frame_header *fc_hdr)
1882	{
1883	#if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
1884	struct lpfc_hba *phba = vport->phba;
1885	struct lpfc_async_xchg_ctx *ctxp, *next_ctxp;
1886	struct nvmefc_tgt_fcp_req *rsp;
1887	uint32_t sid;
1888	uint16_t oxid, xri;
1889	unsigned long iflag = 0;
1890
1891	sid = sli4_sid_from_fc_hdr(fc_hdr);
1892	oxid = be16_to_cpu(fc_hdr->fh_ox_id);
1893
1894	spin_lock_irqsave(&phba->sli4_hba.abts_nvmet_buf_list_lock, iflag);
1895	list_for_each_entry_safe(ctxp, next_ctxp,
1896	&phba->sli4_hba.lpfc_abts_nvmet_ctx_list,
1897	list) {
1898	if (ctxp->oxid != oxid || ctxp->sid != sid)
1899	continue;
1900
1901	xri = ctxp->ctxbuf->sglq->sli4_xritag;
1902
1903	spin_unlock_irqrestore(lock: &phba->sli4_hba.abts_nvmet_buf_list_lock,
1904	flags: iflag);
1905	spin_lock_irqsave(&ctxp->ctxlock, iflag);
1906	ctxp->flag |= LPFC_NVME_ABTS_RCV;
1907	spin_unlock_irqrestore(lock: &ctxp->ctxlock, flags: iflag);
1908
1909	lpfc_nvmeio_data(phba,
1910	"NVMET ABTS RCV: xri x%x CPU %02x rjt %d\n",
1911	xri, raw_smp_processor_id(), 0);
1912
1913	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
1914	"6319 NVMET Rcv ABTS:acc xri x%x\n", xri);
1915
1916	rsp = &ctxp->hdlrctx.fcp_req;
1917	nvmet_fc_rcv_fcp_abort(tgtport: phba->targetport, fcpreq: rsp);
1918
1919	/* Respond with BA_ACC accordingly */
1920	lpfc_sli4_seq_abort_rsp(vport, fc_hdr, aborted: 1);
1921	return 0;
1922	}
1923	spin_unlock_irqrestore(lock: &phba->sli4_hba.abts_nvmet_buf_list_lock, flags: iflag);
1924	/* check the wait list */
1925	if (phba->sli4_hba.nvmet_io_wait_cnt) {
1926	struct rqb_dmabuf *nvmebuf;
1927	struct fc_frame_header *fc_hdr_tmp;
1928	u32 sid_tmp;
1929	u16 oxid_tmp;
1930	bool found = false;
1931
1932	spin_lock_irqsave(&phba->sli4_hba.nvmet_io_wait_lock, iflag);
1933
1934	/* match by oxid and s_id */
1935	list_for_each_entry(nvmebuf,
1936	&phba->sli4_hba.lpfc_nvmet_io_wait_list,
1937	hbuf.list) {
1938	fc_hdr_tmp = (struct fc_frame_header *)
1939	(nvmebuf->hbuf.virt);
1940	oxid_tmp = be16_to_cpu(fc_hdr_tmp->fh_ox_id);
1941	sid_tmp = sli4_sid_from_fc_hdr(fc_hdr_tmp);
1942	if (oxid_tmp != oxid || sid_tmp != sid)
1943	continue;
1944
1945	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
1946	"6321 NVMET Rcv ABTS oxid x%x from x%x "
1947	"is waiting for a ctxp\n",
1948	oxid, sid);
1949
1950	list_del_init(entry: &nvmebuf->hbuf.list);
1951	phba->sli4_hba.nvmet_io_wait_cnt--;
1952	found = true;
1953	break;
1954	}
1955	spin_unlock_irqrestore(lock: &phba->sli4_hba.nvmet_io_wait_lock,
1956	flags: iflag);
1957
1958	/* free buffer since already posted a new DMA buffer to RQ */
1959	if (found) {
1960	nvmebuf->hrq->rqbp->rqb_free_buffer(phba, nvmebuf);
1961	/* Respond with BA_ACC accordingly */
1962	lpfc_sli4_seq_abort_rsp(vport, fc_hdr, aborted: 1);
1963	return 0;
1964	}
1965	}
1966
1967	/* check active list */
1968	ctxp = lpfc_nvmet_get_ctx_for_oxid(phba, oxid, sid);
1969	if (ctxp) {
1970	xri = ctxp->ctxbuf->sglq->sli4_xritag;
1971
1972	spin_lock_irqsave(&ctxp->ctxlock, iflag);
1973	ctxp->flag |= (LPFC_NVME_ABTS_RCV | LPFC_NVME_ABORT_OP);
1974	spin_unlock_irqrestore(lock: &ctxp->ctxlock, flags: iflag);
1975
1976	lpfc_nvmeio_data(phba,
1977	"NVMET ABTS RCV: xri x%x CPU %02x rjt %d\n",
1978	xri, raw_smp_processor_id(), 0);
1979
1980	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
1981	"6322 NVMET Rcv ABTS:acc oxid x%x xri x%x "
1982	"flag x%x state x%x\n",
1983	ctxp->oxid, xri, ctxp->flag, ctxp->state);
1984
1985	if (ctxp->flag & LPFC_NVME_TNOTIFY) {
1986	/* Notify the transport */
1987	nvmet_fc_rcv_fcp_abort(tgtport: phba->targetport,
1988	fcpreq: &ctxp->hdlrctx.fcp_req);
1989	} else {
1990	cancel_work_sync(work: &ctxp->ctxbuf->defer_work);
1991	spin_lock_irqsave(&ctxp->ctxlock, iflag);
1992	lpfc_nvmet_defer_release(phba, ctxp);
1993	spin_unlock_irqrestore(lock: &ctxp->ctxlock, flags: iflag);
1994	}
1995	lpfc_nvmet_sol_fcp_issue_abort(phba, ctxp, ctxp->sid,
1996	ctxp->oxid);
1997
1998	lpfc_sli4_seq_abort_rsp(vport, fc_hdr, aborted: 1);
1999	return 0;
2000	}
2001
2002	lpfc_nvmeio_data(phba, "NVMET ABTS RCV: oxid x%x CPU %02x rjt %d\n",
2003	oxid, raw_smp_processor_id(), 1);
2004
2005	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2006	"6320 NVMET Rcv ABTS:rjt oxid x%x\n", oxid);
2007
2008	/* Respond with BA_RJT accordingly */
2009	lpfc_sli4_seq_abort_rsp(vport, fc_hdr, aborted: 0);
2010	#endif
2011	return 0;
2012	}
2013
2014	static void
2015	lpfc_nvmet_wqfull_flush(struct lpfc_hba *phba, struct lpfc_queue *wq,
2016	struct lpfc_async_xchg_ctx *ctxp)
2017	{
2018	struct lpfc_sli_ring *pring;
2019	struct lpfc_iocbq *nvmewqeq;
2020	struct lpfc_iocbq *next_nvmewqeq;
2021	unsigned long iflags;
2022	struct lpfc_wcqe_complete wcqe;
2023	struct lpfc_wcqe_complete *wcqep;
2024
2025	pring = wq->pring;
2026	wcqep = &wcqe;
2027
2028	/* Fake an ABORT error code back to cmpl routine */
2029	memset(wcqep, 0, sizeof(struct lpfc_wcqe_complete));
2030	bf_set(lpfc_wcqe_c_status, wcqep, IOSTAT_LOCAL_REJECT);
2031	wcqep->parameter = IOERR_ABORT_REQUESTED;
2032
2033	spin_lock_irqsave(&pring->ring_lock, iflags);
2034	list_for_each_entry_safe(nvmewqeq, next_nvmewqeq,
2035	&wq->wqfull_list, list) {
2036	if (ctxp) {
2037	/* Checking for a specific IO to flush */
2038	if (nvmewqeq->context_un.axchg == ctxp) {
2039	list_del(entry: &nvmewqeq->list);
2040	spin_unlock_irqrestore(lock: &pring->ring_lock,
2041	flags: iflags);
2042	memcpy(&nvmewqeq->wcqe_cmpl, wcqep,
2043	sizeof(*wcqep));
2044	lpfc_nvmet_xmt_fcp_op_cmp(phba, cmdwqe: nvmewqeq,
2045	rspwqe: nvmewqeq);
2046	return;
2047	}
2048	continue;
2049	} else {
2050	/* Flush all IOs */
2051	list_del(entry: &nvmewqeq->list);
2052	spin_unlock_irqrestore(lock: &pring->ring_lock, flags: iflags);
2053	memcpy(&nvmewqeq->wcqe_cmpl, wcqep, sizeof(*wcqep));
2054	lpfc_nvmet_xmt_fcp_op_cmp(phba, cmdwqe: nvmewqeq, rspwqe: nvmewqeq);
2055	spin_lock_irqsave(&pring->ring_lock, iflags);
2056	}
2057	}
2058	if (!ctxp)
2059	wq->q_flag &= ~HBA_NVMET_WQFULL;
2060	spin_unlock_irqrestore(lock: &pring->ring_lock, flags: iflags);
2061	}
2062
2063	void
2064	lpfc_nvmet_wqfull_process(struct lpfc_hba *phba,
2065	struct lpfc_queue *wq)
2066	{
2067	#if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
2068	struct lpfc_sli_ring *pring;
2069	struct lpfc_iocbq *nvmewqeq;
2070	struct lpfc_async_xchg_ctx *ctxp;
2071	unsigned long iflags;
2072	int rc;
2073
2074	/*
2075	* Some WQE slots are available, so try to re-issue anything
2076	* on the WQ wqfull_list.
2077	*/
2078	pring = wq->pring;
2079	spin_lock_irqsave(&pring->ring_lock, iflags);
2080	while (!list_empty(head: &wq->wqfull_list)) {
2081	list_remove_head(&wq->wqfull_list, nvmewqeq, struct lpfc_iocbq,
2082	list);
2083	spin_unlock_irqrestore(lock: &pring->ring_lock, flags: iflags);
2084	ctxp = nvmewqeq->context_un.axchg;
2085	rc = lpfc_sli4_issue_wqe(phba, qp: ctxp->hdwq, pwqe: nvmewqeq);
2086	spin_lock_irqsave(&pring->ring_lock, iflags);
2087	if (rc == -EBUSY) {
2088	/* WQ was full again, so put it back on the list */
2089	list_add(new: &nvmewqeq->list, head: &wq->wqfull_list);
2090	spin_unlock_irqrestore(lock: &pring->ring_lock, flags: iflags);
2091	return;
2092	}
2093	if (rc == WQE_SUCCESS) {
2094	#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
2095	if (ctxp->ts_cmd_nvme) {
2096	if (ctxp->hdlrctx.fcp_req.op == NVMET_FCOP_RSP)
2097	ctxp->ts_status_wqput = ktime_get_ns();
2098	else
2099	ctxp->ts_data_wqput = ktime_get_ns();
2100	}
2101	#endif
2102	} else {
2103	WARN_ON(rc);
2104	}
2105	}
2106	wq->q_flag &= ~HBA_NVMET_WQFULL;
2107	spin_unlock_irqrestore(lock: &pring->ring_lock, flags: iflags);
2108
2109	#endif
2110	}
2111
2112	void
2113	lpfc_nvmet_destroy_targetport(struct lpfc_hba *phba)
2114	{
2115	#if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
2116	struct lpfc_nvmet_tgtport *tgtp;
2117	struct lpfc_queue *wq;
2118	uint32_t qidx;
2119	DECLARE_COMPLETION_ONSTACK(tport_unreg_cmp);
2120
2121	if (phba->nvmet_support == 0)
2122	return;
2123	if (phba->targetport) {
2124	tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
2125	for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) {
2126	wq = phba->sli4_hba.hdwq[qidx].io_wq;
2127	lpfc_nvmet_wqfull_flush(phba, wq, NULL);
2128	}
2129	tgtp->tport_unreg_cmp = &tport_unreg_cmp;
2130	nvmet_fc_unregister_targetport(tgtport: phba->targetport);
2131	if (!wait_for_completion_timeout(x: &tport_unreg_cmp,
2132	timeout: msecs_to_jiffies(LPFC_NVMET_WAIT_TMO)))
2133	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2134	"6179 Unreg targetport x%px timeout "
2135	"reached.\n", phba->targetport);
2136	lpfc_nvmet_cleanup_io_context(phba);
2137	}
2138	phba->targetport = NULL;
2139	#endif
2140	}
2141
2142	/**
2143	* lpfc_nvmet_handle_lsreq - Process an NVME LS request
2144	* @phba: pointer to lpfc hba data structure.
2145	* @axchg: pointer to exchange context for the NVME LS request
2146	*
2147	* This routine is used for processing an asynchronously received NVME LS
2148	* request. Any remaining validation is done and the LS is then forwarded
2149	* to the nvmet-fc transport via nvmet_fc_rcv_ls_req().
2150	*
2151	* The calling sequence should be: nvmet_fc_rcv_ls_req() -> (processing)
2152	* -> lpfc_nvmet_xmt_ls_rsp/cmp -> req->done.
2153	* lpfc_nvme_xmt_ls_rsp_cmp should free the allocated axchg.
2154	*
2155	* Returns 0 if LS was handled and delivered to the transport
2156	* Returns 1 if LS failed to be handled and should be dropped
2157	*/
2158	int
2159	lpfc_nvmet_handle_lsreq(struct lpfc_hba *phba,
2160	struct lpfc_async_xchg_ctx *axchg)
2161	{
2162	#if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
2163	struct lpfc_nvmet_tgtport *tgtp = phba->targetport->private;
2164	uint32_t *payload = axchg->payload;
2165	int rc;
2166
2167	atomic_inc(v: &tgtp->rcv_ls_req_in);
2168
2169	/*
2170	* Driver passes the ndlp as the hosthandle argument allowing
2171	* the transport to generate LS requests for any associateions
2172	* that are created.
2173	*/
2174	rc = nvmet_fc_rcv_ls_req(tgtport: phba->targetport, hosthandle: axchg->ndlp, rsp: &axchg->ls_rsp,
2175	lsreqbuf: axchg->payload, lsreqbuf_len: axchg->size);
2176
2177	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
2178	"6037 NVMET Unsol rcv: sz %d rc %d: %08x %08x %08x "
2179	"%08x %08x %08x\n", axchg->size, rc,
2180	*payload, *(payload+1), *(payload+2),
2181	*(payload+3), *(payload+4), *(payload+5));
2182
2183	if (!rc) {
2184	atomic_inc(v: &tgtp->rcv_ls_req_out);
2185	return 0;
2186	}
2187
2188	atomic_inc(v: &tgtp->rcv_ls_req_drop);
2189	#endif
2190	return 1;
2191	}
2192
2193	static void
2194	lpfc_nvmet_process_rcv_fcp_req(struct lpfc_nvmet_ctxbuf *ctx_buf)
2195	{
2196	#if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
2197	struct lpfc_async_xchg_ctx *ctxp = ctx_buf->context;
2198	struct lpfc_hba *phba = ctxp->phba;
2199	struct rqb_dmabuf *nvmebuf = ctxp->rqb_buffer;
2200	struct lpfc_nvmet_tgtport *tgtp;
2201	uint32_t *payload, qno;
2202	uint32_t rc;
2203	unsigned long iflags;
2204
2205	if (!nvmebuf) {
2206	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2207	"6159 process_rcv_fcp_req, nvmebuf is NULL, "
2208	"oxid: x%x flg: x%x state: x%x\n",
2209	ctxp->oxid, ctxp->flag, ctxp->state);
2210	spin_lock_irqsave(&ctxp->ctxlock, iflags);
2211	lpfc_nvmet_defer_release(phba, ctxp);
2212	spin_unlock_irqrestore(lock: &ctxp->ctxlock, flags: iflags);
2213	lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, ctxp->sid,
2214	ctxp->oxid);
2215	return;
2216	}
2217
2218	if (ctxp->flag & LPFC_NVME_ABTS_RCV) {
2219	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2220	"6324 IO oxid x%x aborted\n",
2221	ctxp->oxid);
2222	return;
2223	}
2224
2225	payload = (uint32_t *)(nvmebuf->dbuf.virt);
2226	tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
2227	ctxp->flag |= LPFC_NVME_TNOTIFY;
2228	#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
2229	if (ctxp->ts_isr_cmd)
2230	ctxp->ts_cmd_nvme = ktime_get_ns();
2231	#endif
2232	/*
2233	* The calling sequence should be:
2234	* nvmet_fc_rcv_fcp_req->lpfc_nvmet_xmt_fcp_op/cmp- req->done
2235	* lpfc_nvmet_xmt_fcp_op_cmp should free the allocated ctxp.
2236	* When we return from nvmet_fc_rcv_fcp_req, all relevant info
2237	* the NVME command / FC header is stored.
2238	* A buffer has already been reposted for this IO, so just free
2239	* the nvmebuf.
2240	*/
2241	rc = nvmet_fc_rcv_fcp_req(tgtport: phba->targetport, fcpreq: &ctxp->hdlrctx.fcp_req,
2242	cmdiubuf: payload, cmdiubuf_len: ctxp->size);
2243	/* Process FCP command */
2244	if (rc == 0) {
2245	atomic_inc(v: &tgtp->rcv_fcp_cmd_out);
2246	spin_lock_irqsave(&ctxp->ctxlock, iflags);
2247	if ((ctxp->flag & LPFC_NVME_CTX_REUSE_WQ) ||
2248	(nvmebuf != ctxp->rqb_buffer)) {
2249	spin_unlock_irqrestore(lock: &ctxp->ctxlock, flags: iflags);
2250	return;
2251	}
2252	ctxp->rqb_buffer = NULL;
2253	spin_unlock_irqrestore(lock: &ctxp->ctxlock, flags: iflags);
2254	lpfc_rq_buf_free(phba, mp: &nvmebuf->hbuf); /* repost */
2255	return;
2256	}
2257
2258	/* Processing of FCP command is deferred */
2259	if (rc == -EOVERFLOW) {
2260	lpfc_nvmeio_data(phba, "NVMET RCV BUSY: xri x%x sz %d "
2261	"from %06x\n",
2262	ctxp->oxid, ctxp->size, ctxp->sid);
2263	atomic_inc(v: &tgtp->rcv_fcp_cmd_out);
2264	atomic_inc(v: &tgtp->defer_fod);
2265	spin_lock_irqsave(&ctxp->ctxlock, iflags);
2266	if (ctxp->flag & LPFC_NVME_CTX_REUSE_WQ) {
2267	spin_unlock_irqrestore(lock: &ctxp->ctxlock, flags: iflags);
2268	return;
2269	}
2270	spin_unlock_irqrestore(lock: &ctxp->ctxlock, flags: iflags);
2271	/*
2272	* Post a replacement DMA buffer to RQ and defer
2273	* freeing rcv buffer till .defer_rcv callback
2274	*/
2275	qno = nvmebuf->idx;
2276	lpfc_post_rq_buffer(
2277	phba, hrq: phba->sli4_hba.nvmet_mrq_hdr[qno],
2278	drq: phba->sli4_hba.nvmet_mrq_data[qno], count: 1, idx: qno);
2279	return;
2280	}
2281	ctxp->flag &= ~LPFC_NVME_TNOTIFY;
2282	atomic_inc(v: &tgtp->rcv_fcp_cmd_drop);
2283	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2284	"2582 FCP Drop IO x%x: err x%x: x%x x%x x%x\n",
2285	ctxp->oxid, rc,
2286	atomic_read(&tgtp->rcv_fcp_cmd_in),
2287	atomic_read(&tgtp->rcv_fcp_cmd_out),
2288	atomic_read(&tgtp->xmt_fcp_release));
2289	lpfc_nvmeio_data(phba, "NVMET FCP DROP: xri x%x sz %d from %06x\n",
2290	ctxp->oxid, ctxp->size, ctxp->sid);
2291	spin_lock_irqsave(&ctxp->ctxlock, iflags);
2292	lpfc_nvmet_defer_release(phba, ctxp);
2293	spin_unlock_irqrestore(lock: &ctxp->ctxlock, flags: iflags);
2294	lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, ctxp->sid, ctxp->oxid);
2295	#endif
2296	}
2297
2298	static void
2299	lpfc_nvmet_fcp_rqst_defer_work(struct work_struct *work)
2300	{
2301	#if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
2302	struct lpfc_nvmet_ctxbuf *ctx_buf =
2303	container_of(work, struct lpfc_nvmet_ctxbuf, defer_work);
2304
2305	lpfc_nvmet_process_rcv_fcp_req(ctx_buf);
2306	#endif
2307	}
2308
2309	static struct lpfc_nvmet_ctxbuf *
2310	lpfc_nvmet_replenish_context(struct lpfc_hba *phba,
2311	struct lpfc_nvmet_ctx_info *current_infop)
2312	{
2313	#if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
2314	struct lpfc_nvmet_ctxbuf *ctx_buf = NULL;
2315	struct lpfc_nvmet_ctx_info *get_infop;
2316	int i;
2317
2318	/*
2319	* The current_infop for the MRQ a NVME command IU was received
2320	* on is empty. Our goal is to replenish this MRQs context
2321	* list from a another CPUs.
2322	*
2323	* First we need to pick a context list to start looking on.
2324	* nvmet_ctx_start_cpu has available context the last time
2325	* we needed to replenish this CPU where nvmet_ctx_next_cpu
2326	* is just the next sequential CPU for this MRQ.
2327	*/
2328	if (current_infop->nvmet_ctx_start_cpu)
2329	get_infop = current_infop->nvmet_ctx_start_cpu;
2330	else
2331	get_infop = current_infop->nvmet_ctx_next_cpu;
2332
2333	for (i = 0; i < phba->sli4_hba.num_possible_cpu; i++) {
2334	if (get_infop == current_infop) {
2335	get_infop = get_infop->nvmet_ctx_next_cpu;
2336	continue;
2337	}
2338	spin_lock(lock: &get_infop->nvmet_ctx_list_lock);
2339
2340	/* Just take the entire context list, if there are any */
2341	if (get_infop->nvmet_ctx_list_cnt) {
2342	list_splice_init(list: &get_infop->nvmet_ctx_list,
2343	head: &current_infop->nvmet_ctx_list);
2344	current_infop->nvmet_ctx_list_cnt =
2345	get_infop->nvmet_ctx_list_cnt - 1;
2346	get_infop->nvmet_ctx_list_cnt = 0;
2347	spin_unlock(lock: &get_infop->nvmet_ctx_list_lock);
2348
2349	current_infop->nvmet_ctx_start_cpu = get_infop;
2350	list_remove_head(&current_infop->nvmet_ctx_list,
2351	ctx_buf, struct lpfc_nvmet_ctxbuf,
2352	list);
2353	return ctx_buf;
2354	}
2355
2356	/* Otherwise, move on to the next CPU for this MRQ */
2357	spin_unlock(lock: &get_infop->nvmet_ctx_list_lock);
2358	get_infop = get_infop->nvmet_ctx_next_cpu;
2359	}
2360
2361	#endif
2362	/* Nothing found, all contexts for the MRQ are in-flight */
2363	return NULL;
2364	}
2365
2366	/**
2367	* lpfc_nvmet_unsol_fcp_buffer - Process an unsolicited event data buffer
2368	* @phba: pointer to lpfc hba data structure.
2369	* @idx: relative index of MRQ vector
2370	* @nvmebuf: pointer to lpfc nvme command HBQ data structure.
2371	* @isr_timestamp: in jiffies.
2372	* @cqflag: cq processing information regarding workload.
2373	*
2374	* This routine is used for processing the WQE associated with a unsolicited
2375	* event. It first determines whether there is an existing ndlp that matches
2376	* the DID from the unsolicited WQE. If not, it will create a new one with
2377	* the DID from the unsolicited WQE. The ELS command from the unsolicited
2378	* WQE is then used to invoke the proper routine and to set up proper state
2379	* of the discovery state machine.
2380	**/
2381	static void
2382	lpfc_nvmet_unsol_fcp_buffer(struct lpfc_hba *phba,
2383	uint32_t idx,
2384	struct rqb_dmabuf *nvmebuf,
2385	uint64_t isr_timestamp,
2386	uint8_t cqflag)
2387	{
2388	struct lpfc_async_xchg_ctx *ctxp;
2389	struct lpfc_nvmet_tgtport *tgtp;
2390	struct fc_frame_header *fc_hdr;
2391	struct lpfc_nvmet_ctxbuf *ctx_buf;
2392	struct lpfc_nvmet_ctx_info *current_infop;
2393	uint32_t size, oxid, sid, qno;
2394	unsigned long iflag;
2395	int current_cpu;
2396
2397	if (!IS_ENABLED(CONFIG_NVME_TARGET_FC))
2398	return;
2399
2400	ctx_buf = NULL;
2401	if (!nvmebuf || !phba->targetport) {
2402	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2403	"6157 NVMET FCP Drop IO\n");
2404	if (nvmebuf)
2405	lpfc_rq_buf_free(phba, mp: &nvmebuf->hbuf);
2406	return;
2407	}
2408
2409	/*
2410	* Get a pointer to the context list for this MRQ based on
2411	* the CPU this MRQ IRQ is associated with. If the CPU association
2412	* changes from our initial assumption, the context list could
2413	* be empty, thus it would need to be replenished with the
2414	* context list from another CPU for this MRQ.
2415	*/
2416	current_cpu = raw_smp_processor_id();
2417	current_infop = lpfc_get_ctx_list(phba, current_cpu, idx);
2418	spin_lock_irqsave(&current_infop->nvmet_ctx_list_lock, iflag);
2419	if (current_infop->nvmet_ctx_list_cnt) {
2420	list_remove_head(&current_infop->nvmet_ctx_list,
2421	ctx_buf, struct lpfc_nvmet_ctxbuf, list);
2422	current_infop->nvmet_ctx_list_cnt--;
2423	} else {
2424	ctx_buf = lpfc_nvmet_replenish_context(phba, current_infop);
2425	}
2426	spin_unlock_irqrestore(lock: &current_infop->nvmet_ctx_list_lock, flags: iflag);
2427
2428	fc_hdr = (struct fc_frame_header *)(nvmebuf->hbuf.virt);
2429	oxid = be16_to_cpu(fc_hdr->fh_ox_id);
2430	size = nvmebuf->bytes_recv;
2431
2432	#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
2433	if (phba->hdwqstat_on & LPFC_CHECK_NVMET_IO) {
2434	this_cpu_inc(phba->sli4_hba.c_stat->rcv_io);
2435	if (idx != current_cpu)
2436	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,
2437	"6703 CPU Check rcv: "
2438	"cpu %d expect %d\n",
2439	current_cpu, idx);
2440	}
2441	#endif
2442
2443	lpfc_nvmeio_data(phba, "NVMET FCP RCV: xri x%x sz %d CPU %02x\n",
2444	oxid, size, raw_smp_processor_id());
2445
2446	tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
2447
2448	if (!ctx_buf) {
2449	/* Queue this NVME IO to process later */
2450	spin_lock_irqsave(&phba->sli4_hba.nvmet_io_wait_lock, iflag);
2451	list_add_tail(new: &nvmebuf->hbuf.list,
2452	head: &phba->sli4_hba.lpfc_nvmet_io_wait_list);
2453	phba->sli4_hba.nvmet_io_wait_cnt++;
2454	phba->sli4_hba.nvmet_io_wait_total++;
2455	spin_unlock_irqrestore(lock: &phba->sli4_hba.nvmet_io_wait_lock,
2456	flags: iflag);
2457
2458	/* Post a brand new DMA buffer to RQ */
2459	qno = nvmebuf->idx;
2460	lpfc_post_rq_buffer(
2461	phba, hrq: phba->sli4_hba.nvmet_mrq_hdr[qno],
2462	drq: phba->sli4_hba.nvmet_mrq_data[qno], count: 1, idx: qno);
2463
2464	atomic_inc(v: &tgtp->defer_ctx);
2465	return;
2466	}
2467
2468	sid = sli4_sid_from_fc_hdr(fc_hdr);
2469
2470	ctxp = (struct lpfc_async_xchg_ctx *)ctx_buf->context;
2471	spin_lock_irqsave(&phba->sli4_hba.t_active_list_lock, iflag);
2472	list_add_tail(new: &ctxp->list, head: &phba->sli4_hba.t_active_ctx_list);
2473	spin_unlock_irqrestore(lock: &phba->sli4_hba.t_active_list_lock, flags: iflag);
2474	if (ctxp->state != LPFC_NVME_STE_FREE) {
2475	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2476	"6414 NVMET Context corrupt %d %d oxid x%x\n",
2477	ctxp->state, ctxp->entry_cnt, ctxp->oxid);
2478	}
2479	ctxp->wqeq = NULL;
2480	ctxp->offset = 0;
2481	ctxp->phba = phba;
2482	ctxp->size = size;
2483	ctxp->oxid = oxid;
2484	ctxp->sid = sid;
2485	ctxp->idx = idx;
2486	ctxp->state = LPFC_NVME_STE_RCV;
2487	ctxp->entry_cnt = 1;
2488	ctxp->flag = 0;
2489	ctxp->ctxbuf = ctx_buf;
2490	ctxp->rqb_buffer = (void *)nvmebuf;
2491	ctxp->hdwq = NULL;
2492	spin_lock_init(&ctxp->ctxlock);
2493
2494	#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
2495	if (isr_timestamp)
2496	ctxp->ts_isr_cmd = isr_timestamp;
2497	ctxp->ts_cmd_nvme = 0;
2498	ctxp->ts_nvme_data = 0;
2499	ctxp->ts_data_wqput = 0;
2500	ctxp->ts_isr_data = 0;
2501	ctxp->ts_data_nvme = 0;
2502	ctxp->ts_nvme_status = 0;
2503	ctxp->ts_status_wqput = 0;
2504	ctxp->ts_isr_status = 0;
2505	ctxp->ts_status_nvme = 0;
2506	#endif
2507
2508	atomic_inc(v: &tgtp->rcv_fcp_cmd_in);
2509	/* check for cq processing load */
2510	if (!cqflag) {
2511	lpfc_nvmet_process_rcv_fcp_req(ctx_buf);
2512	return;
2513	}
2514
2515	if (!queue_work(wq: phba->wq, work: &ctx_buf->defer_work)) {
2516	atomic_inc(v: &tgtp->rcv_fcp_cmd_drop);
2517	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2518	"6325 Unable to queue work for oxid x%x. "
2519	"FCP Drop IO [x%x x%x x%x]\n",
2520	ctxp->oxid,
2521	atomic_read(&tgtp->rcv_fcp_cmd_in),
2522	atomic_read(&tgtp->rcv_fcp_cmd_out),
2523	atomic_read(&tgtp->xmt_fcp_release));
2524
2525	spin_lock_irqsave(&ctxp->ctxlock, iflag);
2526	lpfc_nvmet_defer_release(phba, ctxp);
2527	spin_unlock_irqrestore(lock: &ctxp->ctxlock, flags: iflag);
2528	lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, sid, oxid);
2529	}
2530	}
2531
2532	/**
2533	* lpfc_nvmet_unsol_fcp_event - Process an unsolicited event from an nvme nport
2534	* @phba: pointer to lpfc hba data structure.
2535	* @idx: relative index of MRQ vector
2536	* @nvmebuf: pointer to received nvme data structure.
2537	* @isr_timestamp: in jiffies.
2538	* @cqflag: cq processing information regarding workload.
2539	*
2540	* This routine is used to process an unsolicited event received from a SLI
2541	* (Service Level Interface) ring. The actual processing of the data buffer
2542	* associated with the unsolicited event is done by invoking the routine
2543	* lpfc_nvmet_unsol_fcp_buffer() after properly set up the buffer from the
2544	* SLI RQ on which the unsolicited event was received.
2545	**/
2546	void
2547	lpfc_nvmet_unsol_fcp_event(struct lpfc_hba *phba,
2548	uint32_t idx,
2549	struct rqb_dmabuf *nvmebuf,
2550	uint64_t isr_timestamp,
2551	uint8_t cqflag)
2552	{
2553	if (!nvmebuf) {
2554	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2555	"3167 NVMET FCP Drop IO\n");
2556	return;
2557	}
2558	if (phba->nvmet_support == 0) {
2559	lpfc_rq_buf_free(phba, mp: &nvmebuf->hbuf);
2560	return;
2561	}
2562	lpfc_nvmet_unsol_fcp_buffer(phba, idx, nvmebuf, isr_timestamp, cqflag);
2563	}
2564
2565	/**
2566	* lpfc_nvmet_prep_ls_wqe - Allocate and prepare a lpfc wqe data structure
2567	* @phba: pointer to a host N_Port data structure.
2568	* @ctxp: Context info for NVME LS Request
2569	* @rspbuf: DMA buffer of NVME command.
2570	* @rspsize: size of the NVME command.
2571	*
2572	* This routine is used for allocating a lpfc-WQE data structure from
2573	* the driver lpfc-WQE free-list and prepare the WQE with the parameters
2574	* passed into the routine for discovery state machine to issue an Extended
2575	* Link Service (NVME) commands. It is a generic lpfc-WQE allocation
2576	* and preparation routine that is used by all the discovery state machine
2577	* routines and the NVME command-specific fields will be later set up by
2578	* the individual discovery machine routines after calling this routine
2579	* allocating and preparing a generic WQE data structure. It fills in the
2580	* Buffer Descriptor Entries (BDEs), allocates buffers for both command
2581	* payload and response payload (if expected). The reference count on the
2582	* ndlp is incremented by 1 and the reference to the ndlp is put into
2583	* context1 of the WQE data structure for this WQE to hold the ndlp
2584	* reference for the command's callback function to access later.
2585	*
2586	* Return code
2587	* Pointer to the newly allocated/prepared nvme wqe data structure
2588	* NULL - when nvme wqe data structure allocation/preparation failed
2589	**/
2590	static struct lpfc_iocbq *
2591	lpfc_nvmet_prep_ls_wqe(struct lpfc_hba *phba,
2592	struct lpfc_async_xchg_ctx *ctxp,
2593	dma_addr_t rspbuf, uint16_t rspsize)
2594	{
2595	struct lpfc_nodelist *ndlp;
2596	struct lpfc_iocbq *nvmewqe;
2597	union lpfc_wqe128 *wqe;
2598
2599	if (!lpfc_is_link_up(phba)) {
2600	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2601	"6104 NVMET prep LS wqe: link err: "
2602	"NPORT x%x oxid:x%x ste %d\n",
2603	ctxp->sid, ctxp->oxid, ctxp->state);
2604	return NULL;
2605	}
2606
2607	/* Allocate buffer for command wqe */
2608	nvmewqe = lpfc_sli_get_iocbq(phba);
2609	if (nvmewqe == NULL) {
2610	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2611	"6105 NVMET prep LS wqe: No WQE: "
2612	"NPORT x%x oxid x%x ste %d\n",
2613	ctxp->sid, ctxp->oxid, ctxp->state);
2614	return NULL;
2615	}
2616
2617	ndlp = lpfc_findnode_did(phba->pport, ctxp->sid);
2618	if (!ndlp ||
2619	((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) &&
2620	(ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {
2621	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2622	"6106 NVMET prep LS wqe: No ndlp: "
2623	"NPORT x%x oxid x%x ste %d\n",
2624	ctxp->sid, ctxp->oxid, ctxp->state);
2625	goto nvme_wqe_free_wqeq_exit;
2626	}
2627	ctxp->wqeq = nvmewqe;
2628
2629	/* prevent preparing wqe with NULL ndlp reference */
2630	nvmewqe->ndlp = lpfc_nlp_get(ndlp);
2631	if (!nvmewqe->ndlp)
2632	goto nvme_wqe_free_wqeq_exit;
2633	nvmewqe->context_un.axchg = ctxp;
2634
2635	wqe = &nvmewqe->wqe;
2636	memset(wqe, 0, sizeof(union lpfc_wqe));
2637
2638	/* Words 0 - 2 */
2639	wqe->xmit_sequence.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
2640	wqe->xmit_sequence.bde.tus.f.bdeSize = rspsize;
2641	wqe->xmit_sequence.bde.addrLow = le32_to_cpu(putPaddrLow(rspbuf));
2642	wqe->xmit_sequence.bde.addrHigh = le32_to_cpu(putPaddrHigh(rspbuf));
2643
2644	/* Word 3 */
2645
2646	/* Word 4 */
2647
2648	/* Word 5 */
2649	bf_set(wqe_dfctl, &wqe->xmit_sequence.wge_ctl, 0);
2650	bf_set(wqe_ls, &wqe->xmit_sequence.wge_ctl, 1);
2651	bf_set(wqe_la, &wqe->xmit_sequence.wge_ctl, 0);
2652	bf_set(wqe_rctl, &wqe->xmit_sequence.wge_ctl, FC_RCTL_ELS4_REP);
2653	bf_set(wqe_type, &wqe->xmit_sequence.wge_ctl, FC_TYPE_NVME);
2654
2655	/* Word 6 */
2656	bf_set(wqe_ctxt_tag, &wqe->xmit_sequence.wqe_com,
2657	phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
2658	bf_set(wqe_xri_tag, &wqe->xmit_sequence.wqe_com, nvmewqe->sli4_xritag);
2659
2660	/* Word 7 */
2661	bf_set(wqe_cmnd, &wqe->xmit_sequence.wqe_com,
2662	CMD_XMIT_SEQUENCE64_WQE);
2663	bf_set(wqe_ct, &wqe->xmit_sequence.wqe_com, SLI4_CT_RPI);
2664	bf_set(wqe_class, &wqe->xmit_sequence.wqe_com, CLASS3);
2665	bf_set(wqe_pu, &wqe->xmit_sequence.wqe_com, 0);
2666
2667	/* Word 8 */
2668	wqe->xmit_sequence.wqe_com.abort_tag = nvmewqe->iotag;
2669
2670	/* Word 9 */
2671	bf_set(wqe_reqtag, &wqe->xmit_sequence.wqe_com, nvmewqe->iotag);
2672	/* Needs to be set by caller */
2673	bf_set(wqe_rcvoxid, &wqe->xmit_sequence.wqe_com, ctxp->oxid);
2674
2675	/* Word 10 */
2676	bf_set(wqe_dbde, &wqe->xmit_sequence.wqe_com, 1);
2677	bf_set(wqe_iod, &wqe->xmit_sequence.wqe_com, LPFC_WQE_IOD_WRITE);
2678	bf_set(wqe_lenloc, &wqe->xmit_sequence.wqe_com,
2679	LPFC_WQE_LENLOC_WORD12);
2680	bf_set(wqe_ebde_cnt, &wqe->xmit_sequence.wqe_com, 0);
2681
2682	/* Word 11 */
2683	bf_set(wqe_cqid, &wqe->xmit_sequence.wqe_com,
2684	LPFC_WQE_CQ_ID_DEFAULT);
2685	bf_set(wqe_cmd_type, &wqe->xmit_sequence.wqe_com,
2686	OTHER_COMMAND);
2687
2688	/* Word 12 */
2689	wqe->xmit_sequence.xmit_len = rspsize;
2690
2691	nvmewqe->retry = 1;
2692	nvmewqe->vport = phba->pport;
2693	nvmewqe->drvrTimeout = (phba->fc_ratov * 3) + LPFC_DRVR_TIMEOUT;
2694	nvmewqe->cmd_flag |= LPFC_IO_NVME_LS;
2695
2696	/* Xmit NVMET response to remote NPORT <did> */
2697	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
2698	"6039 Xmit NVMET LS response to remote "
2699	"NPORT x%x iotag:x%x oxid:x%x size:x%x\n",
2700	ndlp->nlp_DID, nvmewqe->iotag, ctxp->oxid,
2701	rspsize);
2702	return nvmewqe;
2703
2704	nvme_wqe_free_wqeq_exit:
2705	nvmewqe->context_un.axchg = NULL;
2706	nvmewqe->ndlp = NULL;
2707	nvmewqe->bpl_dmabuf = NULL;
2708	lpfc_sli_release_iocbq(phba, nvmewqe);
2709	return NULL;
2710	}
2711
2712
2713	static struct lpfc_iocbq *
2714	lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *phba,
2715	struct lpfc_async_xchg_ctx *ctxp)
2716	{
2717	struct nvmefc_tgt_fcp_req *rsp = &ctxp->hdlrctx.fcp_req;
2718	struct lpfc_nvmet_tgtport *tgtp;
2719	struct sli4_sge *sgl;
2720	struct lpfc_nodelist *ndlp;
2721	struct lpfc_iocbq *nvmewqe;
2722	struct scatterlist *sgel;
2723	union lpfc_wqe128 *wqe;
2724	struct ulp_bde64 *bde;
2725	dma_addr_t physaddr;
2726	int i, cnt, nsegs;
2727	bool use_pbde = false;
2728	int xc = 1;
2729
2730	if (!lpfc_is_link_up(phba)) {
2731	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2732	"6107 NVMET prep FCP wqe: link err:"
2733	"NPORT x%x oxid x%x ste %d\n",
2734	ctxp->sid, ctxp->oxid, ctxp->state);
2735	return NULL;
2736	}
2737
2738	ndlp = lpfc_findnode_did(phba->pport, ctxp->sid);
2739	if (!ndlp ||
2740	((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) &&
2741	(ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {
2742	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2743	"6108 NVMET prep FCP wqe: no ndlp: "
2744	"NPORT x%x oxid x%x ste %d\n",
2745	ctxp->sid, ctxp->oxid, ctxp->state);
2746	return NULL;
2747	}
2748
2749	if (rsp->sg_cnt > lpfc_tgttemplate.max_sgl_segments) {
2750	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2751	"6109 NVMET prep FCP wqe: seg cnt err: "
2752	"NPORT x%x oxid x%x ste %d cnt %d\n",
2753	ctxp->sid, ctxp->oxid, ctxp->state,
2754	phba->cfg_nvme_seg_cnt);
2755	return NULL;
2756	}
2757	nsegs = rsp->sg_cnt;
2758
2759	tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
2760	nvmewqe = ctxp->wqeq;
2761	if (nvmewqe == NULL) {
2762	/* Allocate buffer for command wqe */
2763	nvmewqe = ctxp->ctxbuf->iocbq;
2764	if (nvmewqe == NULL) {
2765	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2766	"6110 NVMET prep FCP wqe: No "
2767	"WQE: NPORT x%x oxid x%x ste %d\n",
2768	ctxp->sid, ctxp->oxid, ctxp->state);
2769	return NULL;
2770	}
2771	ctxp->wqeq = nvmewqe;
2772	xc = 0; /* create new XRI */
2773	nvmewqe->sli4_lxritag = NO_XRI;
2774	nvmewqe->sli4_xritag = NO_XRI;
2775	}
2776
2777	/* Sanity check */
2778	if (((ctxp->state == LPFC_NVME_STE_RCV) &&
2779	(ctxp->entry_cnt == 1)) ||
2780	(ctxp->state == LPFC_NVME_STE_DATA)) {
2781	wqe = &nvmewqe->wqe;
2782	} else {
2783	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2784	"6111 Wrong state NVMET FCP: %d cnt %d\n",
2785	ctxp->state, ctxp->entry_cnt);
2786	return NULL;
2787	}
2788
2789	sgl = (struct sli4_sge *)ctxp->ctxbuf->sglq->sgl;
2790	switch (rsp->op) {
2791	case NVMET_FCOP_READDATA:
2792	case NVMET_FCOP_READDATA_RSP:
2793	/* From the tsend template, initialize words 7 - 11 */
2794	memcpy(&wqe->words[7],
2795	&lpfc_tsend_cmd_template.words[7],
2796	sizeof(uint32_t) * 5);
2797
2798	/* Words 0 - 2 : The first sg segment */
2799	sgel = &rsp->sg[0];
2800	physaddr = sg_dma_address(sgel);
2801	wqe->fcp_tsend.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
2802	wqe->fcp_tsend.bde.tus.f.bdeSize = sg_dma_len(sgel);
2803	wqe->fcp_tsend.bde.addrLow = cpu_to_le32(putPaddrLow(physaddr));
2804	wqe->fcp_tsend.bde.addrHigh =
2805	cpu_to_le32(putPaddrHigh(physaddr));
2806
2807	/* Word 3 */
2808	wqe->fcp_tsend.payload_offset_len = 0;
2809
2810	/* Word 4 */
2811	wqe->fcp_tsend.relative_offset = ctxp->offset;
2812
2813	/* Word 5 */
2814	wqe->fcp_tsend.reserved = 0;
2815
2816	/* Word 6 */
2817	bf_set(wqe_ctxt_tag, &wqe->fcp_tsend.wqe_com,
2818	phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
2819	bf_set(wqe_xri_tag, &wqe->fcp_tsend.wqe_com,
2820	nvmewqe->sli4_xritag);
2821
2822	/* Word 7 - set ar later */
2823
2824	/* Word 8 */
2825	wqe->fcp_tsend.wqe_com.abort_tag = nvmewqe->iotag;
2826
2827	/* Word 9 */
2828	bf_set(wqe_reqtag, &wqe->fcp_tsend.wqe_com, nvmewqe->iotag);
2829	bf_set(wqe_rcvoxid, &wqe->fcp_tsend.wqe_com, ctxp->oxid);
2830
2831	/* Word 10 - set wqes later, in template xc=1 */
2832	if (!xc)
2833	bf_set(wqe_xc, &wqe->fcp_tsend.wqe_com, 0);
2834
2835	/* Word 12 */
2836	wqe->fcp_tsend.fcp_data_len = rsp->transfer_length;
2837
2838	/* Setup 2 SKIP SGEs */
2839	sgl->addr_hi = 0;
2840	sgl->addr_lo = 0;
2841	sgl->word2 = 0;
2842	bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);
2843	sgl->word2 = cpu_to_le32(sgl->word2);
2844	sgl->sge_len = 0;
2845	sgl++;
2846	sgl->addr_hi = 0;
2847	sgl->addr_lo = 0;
2848	sgl->word2 = 0;
2849	bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);
2850	sgl->word2 = cpu_to_le32(sgl->word2);
2851	sgl->sge_len = 0;
2852	sgl++;
2853	if (rsp->op == NVMET_FCOP_READDATA_RSP) {
2854	atomic_inc(v: &tgtp->xmt_fcp_read_rsp);
2855
2856	/* In template ar=1 wqes=0 sup=0 irsp=0 irsplen=0 */
2857
2858	if (rsp->rsplen == LPFC_NVMET_SUCCESS_LEN) {
2859	if (test_bit(NLP_SUPPRESS_RSP, &ndlp->nlp_flag))
2860	bf_set(wqe_sup,
2861	&wqe->fcp_tsend.wqe_com, 1);
2862	} else {
2863	bf_set(wqe_wqes, &wqe->fcp_tsend.wqe_com, 1);
2864	bf_set(wqe_irsp, &wqe->fcp_tsend.wqe_com, 1);
2865	bf_set(wqe_irsplen, &wqe->fcp_tsend.wqe_com,
2866	((rsp->rsplen >> 2) - 1));
2867	memcpy(&wqe->words[16], rsp->rspaddr,
2868	rsp->rsplen);
2869	}
2870	} else {
2871	atomic_inc(v: &tgtp->xmt_fcp_read);
2872
2873	/* In template ar=1 wqes=0 sup=0 irsp=0 irsplen=0 */
2874	bf_set(wqe_ar, &wqe->fcp_tsend.wqe_com, 0);
2875	}
2876	break;
2877
2878	case NVMET_FCOP_WRITEDATA:
2879	/* From the treceive template, initialize words 3 - 11 */
2880	memcpy(&wqe->words[3],
2881	&lpfc_treceive_cmd_template.words[3],
2882	sizeof(uint32_t) * 9);
2883
2884	/* Words 0 - 2 : First SGE is skipped, set invalid BDE type */
2885	wqe->fcp_treceive.bde.tus.f.bdeFlags = LPFC_SGE_TYPE_SKIP;
2886	wqe->fcp_treceive.bde.tus.f.bdeSize = 0;
2887	wqe->fcp_treceive.bde.addrLow = 0;
2888	wqe->fcp_treceive.bde.addrHigh = 0;
2889
2890	/* Word 4 */
2891	wqe->fcp_treceive.relative_offset = ctxp->offset;
2892
2893	/* Word 6 */
2894	bf_set(wqe_ctxt_tag, &wqe->fcp_treceive.wqe_com,
2895	phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
2896	bf_set(wqe_xri_tag, &wqe->fcp_treceive.wqe_com,
2897	nvmewqe->sli4_xritag);
2898
2899	/* Word 7 */
2900
2901	/* Word 8 */
2902	wqe->fcp_treceive.wqe_com.abort_tag = nvmewqe->iotag;
2903
2904	/* Word 9 */
2905	bf_set(wqe_reqtag, &wqe->fcp_treceive.wqe_com, nvmewqe->iotag);
2906	bf_set(wqe_rcvoxid, &wqe->fcp_treceive.wqe_com, ctxp->oxid);
2907
2908	/* Word 10 - in template xc=1 */
2909	if (!xc)
2910	bf_set(wqe_xc, &wqe->fcp_treceive.wqe_com, 0);
2911
2912	/* Word 11 - check for pbde */
2913	if (nsegs == 1 && phba->cfg_enable_pbde) {
2914	use_pbde = true;
2915	/* Word 11 - PBDE bit already preset by template */
2916	} else {
2917	/* Overwrite default template setting */
2918	bf_set(wqe_pbde, &wqe->fcp_treceive.wqe_com, 0);
2919	}
2920
2921	/* Word 12 */
2922	wqe->fcp_tsend.fcp_data_len = rsp->transfer_length;
2923
2924	/* Setup 2 SKIP SGEs */
2925	sgl->addr_hi = 0;
2926	sgl->addr_lo = 0;
2927	sgl->word2 = 0;
2928	bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);
2929	sgl->word2 = cpu_to_le32(sgl->word2);
2930	sgl->sge_len = 0;
2931	sgl++;
2932	sgl->addr_hi = 0;
2933	sgl->addr_lo = 0;
2934	sgl->word2 = 0;
2935	bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);
2936	sgl->word2 = cpu_to_le32(sgl->word2);
2937	sgl->sge_len = 0;
2938	sgl++;
2939	atomic_inc(v: &tgtp->xmt_fcp_write);
2940	break;
2941
2942	case NVMET_FCOP_RSP:
2943	/* From the treceive template, initialize words 4 - 11 */
2944	memcpy(&wqe->words[4],
2945	&lpfc_trsp_cmd_template.words[4],
2946	sizeof(uint32_t) * 8);
2947
2948	/* Words 0 - 2 */
2949	physaddr = rsp->rspdma;
2950	wqe->fcp_trsp.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
2951	wqe->fcp_trsp.bde.tus.f.bdeSize = rsp->rsplen;
2952	wqe->fcp_trsp.bde.addrLow =
2953	cpu_to_le32(putPaddrLow(physaddr));
2954	wqe->fcp_trsp.bde.addrHigh =
2955	cpu_to_le32(putPaddrHigh(physaddr));
2956
2957	/* Word 3 */
2958	wqe->fcp_trsp.response_len = rsp->rsplen;
2959
2960	/* Word 6 */
2961	bf_set(wqe_ctxt_tag, &wqe->fcp_trsp.wqe_com,
2962	phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
2963	bf_set(wqe_xri_tag, &wqe->fcp_trsp.wqe_com,
2964	nvmewqe->sli4_xritag);
2965
2966	/* Word 7 */
2967
2968	/* Word 8 */
2969	wqe->fcp_trsp.wqe_com.abort_tag = nvmewqe->iotag;
2970
2971	/* Word 9 */
2972	bf_set(wqe_reqtag, &wqe->fcp_trsp.wqe_com, nvmewqe->iotag);
2973	bf_set(wqe_rcvoxid, &wqe->fcp_trsp.wqe_com, ctxp->oxid);
2974
2975	/* Word 10 */
2976	if (xc)
2977	bf_set(wqe_xc, &wqe->fcp_trsp.wqe_com, 1);
2978
2979	/* Word 11 */
2980	/* In template wqes=0 irsp=0 irsplen=0 - good response */
2981	if (rsp->rsplen != LPFC_NVMET_SUCCESS_LEN) {
2982	/* Bad response - embed it */
2983	bf_set(wqe_wqes, &wqe->fcp_trsp.wqe_com, 1);
2984	bf_set(wqe_irsp, &wqe->fcp_trsp.wqe_com, 1);
2985	bf_set(wqe_irsplen, &wqe->fcp_trsp.wqe_com,
2986	((rsp->rsplen >> 2) - 1));
2987	memcpy(&wqe->words[16], rsp->rspaddr, rsp->rsplen);
2988	}
2989
2990	/* Word 12 */
2991	wqe->fcp_trsp.rsvd_12_15[0] = 0;
2992
2993	/* Use rspbuf, NOT sg list */
2994	nsegs = 0;
2995	sgl->word2 = 0;
2996	atomic_inc(v: &tgtp->xmt_fcp_rsp);
2997	break;
2998
2999	default:
3000	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,
3001	"6064 Unknown Rsp Op %d\n",
3002	rsp->op);
3003	return NULL;
3004	}
3005
3006	nvmewqe->retry = 1;
3007	nvmewqe->vport = phba->pport;
3008	nvmewqe->drvrTimeout = (phba->fc_ratov * 3) + LPFC_DRVR_TIMEOUT;
3009	nvmewqe->ndlp = ndlp;
3010
3011	for_each_sg(rsp->sg, sgel, nsegs, i) {
3012	physaddr = sg_dma_address(sgel);
3013	cnt = sg_dma_len(sgel);
3014	sgl->addr_hi = putPaddrHigh(physaddr);
3015	sgl->addr_lo = putPaddrLow(physaddr);
3016	sgl->word2 = 0;
3017	bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_DATA);
3018	bf_set(lpfc_sli4_sge_offset, sgl, ctxp->offset);
3019	if ((i+1) == rsp->sg_cnt)
3020	bf_set(lpfc_sli4_sge_last, sgl, 1);
3021	sgl->word2 = cpu_to_le32(sgl->word2);
3022	sgl->sge_len = cpu_to_le32(cnt);
3023	sgl++;
3024	ctxp->offset += cnt;
3025	}
3026
3027	bde = (struct ulp_bde64 *)&wqe->words[13];
3028	if (use_pbde) {
3029	/* decrement sgl ptr backwards once to first data sge */
3030	sgl--;
3031
3032	/* Words 13-15 (PBDE) */
3033	bde->addrLow = sgl->addr_lo;
3034	bde->addrHigh = sgl->addr_hi;
3035	bde->tus.f.bdeSize = le32_to_cpu(sgl->sge_len);
3036	bde->tus.f.bdeFlags = BUFF_TYPE_BDE_64;
3037	bde->tus.w = cpu_to_le32(bde->tus.w);
3038	} else {
3039	memset(bde, 0, sizeof(struct ulp_bde64));
3040	}
3041	ctxp->state = LPFC_NVME_STE_DATA;
3042	ctxp->entry_cnt++;
3043	return nvmewqe;
3044	}
3045
3046	/**
3047	* lpfc_nvmet_sol_fcp_abort_cmp - Completion handler for ABTS
3048	* @phba: Pointer to HBA context object.
3049	* @cmdwqe: Pointer to driver command WQE object.
3050	* @rspwqe: Pointer to driver response WQE object.
3051	*
3052	* The function is called from SLI ring event handler with no
3053	* lock held. This function is the completion handler for NVME ABTS for FCP cmds
3054	* The function frees memory resources used for the NVME commands.
3055	**/
3056	static void
3057	lpfc_nvmet_sol_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
3058	struct lpfc_iocbq *rspwqe)
3059	{
3060	struct lpfc_async_xchg_ctx *ctxp;
3061	struct lpfc_nvmet_tgtport *tgtp;
3062	uint32_t result;
3063	unsigned long flags;
3064	bool released = false;
3065	struct lpfc_wcqe_complete *wcqe = &rspwqe->wcqe_cmpl;
3066
3067	ctxp = cmdwqe->context_un.axchg;
3068	result = wcqe->parameter;
3069
3070	tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
3071	if (ctxp->flag & LPFC_NVME_ABORT_OP)
3072	atomic_inc(v: &tgtp->xmt_fcp_abort_cmpl);
3073
3074	spin_lock_irqsave(&ctxp->ctxlock, flags);
3075	ctxp->state = LPFC_NVME_STE_DONE;
3076
3077	/* Check if we already received a free context call
3078	* and we have completed processing an abort situation.
3079	*/
3080	if ((ctxp->flag & LPFC_NVME_CTX_RLS) &&
3081	!(ctxp->flag & LPFC_NVME_XBUSY)) {
3082	spin_lock(lock: &phba->sli4_hba.abts_nvmet_buf_list_lock);
3083	list_del_init(entry: &ctxp->list);
3084	spin_unlock(lock: &phba->sli4_hba.abts_nvmet_buf_list_lock);
3085	released = true;
3086	}
3087	ctxp->flag &= ~LPFC_NVME_ABORT_OP;
3088	spin_unlock_irqrestore(lock: &ctxp->ctxlock, flags);
3089	atomic_inc(v: &tgtp->xmt_abort_rsp);
3090
3091	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
3092	"6165 ABORT cmpl: oxid x%x flg x%x (%d) "
3093	"WCQE: %08x %08x %08x %08x\n",
3094	ctxp->oxid, ctxp->flag, released,
3095	wcqe->word0, wcqe->total_data_placed,
3096	result, wcqe->word3);
3097
3098	cmdwqe->rsp_dmabuf = NULL;
3099	cmdwqe->bpl_dmabuf = NULL;
3100	/*
3101	* if transport has released ctx, then can reuse it. Otherwise,
3102	* will be recycled by transport release call.
3103	*/
3104	if (released)
3105	lpfc_nvmet_ctxbuf_post(phba, ctx_buf: ctxp->ctxbuf);
3106
3107	/* This is the iocbq for the abort, not the command */
3108	lpfc_sli_release_iocbq(phba, cmdwqe);
3109
3110	/* Since iaab/iaar are NOT set, there is no work left.
3111	* For LPFC_NVME_XBUSY, lpfc_sli4_nvmet_xri_aborted
3112	* should have been called already.
3113	*/
3114	}
3115
3116	/**
3117	* lpfc_nvmet_unsol_fcp_abort_cmp - Completion handler for ABTS
3118	* @phba: Pointer to HBA context object.
3119	* @cmdwqe: Pointer to driver command WQE object.
3120	* @rspwqe: Pointer to driver response WQE object.
3121	*
3122	* The function is called from SLI ring event handler with no
3123	* lock held. This function is the completion handler for NVME ABTS for FCP cmds
3124	* The function frees memory resources used for the NVME commands.
3125	**/
3126	static void
3127	lpfc_nvmet_unsol_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
3128	struct lpfc_iocbq *rspwqe)
3129	{
3130	struct lpfc_async_xchg_ctx *ctxp;
3131	struct lpfc_nvmet_tgtport *tgtp;
3132	unsigned long flags;
3133	uint32_t result;
3134	bool released = false;
3135	struct lpfc_wcqe_complete *wcqe = &rspwqe->wcqe_cmpl;
3136
3137	ctxp = cmdwqe->context_un.axchg;
3138	result = wcqe->parameter;
3139
3140	if (!ctxp) {
3141	/* if context is clear, related io alrady complete */
3142	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
3143	"6070 ABTS cmpl: WCQE: %08x %08x %08x %08x\n",
3144	wcqe->word0, wcqe->total_data_placed,
3145	result, wcqe->word3);
3146	return;
3147	}
3148
3149	tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
3150	spin_lock_irqsave(&ctxp->ctxlock, flags);
3151	if (ctxp->flag & LPFC_NVME_ABORT_OP)
3152	atomic_inc(v: &tgtp->xmt_fcp_abort_cmpl);
3153
3154	/* Sanity check */
3155	if (ctxp->state != LPFC_NVME_STE_ABORT) {
3156	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
3157	"6112 ABTS Wrong state:%d oxid x%x\n",
3158	ctxp->state, ctxp->oxid);
3159	}
3160
3161	/* Check if we already received a free context call
3162	* and we have completed processing an abort situation.
3163	*/
3164	ctxp->state = LPFC_NVME_STE_DONE;
3165	if ((ctxp->flag & LPFC_NVME_CTX_RLS) &&
3166	!(ctxp->flag & LPFC_NVME_XBUSY)) {
3167	spin_lock(lock: &phba->sli4_hba.abts_nvmet_buf_list_lock);
3168	list_del_init(entry: &ctxp->list);
3169	spin_unlock(lock: &phba->sli4_hba.abts_nvmet_buf_list_lock);
3170	released = true;
3171	}
3172	ctxp->flag &= ~LPFC_NVME_ABORT_OP;
3173	spin_unlock_irqrestore(lock: &ctxp->ctxlock, flags);
3174	atomic_inc(v: &tgtp->xmt_abort_rsp);
3175
3176	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
3177	"6316 ABTS cmpl oxid x%x flg x%x (%x) "
3178	"WCQE: %08x %08x %08x %08x\n",
3179	ctxp->oxid, ctxp->flag, released,
3180	wcqe->word0, wcqe->total_data_placed,
3181	result, wcqe->word3);
3182
3183	cmdwqe->rsp_dmabuf = NULL;
3184	cmdwqe->bpl_dmabuf = NULL;
3185	/*
3186	* if transport has released ctx, then can reuse it. Otherwise,
3187	* will be recycled by transport release call.
3188	*/
3189	if (released)
3190	lpfc_nvmet_ctxbuf_post(phba, ctx_buf: ctxp->ctxbuf);
3191
3192	/* Since iaab/iaar are NOT set, there is no work left.
3193	* For LPFC_NVME_XBUSY, lpfc_sli4_nvmet_xri_aborted
3194	* should have been called already.
3195	*/
3196	}
3197
3198	/**
3199	* lpfc_nvmet_xmt_ls_abort_cmp - Completion handler for ABTS
3200	* @phba: Pointer to HBA context object.
3201	* @cmdwqe: Pointer to driver command WQE object.
3202	* @rspwqe: Pointer to driver response WQE object.
3203	*
3204	* The function is called from SLI ring event handler with no
3205	* lock held. This function is the completion handler for NVME ABTS for LS cmds
3206	* The function frees memory resources used for the NVME commands.
3207	**/
3208	static void
3209	lpfc_nvmet_xmt_ls_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
3210	struct lpfc_iocbq *rspwqe)
3211	{
3212	struct lpfc_async_xchg_ctx *ctxp;
3213	struct lpfc_nvmet_tgtport *tgtp;
3214	uint32_t result;
3215	struct lpfc_wcqe_complete *wcqe = &rspwqe->wcqe_cmpl;
3216
3217	ctxp = cmdwqe->context_un.axchg;
3218	result = wcqe->parameter;
3219
3220	if (phba->nvmet_support) {
3221	tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
3222	atomic_inc(v: &tgtp->xmt_ls_abort_cmpl);
3223	}
3224
3225	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
3226	"6083 Abort cmpl: ctx x%px WCQE:%08x %08x %08x %08x\n",
3227	ctxp, wcqe->word0, wcqe->total_data_placed,
3228	result, wcqe->word3);
3229
3230	if (!ctxp) {
3231	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
3232	"6415 NVMET LS Abort No ctx: WCQE: "
3233	"%08x %08x %08x %08x\n",
3234	wcqe->word0, wcqe->total_data_placed,
3235	result, wcqe->word3);
3236
3237	lpfc_sli_release_iocbq(phba, cmdwqe);
3238	return;
3239	}
3240
3241	if (ctxp->state != LPFC_NVME_STE_LS_ABORT) {
3242	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
3243	"6416 NVMET LS abort cmpl state mismatch: "
3244	"oxid x%x: %d %d\n",
3245	ctxp->oxid, ctxp->state, ctxp->entry_cnt);
3246	}
3247
3248	cmdwqe->rsp_dmabuf = NULL;
3249	cmdwqe->bpl_dmabuf = NULL;
3250	lpfc_sli_release_iocbq(phba, cmdwqe);
3251	kfree(objp: ctxp);
3252	}
3253
3254	static int
3255	lpfc_nvmet_unsol_issue_abort(struct lpfc_hba *phba,
3256	struct lpfc_async_xchg_ctx *ctxp,
3257	uint32_t sid, uint16_t xri)
3258	{
3259	struct lpfc_nvmet_tgtport *tgtp = NULL;
3260	struct lpfc_iocbq *abts_wqeq;
3261	union lpfc_wqe128 *wqe_abts;
3262	struct lpfc_nodelist *ndlp;
3263
3264	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
3265	"6067 ABTS: sid %x xri x%x/x%x\n",
3266	sid, xri, ctxp->wqeq->sli4_xritag);
3267
3268	if (phba->nvmet_support && phba->targetport)
3269	tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
3270
3271	ndlp = lpfc_findnode_did(phba->pport, sid);
3272	if (!ndlp ||
3273	((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) &&
3274	(ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {
3275	if (tgtp)
3276	atomic_inc(v: &tgtp->xmt_abort_rsp_error);
3277	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
3278	"6134 Drop ABTS - wrong NDLP state x%x.\n",
3279	(ndlp) ? ndlp->nlp_state : NLP_STE_MAX_STATE);
3280
3281	/* No failure to an ABTS request. */
3282	return 0;
3283	}
3284
3285	abts_wqeq = ctxp->wqeq;
3286	wqe_abts = &abts_wqeq->wqe;
3287
3288	/*
3289	* Since we zero the whole WQE, we need to ensure we set the WQE fields
3290	* that were initialized in lpfc_sli4_nvmet_alloc.
3291	*/
3292	memset(wqe_abts, 0, sizeof(union lpfc_wqe));
3293
3294	/* Word 5 */
3295	bf_set(wqe_dfctl, &wqe_abts->xmit_sequence.wge_ctl, 0);
3296	bf_set(wqe_ls, &wqe_abts->xmit_sequence.wge_ctl, 1);
3297	bf_set(wqe_la, &wqe_abts->xmit_sequence.wge_ctl, 0);
3298	bf_set(wqe_rctl, &wqe_abts->xmit_sequence.wge_ctl, FC_RCTL_BA_ABTS);
3299	bf_set(wqe_type, &wqe_abts->xmit_sequence.wge_ctl, FC_TYPE_BLS);
3300
3301	/* Word 6 */
3302	bf_set(wqe_ctxt_tag, &wqe_abts->xmit_sequence.wqe_com,
3303	phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
3304	bf_set(wqe_xri_tag, &wqe_abts->xmit_sequence.wqe_com,
3305	abts_wqeq->sli4_xritag);
3306
3307	/* Word 7 */
3308	bf_set(wqe_cmnd, &wqe_abts->xmit_sequence.wqe_com,
3309	CMD_XMIT_SEQUENCE64_WQE);
3310	bf_set(wqe_ct, &wqe_abts->xmit_sequence.wqe_com, SLI4_CT_RPI);
3311	bf_set(wqe_class, &wqe_abts->xmit_sequence.wqe_com, CLASS3);
3312	bf_set(wqe_pu, &wqe_abts->xmit_sequence.wqe_com, 0);
3313
3314	/* Word 8 */
3315	wqe_abts->xmit_sequence.wqe_com.abort_tag = abts_wqeq->iotag;
3316
3317	/* Word 9 */
3318	bf_set(wqe_reqtag, &wqe_abts->xmit_sequence.wqe_com, abts_wqeq->iotag);
3319	/* Needs to be set by caller */
3320	bf_set(wqe_rcvoxid, &wqe_abts->xmit_sequence.wqe_com, xri);
3321
3322	/* Word 10 */
3323	bf_set(wqe_iod, &wqe_abts->xmit_sequence.wqe_com, LPFC_WQE_IOD_WRITE);
3324	bf_set(wqe_lenloc, &wqe_abts->xmit_sequence.wqe_com,
3325	LPFC_WQE_LENLOC_WORD12);
3326	bf_set(wqe_ebde_cnt, &wqe_abts->xmit_sequence.wqe_com, 0);
3327	bf_set(wqe_qosd, &wqe_abts->xmit_sequence.wqe_com, 0);
3328
3329	/* Word 11 */
3330	bf_set(wqe_cqid, &wqe_abts->xmit_sequence.wqe_com,
3331	LPFC_WQE_CQ_ID_DEFAULT);
3332	bf_set(wqe_cmd_type, &wqe_abts->xmit_sequence.wqe_com,
3333	OTHER_COMMAND);
3334
3335	abts_wqeq->vport = phba->pport;
3336	abts_wqeq->ndlp = ndlp;
3337	abts_wqeq->context_un.axchg = ctxp;
3338	abts_wqeq->bpl_dmabuf = NULL;
3339	abts_wqeq->num_bdes = 0;
3340	/* hba_wqidx should already be setup from command we are aborting */
3341	abts_wqeq->iocb.ulpCommand = CMD_XMIT_SEQUENCE64_CR;
3342	abts_wqeq->iocb.ulpLe = 1;
3343
3344	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
3345	"6069 Issue ABTS to xri x%x reqtag x%x\n",
3346	xri, abts_wqeq->iotag);
3347	return 1;
3348	}
3349
3350	static int
3351	lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *phba,
3352	struct lpfc_async_xchg_ctx *ctxp,
3353	uint32_t sid, uint16_t xri)
3354	{
3355	struct lpfc_nvmet_tgtport *tgtp;
3356	struct lpfc_iocbq *abts_wqeq;
3357	struct lpfc_nodelist *ndlp;
3358	unsigned long flags;
3359	bool ia;
3360	int rc;
3361
3362	tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
3363	if (!ctxp->wqeq) {
3364	ctxp->wqeq = ctxp->ctxbuf->iocbq;
3365	ctxp->wqeq->hba_wqidx = 0;
3366	}
3367
3368	ndlp = lpfc_findnode_did(phba->pport, sid);
3369	if (!ndlp ||
3370	((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) &&
3371	(ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {
3372	atomic_inc(v: &tgtp->xmt_abort_rsp_error);
3373	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
3374	"6160 Drop ABORT - wrong NDLP state x%x.\n",
3375	(ndlp) ? ndlp->nlp_state : NLP_STE_MAX_STATE);
3376
3377	/* No failure to an ABTS request. */
3378	spin_lock_irqsave(&ctxp->ctxlock, flags);
3379	ctxp->flag &= ~LPFC_NVME_ABORT_OP;
3380	spin_unlock_irqrestore(lock: &ctxp->ctxlock, flags);
3381	return 0;
3382	}
3383
3384	/* Issue ABTS for this WQE based on iotag */
3385	ctxp->abort_wqeq = lpfc_sli_get_iocbq(phba);
3386	spin_lock_irqsave(&ctxp->ctxlock, flags);
3387	if (!ctxp->abort_wqeq) {
3388	atomic_inc(v: &tgtp->xmt_abort_rsp_error);
3389	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
3390	"6161 ABORT failed: No wqeqs: "
3391	"xri: x%x\n", ctxp->oxid);
3392	/* No failure to an ABTS request. */
3393	ctxp->flag &= ~LPFC_NVME_ABORT_OP;
3394	spin_unlock_irqrestore(lock: &ctxp->ctxlock, flags);
3395	return 0;
3396	}
3397	abts_wqeq = ctxp->abort_wqeq;
3398	ctxp->state = LPFC_NVME_STE_ABORT;
3399	ia = (ctxp->flag & LPFC_NVME_ABTS_RCV) ? true : false;
3400	spin_unlock_irqrestore(lock: &ctxp->ctxlock, flags);
3401
3402	/* Announce entry to new IO submit field. */
3403	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
3404	"6162 ABORT Request to rport DID x%06x "
3405	"for xri x%x x%x\n",
3406	ctxp->sid, ctxp->oxid, ctxp->wqeq->sli4_xritag);
3407
3408	/* If the hba is getting reset, this flag is set. It is
3409	* cleared when the reset is complete and rings reestablished.
3410	*/
3411	/* driver queued commands are in process of being flushed */
3412	if (test_bit(HBA_IOQ_FLUSH, &phba->hba_flag)) {
3413	atomic_inc(v: &tgtp->xmt_abort_rsp_error);
3414	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
3415	"6163 Driver in reset cleanup - flushing "
3416	"NVME Req now. hba_flag x%lx oxid x%x\n",
3417	phba->hba_flag, ctxp->oxid);
3418	lpfc_sli_release_iocbq(phba, abts_wqeq);
3419	spin_lock_irqsave(&ctxp->ctxlock, flags);
3420	ctxp->flag &= ~LPFC_NVME_ABORT_OP;
3421	spin_unlock_irqrestore(lock: &ctxp->ctxlock, flags);
3422	return 0;
3423	}
3424
3425	spin_lock_irqsave(&phba->hbalock, flags);
3426	/* Outstanding abort is in progress */
3427	if (abts_wqeq->cmd_flag & LPFC_DRIVER_ABORTED) {
3428	spin_unlock_irqrestore(lock: &phba->hbalock, flags);
3429	atomic_inc(v: &tgtp->xmt_abort_rsp_error);
3430	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
3431	"6164 Outstanding NVME I/O Abort Request "
3432	"still pending on oxid x%x\n",
3433	ctxp->oxid);
3434	lpfc_sli_release_iocbq(phba, abts_wqeq);
3435	spin_lock_irqsave(&ctxp->ctxlock, flags);
3436	ctxp->flag &= ~LPFC_NVME_ABORT_OP;
3437	spin_unlock_irqrestore(lock: &ctxp->ctxlock, flags);
3438	return 0;
3439	}
3440
3441	/* Ready - mark outstanding as aborted by driver. */
3442	abts_wqeq->cmd_flag |= LPFC_DRIVER_ABORTED;
3443
3444	lpfc_sli_prep_abort_xri(phba, cmdiocbq: abts_wqeq, ulp_context: ctxp->wqeq->sli4_xritag,
3445	iotag: abts_wqeq->iotag, CLASS3,
3446	LPFC_WQE_CQ_ID_DEFAULT, ia, wqec: true);
3447
3448	/* ABTS WQE must go to the same WQ as the WQE to be aborted */
3449	abts_wqeq->hba_wqidx = ctxp->wqeq->hba_wqidx;
3450	abts_wqeq->cmd_cmpl = lpfc_nvmet_sol_fcp_abort_cmp;
3451	abts_wqeq->cmd_flag |= LPFC_IO_NVME;
3452	abts_wqeq->context_un.axchg = ctxp;
3453	abts_wqeq->vport = phba->pport;
3454	if (!ctxp->hdwq)
3455	ctxp->hdwq = &phba->sli4_hba.hdwq[abts_wqeq->hba_wqidx];
3456
3457	rc = lpfc_sli4_issue_wqe(phba, qp: ctxp->hdwq, pwqe: abts_wqeq);
3458	spin_unlock_irqrestore(lock: &phba->hbalock, flags);
3459	if (rc == WQE_SUCCESS) {
3460	atomic_inc(v: &tgtp->xmt_abort_sol);
3461	return 0;
3462	}
3463
3464	atomic_inc(v: &tgtp->xmt_abort_rsp_error);
3465	spin_lock_irqsave(&ctxp->ctxlock, flags);
3466	ctxp->flag &= ~LPFC_NVME_ABORT_OP;
3467	spin_unlock_irqrestore(lock: &ctxp->ctxlock, flags);
3468	lpfc_sli_release_iocbq(phba, abts_wqeq);
3469	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
3470	"6166 Failed ABORT issue_wqe with status x%x "
3471	"for oxid x%x.\n",
3472	rc, ctxp->oxid);
3473	return 1;
3474	}
3475
3476	static int
3477	lpfc_nvmet_unsol_fcp_issue_abort(struct lpfc_hba *phba,
3478	struct lpfc_async_xchg_ctx *ctxp,
3479	uint32_t sid, uint16_t xri)
3480	{
3481	struct lpfc_nvmet_tgtport *tgtp;
3482	struct lpfc_iocbq *abts_wqeq;
3483	unsigned long flags;
3484	bool released = false;
3485	int rc;
3486
3487	tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
3488	if (!ctxp->wqeq) {
3489	ctxp->wqeq = ctxp->ctxbuf->iocbq;
3490	ctxp->wqeq->hba_wqidx = 0;
3491	}
3492
3493	if (ctxp->state == LPFC_NVME_STE_FREE) {
3494	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
3495	"6417 NVMET ABORT ctx freed %d %d oxid x%x\n",
3496	ctxp->state, ctxp->entry_cnt, ctxp->oxid);
3497	rc = WQE_BUSY;
3498	goto aerr;
3499	}
3500	ctxp->state = LPFC_NVME_STE_ABORT;
3501	ctxp->entry_cnt++;
3502	rc = lpfc_nvmet_unsol_issue_abort(phba, ctxp, sid, xri);
3503	if (rc == 0)
3504	goto aerr;
3505
3506	spin_lock_irqsave(&phba->hbalock, flags);
3507	abts_wqeq = ctxp->wqeq;
3508	abts_wqeq->cmd_cmpl = lpfc_nvmet_unsol_fcp_abort_cmp;
3509	abts_wqeq->cmd_flag |= LPFC_IO_NVMET;
3510	if (!ctxp->hdwq)
3511	ctxp->hdwq = &phba->sli4_hba.hdwq[abts_wqeq->hba_wqidx];
3512
3513	rc = lpfc_sli4_issue_wqe(phba, qp: ctxp->hdwq, pwqe: abts_wqeq);
3514	spin_unlock_irqrestore(lock: &phba->hbalock, flags);
3515	if (rc == WQE_SUCCESS) {
3516	return 0;
3517	}
3518
3519	aerr:
3520	spin_lock_irqsave(&ctxp->ctxlock, flags);
3521	if (ctxp->flag & LPFC_NVME_CTX_RLS) {
3522	spin_lock(lock: &phba->sli4_hba.abts_nvmet_buf_list_lock);
3523	list_del_init(entry: &ctxp->list);
3524	spin_unlock(lock: &phba->sli4_hba.abts_nvmet_buf_list_lock);
3525	released = true;
3526	}
3527	ctxp->flag &= ~(LPFC_NVME_ABORT_OP | LPFC_NVME_CTX_RLS);
3528	spin_unlock_irqrestore(lock: &ctxp->ctxlock, flags);
3529
3530	atomic_inc(v: &tgtp->xmt_abort_rsp_error);
3531	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
3532	"6135 Failed to Issue ABTS for oxid x%x. Status x%x "
3533	"(%x)\n",
3534	ctxp->oxid, rc, released);
3535	if (released)
3536	lpfc_nvmet_ctxbuf_post(phba, ctx_buf: ctxp->ctxbuf);
3537	return 1;
3538	}
3539
3540	/**
3541	* lpfc_nvme_unsol_ls_issue_abort - issue ABTS on an exchange received
3542	* via async frame receive where the frame is not handled.
3543	* @phba: pointer to adapter structure
3544	* @ctxp: pointer to the asynchronously received received sequence
3545	* @sid: address of the remote port to send the ABTS to
3546	* @xri: oxid value to for the ABTS (other side's exchange id).
3547	**/
3548	int
3549	lpfc_nvme_unsol_ls_issue_abort(struct lpfc_hba *phba,
3550	struct lpfc_async_xchg_ctx *ctxp,
3551	uint32_t sid, uint16_t xri)
3552	{
3553	struct lpfc_nvmet_tgtport *tgtp = NULL;
3554	struct lpfc_iocbq *abts_wqeq;
3555	unsigned long flags;
3556	int rc;
3557
3558	if ((ctxp->state == LPFC_NVME_STE_LS_RCV && ctxp->entry_cnt == 1) ||
3559	(ctxp->state == LPFC_NVME_STE_LS_RSP && ctxp->entry_cnt == 2)) {
3560	ctxp->state = LPFC_NVME_STE_LS_ABORT;
3561	ctxp->entry_cnt++;
3562	} else {
3563	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
3564	"6418 NVMET LS abort state mismatch "
3565	"IO x%x: %d %d\n",
3566	ctxp->oxid, ctxp->state, ctxp->entry_cnt);
3567	ctxp->state = LPFC_NVME_STE_LS_ABORT;
3568	}
3569
3570	if (phba->nvmet_support && phba->targetport)
3571	tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
3572
3573	if (!ctxp->wqeq) {
3574	/* Issue ABTS for this WQE based on iotag */
3575	ctxp->wqeq = lpfc_sli_get_iocbq(phba);
3576	if (!ctxp->wqeq) {
3577	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
3578	"6068 Abort failed: No wqeqs: "
3579	"xri: x%x\n", xri);
3580	/* No failure to an ABTS request. */
3581	kfree(objp: ctxp);
3582	return 0;
3583	}
3584	}
3585	abts_wqeq = ctxp->wqeq;
3586
3587	if (lpfc_nvmet_unsol_issue_abort(phba, ctxp, sid, xri) == 0) {
3588	rc = WQE_BUSY;
3589	goto out;
3590	}
3591
3592	spin_lock_irqsave(&phba->hbalock, flags);
3593	abts_wqeq->cmd_cmpl = lpfc_nvmet_xmt_ls_abort_cmp;
3594	abts_wqeq->cmd_flag |= LPFC_IO_NVME_LS;
3595	rc = lpfc_sli4_issue_wqe(phba, qp: ctxp->hdwq, pwqe: abts_wqeq);
3596	spin_unlock_irqrestore(lock: &phba->hbalock, flags);
3597	if (rc == WQE_SUCCESS) {
3598	if (tgtp)
3599	atomic_inc(v: &tgtp->xmt_abort_unsol);
3600	return 0;
3601	}
3602	out:
3603	if (tgtp)
3604	atomic_inc(v: &tgtp->xmt_abort_rsp_error);
3605	abts_wqeq->rsp_dmabuf = NULL;
3606	abts_wqeq->bpl_dmabuf = NULL;
3607	lpfc_sli_release_iocbq(phba, abts_wqeq);
3608	lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
3609	"6056 Failed to Issue ABTS. Status x%x\n", rc);
3610	return 1;
3611	}
3612
3613	/**
3614	* lpfc_nvmet_invalidate_host
3615	*
3616	* @phba: pointer to the driver instance bound to an adapter port.
3617	* @ndlp: pointer to an lpfc_nodelist type
3618	*
3619	* This routine upcalls the nvmet transport to invalidate an NVME
3620	* host to which this target instance had active connections.
3621	*/
3622	void
3623	lpfc_nvmet_invalidate_host(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp)
3624	{
3625	u32 ndlp_has_hh;
3626	struct lpfc_nvmet_tgtport *tgtp;
3627
3628	lpfc_printf_log(phba, KERN_INFO,
3629	LOG_NVME | LOG_NVME_ABTS | LOG_NVME_DISC,
3630	"6203 Invalidating hosthandle x%px\n",
3631	ndlp);
3632
3633	tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
3634	atomic_set(v: &tgtp->state, LPFC_NVMET_INV_HOST_ACTIVE);
3635
3636	spin_lock_irq(lock: &ndlp->lock);
3637	ndlp_has_hh = ndlp->fc4_xpt_flags & NLP_XPT_HAS_HH;
3638	spin_unlock_irq(lock: &ndlp->lock);
3639
3640	/* Do not invalidate any nodes that do not have a hosthandle.
3641	* The host_release callbk will cause a node reference
3642	* count imbalance and a crash.
3643	*/
3644	if (!ndlp_has_hh) {
3645	lpfc_printf_log(phba, KERN_INFO,
3646	LOG_NVME | LOG_NVME_ABTS | LOG_NVME_DISC,
3647	"6204 Skip invalidate on node x%px DID x%x\n",
3648	ndlp, ndlp->nlp_DID);
3649	return;
3650	}
3651
3652	#if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
3653	/* Need to get the nvmet_fc_target_port pointer here.*/
3654	nvmet_fc_invalidate_host(tgtport: phba->targetport, hosthandle: ndlp);
3655	#endif
3656	}
3657