1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* Linux MegaRAID driver for SAS based RAID controllers
4	*
5	* Copyright (c) 2009-2013 LSI Corporation
6	* Copyright (c) 2013-2016 Avago Technologies
7	* Copyright (c) 2016-2018 Broadcom Inc.
8	*
9	* FILE: megaraid_sas_fusion.c
10	*
11	* Authors: Broadcom Inc.
12	* Sumant Patro
13	* Adam Radford
14	* Kashyap Desai <kashyap.desai@broadcom.com>
15	* Sumit Saxena <sumit.saxena@broadcom.com>
16	*
17	* Send feedback to: megaraidlinux.pdl@broadcom.com
18	*/
19
20	#include <linux/kernel.h>
21	#include <linux/types.h>
22	#include <linux/pci.h>
23	#include <linux/list.h>
24	#include <linux/moduleparam.h>
25	#include <linux/module.h>
26	#include <linux/spinlock.h>
27	#include <linux/interrupt.h>
28	#include <linux/delay.h>
29	#include <linux/uio.h>
30	#include <linux/uaccess.h>
31	#include <linux/fs.h>
32	#include <linux/compat.h>
33	#include <linux/blkdev.h>
34	#include <linux/mutex.h>
35	#include <linux/poll.h>
36	#include <linux/vmalloc.h>
37	#include <linux/workqueue.h>
38	#include <linux/irq_poll.h>
39
40	#include <scsi/scsi.h>
41	#include <scsi/scsi_cmnd.h>
42	#include <scsi/scsi_device.h>
43	#include <scsi/scsi_host.h>
44	#include <scsi/scsi_dbg.h>
45	#include <linux/dmi.h>
46
47	#include "megaraid_sas_fusion.h"
48	#include "megaraid_sas.h"
49
50
51	extern void
52	megasas_complete_cmd(struct megasas_instance *instance,
53	struct megasas_cmd *cmd, u8 alt_status);
54	int
55	wait_and_poll(struct megasas_instance *instance, struct megasas_cmd *cmd,
56	int seconds);
57
58	int
59	megasas_clear_intr_fusion(struct megasas_instance *instance);
60
61	int megasas_transition_to_ready(struct megasas_instance *instance, int ocr);
62
63	extern u32 megasas_dbg_lvl;
64	int megasas_sriov_start_heartbeat(struct megasas_instance *instance,
65	int initial);
66	extern struct megasas_mgmt_info megasas_mgmt_info;
67	extern unsigned int resetwaittime;
68	extern unsigned int dual_qdepth_disable;
69	static void megasas_free_rdpq_fusion(struct megasas_instance *instance);
70	static void megasas_free_reply_fusion(struct megasas_instance *instance);
71	static inline
72	void megasas_configure_queue_sizes(struct megasas_instance *instance);
73	static void megasas_fusion_crash_dump(struct megasas_instance *instance);
74
75	/**
76	* megasas_adp_reset_wait_for_ready - initiate chip reset and wait for
77	* controller to come to ready state
78	* @instance: adapter's soft state
79	* @do_adp_reset: If true, do a chip reset
80	* @ocr_context: If called from OCR context this will
81	* be set to 1, else 0
82	*
83	* This function initiates a chip reset followed by a wait for controller to
84	* transition to ready state.
85	* During this, driver will block all access to PCI config space from userspace
86	*/
87	int
88	megasas_adp_reset_wait_for_ready(struct megasas_instance *instance,
89	bool do_adp_reset,
90	int ocr_context)
91	{
92	int ret = FAILED;
93
94	/*
95	* Block access to PCI config space from userspace
96	* when diag reset is initiated from driver
97	*/
98	if (megasas_dbg_lvl & OCR_DEBUG)
99	dev_info(&instance->pdev->dev,
100	"Block access to PCI config space %s %d\n",
101	__func__, __LINE__);
102
103	pci_cfg_access_lock(dev: instance->pdev);
104
105	if (do_adp_reset) {
106	if (instance->instancet->adp_reset
107	(instance, instance->reg_set))
108	goto out;
109	}
110
111	/* Wait for FW to become ready */
112	if (megasas_transition_to_ready(instance, ocr: ocr_context)) {
113	dev_warn(&instance->pdev->dev,
114	"Failed to transition controller to ready for scsi%d.\n",
115	instance->host->host_no);
116	goto out;
117	}
118
119	ret = SUCCESS;
120	out:
121	if (megasas_dbg_lvl & OCR_DEBUG)
122	dev_info(&instance->pdev->dev,
123	"Unlock access to PCI config space %s %d\n",
124	__func__, __LINE__);
125
126	pci_cfg_access_unlock(dev: instance->pdev);
127
128	return ret;
129	}
130
131	/**
132	* megasas_check_same_4gb_region - check if allocation
133	* crosses same 4GB boundary or not
134	* @instance: adapter's soft instance
135	* @start_addr: start address of DMA allocation
136	* @size: size of allocation in bytes
137	* @return: true : allocation does not cross same
138	* 4GB boundary
139	* false: allocation crosses same
140	* 4GB boundary
141	*/
142	static inline bool megasas_check_same_4gb_region
143	(struct megasas_instance *instance, dma_addr_t start_addr, size_t size)
144	{
145	dma_addr_t end_addr;
146
147	end_addr = start_addr + size;
148
149	if (upper_32_bits(start_addr) != upper_32_bits(end_addr)) {
150	dev_err(&instance->pdev->dev,
151	"Failed to get same 4GB boundary: start_addr: 0x%llx end_addr: 0x%llx\n",
152	(unsigned long long)start_addr,
153	(unsigned long long)end_addr);
154	return false;
155	}
156
157	return true;
158	}
159
160	/**
161	* megasas_enable_intr_fusion - Enables interrupts
162	* @instance: adapter's soft instance
163	*/
164	static void
165	megasas_enable_intr_fusion(struct megasas_instance *instance)
166	{
167	struct megasas_register_set __iomem *regs;
168	regs = instance->reg_set;
169
170	instance->mask_interrupts = 0;
171	/* For Thunderbolt/Invader also clear intr on enable */
172	writel(val: ~0, addr: &regs->outbound_intr_status);
173	readl(addr: &regs->outbound_intr_status);
174
175	writel(val: ~MFI_FUSION_ENABLE_INTERRUPT_MASK, addr: &(regs)->outbound_intr_mask);
176
177	/* Dummy readl to force pci flush */
178	dev_info(&instance->pdev->dev, "%s is called outbound_intr_mask:0x%08x\n",
179	__func__, readl(&regs->outbound_intr_mask));
180	}
181
182	/**
183	* megasas_disable_intr_fusion - Disables interrupt
184	* @instance: adapter's soft instance
185	*/
186	static void
187	megasas_disable_intr_fusion(struct megasas_instance *instance)
188	{
189	u32 mask = 0xFFFFFFFF;
190	struct megasas_register_set __iomem *regs;
191	regs = instance->reg_set;
192	instance->mask_interrupts = 1;
193
194	writel(val: mask, addr: &regs->outbound_intr_mask);
195	/* Dummy readl to force pci flush */
196	dev_info(&instance->pdev->dev, "%s is called outbound_intr_mask:0x%08x\n",
197	__func__, readl(&regs->outbound_intr_mask));
198	}
199
200	int
201	megasas_clear_intr_fusion(struct megasas_instance *instance)
202	{
203	u32 status;
204	struct megasas_register_set __iomem *regs;
205	regs = instance->reg_set;
206	/*
207	* Check if it is our interrupt
208	*/
209	status = megasas_readl(instance,
210	addr: &regs->outbound_intr_status);
211
212	if (status & 1) {
213	writel(val: status, addr: &regs->outbound_intr_status);
214	readl(addr: &regs->outbound_intr_status);
215	return 1;
216	}
217	if (!(status & MFI_FUSION_ENABLE_INTERRUPT_MASK))
218	return 0;
219
220	return 1;
221	}
222
223	static inline void
224	megasas_sdev_busy_inc(struct megasas_instance *instance,
225	struct scsi_cmnd *scmd)
226	{
227	if (instance->perf_mode == MR_BALANCED_PERF_MODE) {
228	struct MR_PRIV_DEVICE *mr_device_priv_data =
229	scmd->device->hostdata;
230	atomic_inc(v: &mr_device_priv_data->sdev_priv_busy);
231	}
232	}
233
234	static inline void
235	megasas_sdev_busy_dec(struct megasas_instance *instance,
236	struct scsi_cmnd *scmd)
237	{
238	if (instance->perf_mode == MR_BALANCED_PERF_MODE) {
239	struct MR_PRIV_DEVICE *mr_device_priv_data =
240	scmd->device->hostdata;
241	atomic_dec(v: &mr_device_priv_data->sdev_priv_busy);
242	}
243	}
244
245	static inline int
246	megasas_sdev_busy_read(struct megasas_instance *instance,
247	struct scsi_cmnd *scmd)
248	{
249	if (instance->perf_mode == MR_BALANCED_PERF_MODE) {
250	struct MR_PRIV_DEVICE *mr_device_priv_data =
251	scmd->device->hostdata;
252	return atomic_read(v: &mr_device_priv_data->sdev_priv_busy);
253	}
254	return 0;
255	}
256
257	/**
258	* megasas_get_cmd_fusion - Get a command from the free pool
259	* @instance: Adapter soft state
260	* @blk_tag: Command tag
261	*
262	* Returns a blk_tag indexed mpt frame
263	*/
264	inline struct megasas_cmd_fusion *megasas_get_cmd_fusion(struct megasas_instance
265	*instance, u32 blk_tag)
266	{
267	struct fusion_context *fusion;
268
269	fusion = instance->ctrl_context;
270	return fusion->cmd_list[blk_tag];
271	}
272
273	/**
274	* megasas_return_cmd_fusion - Return a cmd to free command pool
275	* @instance: Adapter soft state
276	* @cmd: Command packet to be returned to free command pool
277	*/
278	inline void megasas_return_cmd_fusion(struct megasas_instance *instance,
279	struct megasas_cmd_fusion *cmd)
280	{
281	cmd->scmd = NULL;
282	memset(cmd->io_request, 0, MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE);
283	cmd->r1_alt_dev_handle = MR_DEVHANDLE_INVALID;
284	cmd->cmd_completed = false;
285	}
286
287	/**
288	* megasas_write_64bit_req_desc - PCI writes 64bit request descriptor
289	* @instance: Adapter soft state
290	* @req_desc: 64bit Request descriptor
291	*/
292	static void
293	megasas_write_64bit_req_desc(struct megasas_instance *instance,
294	union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc)
295	{
296	#if defined(writeq) && defined(CONFIG_64BIT)
297	u64 req_data = (((u64)le32_to_cpu(req_desc->u.high) << 32) |
298	le32_to_cpu(req_desc->u.low));
299	writeq(val: req_data, addr: &instance->reg_set->inbound_low_queue_port);
300	#else
301	unsigned long flags;
302	spin_lock_irqsave(&instance->hba_lock, flags);
303	writel(le32_to_cpu(req_desc->u.low),
304	&instance->reg_set->inbound_low_queue_port);
305	writel(le32_to_cpu(req_desc->u.high),
306	&instance->reg_set->inbound_high_queue_port);
307	spin_unlock_irqrestore(&instance->hba_lock, flags);
308	#endif
309	}
310
311	/**
312	* megasas_fire_cmd_fusion - Sends command to the FW
313	* @instance: Adapter soft state
314	* @req_desc: 32bit or 64bit Request descriptor
315	*
316	* Perform PCI Write. AERO SERIES supports 32 bit Descriptor.
317	* Prior to AERO_SERIES support 64 bit Descriptor.
318	*/
319	static void
320	megasas_fire_cmd_fusion(struct megasas_instance *instance,
321	union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc)
322	{
323	if (instance->atomic_desc_support)
324	writel(le32_to_cpu(req_desc->u.low),
325	addr: &instance->reg_set->inbound_single_queue_port);
326	else
327	megasas_write_64bit_req_desc(instance, req_desc);
328	}
329
330	/**
331	* megasas_fusion_update_can_queue - Do all Adapter Queue depth related calculations here
332	* @instance: Adapter soft state
333	* @fw_boot_context: Whether this function called during probe or after OCR
334	*
335	* This function is only for fusion controllers.
336	* Update host can queue, if firmware downgrade max supported firmware commands.
337	* Firmware upgrade case will be skipped because underlying firmware has
338	* more resource than exposed to the OS.
339	*
340	*/
341	static void
342	megasas_fusion_update_can_queue(struct megasas_instance *instance, int fw_boot_context)
343	{
344	u16 cur_max_fw_cmds = 0;
345	u16 ldio_threshold = 0;
346
347	/* ventura FW does not fill outbound_scratch_pad_2 with queue depth */
348	if (instance->adapter_type < VENTURA_SERIES)
349	cur_max_fw_cmds =
350	megasas_readl(instance,
351	addr: &instance->reg_set->outbound_scratch_pad_2) & 0x00FFFF;
352
353	if (dual_qdepth_disable || !cur_max_fw_cmds)
354	cur_max_fw_cmds = instance->instancet->read_fw_status_reg(instance) & 0x00FFFF;
355	else
356	ldio_threshold =
357	(instance->instancet->read_fw_status_reg(instance) & 0x00FFFF) - MEGASAS_FUSION_IOCTL_CMDS;
358
359	dev_info(&instance->pdev->dev,
360	"Current firmware supports maximum commands: %d\t LDIO threshold: %d\n",
361	cur_max_fw_cmds, ldio_threshold);
362
363	if (fw_boot_context == OCR_CONTEXT) {
364	cur_max_fw_cmds = cur_max_fw_cmds - 1;
365	if (cur_max_fw_cmds < instance->max_fw_cmds) {
366	instance->cur_can_queue =
367	cur_max_fw_cmds - (MEGASAS_FUSION_INTERNAL_CMDS +
368	MEGASAS_FUSION_IOCTL_CMDS);
369	instance->host->can_queue = instance->cur_can_queue;
370	instance->ldio_threshold = ldio_threshold;
371	}
372	} else {
373	instance->max_fw_cmds = cur_max_fw_cmds;
374	instance->ldio_threshold = ldio_threshold;
375
376	if (reset_devices)
377	instance->max_fw_cmds = min(instance->max_fw_cmds,
378	(u16)MEGASAS_KDUMP_QUEUE_DEPTH);
379	/*
380	* Reduce the max supported cmds by 1. This is to ensure that the
381	* reply_q_sz (1 more than the max cmd that driver may send)
382	* does not exceed max cmds that the FW can support
383	*/
384	instance->max_fw_cmds = instance->max_fw_cmds-1;
385	}
386	}
387
388	static inline void
389	megasas_get_msix_index(struct megasas_instance *instance,
390	struct scsi_cmnd *scmd,
391	struct megasas_cmd_fusion *cmd,
392	u8 data_arms)
393	{
394	if (instance->perf_mode == MR_BALANCED_PERF_MODE &&
395	(megasas_sdev_busy_read(instance, scmd) >
396	(data_arms * MR_DEVICE_HIGH_IOPS_DEPTH))) {
397	cmd->request_desc->SCSIIO.MSIxIndex =
398	mega_mod64(dividend: (atomic64_add_return(i: 1, v: &instance->high_iops_outstanding) /
399	MR_HIGH_IOPS_BATCH_COUNT), divisor: instance->low_latency_index_start);
400	} else if (instance->msix_load_balance) {
401	cmd->request_desc->SCSIIO.MSIxIndex =
402	(mega_mod64(dividend: atomic64_add_return(i: 1, v: &instance->total_io_count),
403	divisor: instance->msix_vectors));
404	} else if (instance->host->nr_hw_queues > 1) {
405	u32 tag = blk_mq_unique_tag(rq: scsi_cmd_to_rq(scmd));
406
407	cmd->request_desc->SCSIIO.MSIxIndex = blk_mq_unique_tag_to_hwq(unique_tag: tag) +
408	instance->low_latency_index_start;
409	} else {
410	cmd->request_desc->SCSIIO.MSIxIndex =
411	instance->reply_map[raw_smp_processor_id()];
412	}
413	}
414
415	/**
416	* megasas_free_cmds_fusion - Free all the cmds in the free cmd pool
417	* @instance: Adapter soft state
418	*/
419	void
420	megasas_free_cmds_fusion(struct megasas_instance *instance)
421	{
422	int i;
423	struct fusion_context *fusion = instance->ctrl_context;
424	struct megasas_cmd_fusion *cmd;
425
426	if (fusion->sense)
427	dma_pool_free(pool: fusion->sense_dma_pool, vaddr: fusion->sense,
428	addr: fusion->sense_phys_addr);
429
430	/* SG */
431	if (fusion->cmd_list) {
432	for (i = 0; i < instance->max_mpt_cmds; i++) {
433	cmd = fusion->cmd_list[i];
434	if (cmd) {
435	if (cmd->sg_frame)
436	dma_pool_free(pool: fusion->sg_dma_pool,
437	vaddr: cmd->sg_frame,
438	addr: cmd->sg_frame_phys_addr);
439	}
440	kfree(objp: cmd);
441	}
442	kfree(objp: fusion->cmd_list);
443	}
444
445	if (fusion->sg_dma_pool) {
446	dma_pool_destroy(pool: fusion->sg_dma_pool);
447	fusion->sg_dma_pool = NULL;
448	}
449	if (fusion->sense_dma_pool) {
450	dma_pool_destroy(pool: fusion->sense_dma_pool);
451	fusion->sense_dma_pool = NULL;
452	}
453
454
455	/* Reply Frame, Desc*/
456	if (instance->is_rdpq)
457	megasas_free_rdpq_fusion(instance);
458	else
459	megasas_free_reply_fusion(instance);
460
461	/* Request Frame, Desc*/
462	if (fusion->req_frames_desc)
463	dma_free_coherent(dev: &instance->pdev->dev,
464	size: fusion->request_alloc_sz, cpu_addr: fusion->req_frames_desc,
465	dma_handle: fusion->req_frames_desc_phys);
466	if (fusion->io_request_frames)
467	dma_pool_free(pool: fusion->io_request_frames_pool,
468	vaddr: fusion->io_request_frames,
469	addr: fusion->io_request_frames_phys);
470	if (fusion->io_request_frames_pool) {
471	dma_pool_destroy(pool: fusion->io_request_frames_pool);
472	fusion->io_request_frames_pool = NULL;
473	}
474	}
475
476	/**
477	* megasas_create_sg_sense_fusion - Creates DMA pool for cmd frames
478	* @instance: Adapter soft state
479	*
480	*/
481	static int megasas_create_sg_sense_fusion(struct megasas_instance *instance)
482	{
483	int i;
484	u16 max_cmd;
485	struct fusion_context *fusion;
486	struct megasas_cmd_fusion *cmd;
487	int sense_sz;
488	u32 offset;
489
490	fusion = instance->ctrl_context;
491	max_cmd = instance->max_fw_cmds;
492	sense_sz = instance->max_mpt_cmds * SCSI_SENSE_BUFFERSIZE;
493
494	fusion->sg_dma_pool =
495	dma_pool_create(name: "mr_sg", dev: &instance->pdev->dev,
496	size: instance->max_chain_frame_sz,
497	MR_DEFAULT_NVME_PAGE_SIZE, allocation: 0);
498	/* SCSI_SENSE_BUFFERSIZE = 96 bytes */
499	fusion->sense_dma_pool =
500	dma_pool_create(name: "mr_sense", dev: &instance->pdev->dev,
501	size: sense_sz, align: 64, allocation: 0);
502
503	if (!fusion->sense_dma_pool || !fusion->sg_dma_pool) {
504	dev_err(&instance->pdev->dev,
505	"Failed from %s %d\n", __func__, __LINE__);
506	return -ENOMEM;
507	}
508
509	fusion->sense = dma_pool_alloc(pool: fusion->sense_dma_pool,
510	GFP_KERNEL, handle: &fusion->sense_phys_addr);
511	if (!fusion->sense) {
512	dev_err(&instance->pdev->dev,
513	"failed from %s %d\n", __func__, __LINE__);
514	return -ENOMEM;
515	}
516
517	/* sense buffer, request frame and reply desc pool requires to be in
518	* same 4 gb region. Below function will check this.
519	* In case of failure, new pci pool will be created with updated
520	* alignment.
521	* Older allocation and pool will be destroyed.
522	* Alignment will be used such a way that next allocation if success,
523	* will always meet same 4gb region requirement.
524	* Actual requirement is not alignment, but we need start and end of
525	* DMA address must have same upper 32 bit address.
526	*/
527
528	if (!megasas_check_same_4gb_region(instance, start_addr: fusion->sense_phys_addr,
529	size: sense_sz)) {
530	dma_pool_free(pool: fusion->sense_dma_pool, vaddr: fusion->sense,
531	addr: fusion->sense_phys_addr);
532	fusion->sense = NULL;
533	dma_pool_destroy(pool: fusion->sense_dma_pool);
534
535	fusion->sense_dma_pool =
536	dma_pool_create(name: "mr_sense_align", dev: &instance->pdev->dev,
537	size: sense_sz, roundup_pow_of_two(sense_sz),
538	allocation: 0);
539	if (!fusion->sense_dma_pool) {
540	dev_err(&instance->pdev->dev,
541	"Failed from %s %d\n", __func__, __LINE__);
542	return -ENOMEM;
543	}
544	fusion->sense = dma_pool_alloc(pool: fusion->sense_dma_pool,
545	GFP_KERNEL,
546	handle: &fusion->sense_phys_addr);
547	if (!fusion->sense) {
548	dev_err(&instance->pdev->dev,
549	"failed from %s %d\n", __func__, __LINE__);
550	return -ENOMEM;
551	}
552	}
553
554	/*
555	* Allocate and attach a frame to each of the commands in cmd_list
556	*/
557	for (i = 0; i < max_cmd; i++) {
558	cmd = fusion->cmd_list[i];
559	cmd->sg_frame = dma_pool_alloc(pool: fusion->sg_dma_pool,
560	GFP_KERNEL, handle: &cmd->sg_frame_phys_addr);
561
562	offset = SCSI_SENSE_BUFFERSIZE * i;
563	cmd->sense = (u8 *)fusion->sense + offset;
564	cmd->sense_phys_addr = fusion->sense_phys_addr + offset;
565
566	if (!cmd->sg_frame) {
567	dev_err(&instance->pdev->dev,
568	"Failed from %s %d\n", __func__, __LINE__);
569	return -ENOMEM;
570	}
571	}
572
573	/* create sense buffer for the raid 1/10 fp */
574	for (i = max_cmd; i < instance->max_mpt_cmds; i++) {
575	cmd = fusion->cmd_list[i];
576	offset = SCSI_SENSE_BUFFERSIZE * i;
577	cmd->sense = (u8 *)fusion->sense + offset;
578	cmd->sense_phys_addr = fusion->sense_phys_addr + offset;
579
580	}
581
582	return 0;
583	}
584
585	static int
586	megasas_alloc_cmdlist_fusion(struct megasas_instance *instance)
587	{
588	u32 max_mpt_cmd, i, j;
589	struct fusion_context *fusion;
590
591	fusion = instance->ctrl_context;
592
593	max_mpt_cmd = instance->max_mpt_cmds;
594
595	/*
596	* fusion->cmd_list is an array of struct megasas_cmd_fusion pointers.
597	* Allocate the dynamic array first and then allocate individual
598	* commands.
599	*/
600	fusion->cmd_list =
601	kcalloc(n: max_mpt_cmd, size: sizeof(struct megasas_cmd_fusion *),
602	GFP_KERNEL);
603	if (!fusion->cmd_list) {
604	dev_err(&instance->pdev->dev,
605	"Failed from %s %d\n", __func__, __LINE__);
606	return -ENOMEM;
607	}
608
609	for (i = 0; i < max_mpt_cmd; i++) {
610	fusion->cmd_list[i] = kzalloc(size: sizeof(struct megasas_cmd_fusion),
611	GFP_KERNEL);
612	if (!fusion->cmd_list[i]) {
613	for (j = 0; j < i; j++)
614	kfree(objp: fusion->cmd_list[j]);
615	kfree(objp: fusion->cmd_list);
616	dev_err(&instance->pdev->dev,
617	"Failed from %s %d\n", __func__, __LINE__);
618	return -ENOMEM;
619	}
620	}
621
622	return 0;
623	}
624
625	static int
626	megasas_alloc_request_fusion(struct megasas_instance *instance)
627	{
628	struct fusion_context *fusion;
629
630	fusion = instance->ctrl_context;
631
632	retry_alloc:
633	fusion->io_request_frames_pool =
634	dma_pool_create(name: "mr_ioreq", dev: &instance->pdev->dev,
635	size: fusion->io_frames_alloc_sz, align: 16, allocation: 0);
636
637	if (!fusion->io_request_frames_pool) {
638	dev_err(&instance->pdev->dev,
639	"Failed from %s %d\n", __func__, __LINE__);
640	return -ENOMEM;
641	}
642
643	fusion->io_request_frames =
644	dma_pool_alloc(pool: fusion->io_request_frames_pool,
645	GFP_KERNEL | __GFP_NOWARN,
646	handle: &fusion->io_request_frames_phys);
647	if (!fusion->io_request_frames) {
648	if (instance->max_fw_cmds >= (MEGASAS_REDUCE_QD_COUNT * 2)) {
649	instance->max_fw_cmds -= MEGASAS_REDUCE_QD_COUNT;
650	dma_pool_destroy(pool: fusion->io_request_frames_pool);
651	megasas_configure_queue_sizes(instance);
652	goto retry_alloc;
653	} else {
654	dev_err(&instance->pdev->dev,
655	"Failed from %s %d\n", __func__, __LINE__);
656	return -ENOMEM;
657	}
658	}
659
660	if (!megasas_check_same_4gb_region(instance,
661	start_addr: fusion->io_request_frames_phys,
662	size: fusion->io_frames_alloc_sz)) {
663	dma_pool_free(pool: fusion->io_request_frames_pool,
664	vaddr: fusion->io_request_frames,
665	addr: fusion->io_request_frames_phys);
666	fusion->io_request_frames = NULL;
667	dma_pool_destroy(pool: fusion->io_request_frames_pool);
668
669	fusion->io_request_frames_pool =
670	dma_pool_create(name: "mr_ioreq_align",
671	dev: &instance->pdev->dev,
672	size: fusion->io_frames_alloc_sz,
673	roundup_pow_of_two(fusion->io_frames_alloc_sz),
674	allocation: 0);
675
676	if (!fusion->io_request_frames_pool) {
677	dev_err(&instance->pdev->dev,
678	"Failed from %s %d\n", __func__, __LINE__);
679	return -ENOMEM;
680	}
681
682	fusion->io_request_frames =
683	dma_pool_alloc(pool: fusion->io_request_frames_pool,
684	GFP_KERNEL | __GFP_NOWARN,
685	handle: &fusion->io_request_frames_phys);
686
687	if (!fusion->io_request_frames) {
688	dev_err(&instance->pdev->dev,
689	"Failed from %s %d\n", __func__, __LINE__);
690	return -ENOMEM;
691	}
692	}
693
694	fusion->req_frames_desc =
695	dma_alloc_coherent(dev: &instance->pdev->dev,
696	size: fusion->request_alloc_sz,
697	dma_handle: &fusion->req_frames_desc_phys, GFP_KERNEL);
698	if (!fusion->req_frames_desc) {
699	dev_err(&instance->pdev->dev,
700	"Failed from %s %d\n", __func__, __LINE__);
701	return -ENOMEM;
702	}
703
704	return 0;
705	}
706
707	static int
708	megasas_alloc_reply_fusion(struct megasas_instance *instance)
709	{
710	int i, count;
711	struct fusion_context *fusion;
712	union MPI2_REPLY_DESCRIPTORS_UNION *reply_desc;
713	fusion = instance->ctrl_context;
714
715	count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
716	count += instance->iopoll_q_count;
717
718	fusion->reply_frames_desc_pool =
719	dma_pool_create(name: "mr_reply", dev: &instance->pdev->dev,
720	size: fusion->reply_alloc_sz * count, align: 16, allocation: 0);
721
722	if (!fusion->reply_frames_desc_pool) {
723	dev_err(&instance->pdev->dev,
724	"Failed from %s %d\n", __func__, __LINE__);
725	return -ENOMEM;
726	}
727
728	fusion->reply_frames_desc[0] =
729	dma_pool_alloc(pool: fusion->reply_frames_desc_pool,
730	GFP_KERNEL, handle: &fusion->reply_frames_desc_phys[0]);
731	if (!fusion->reply_frames_desc[0]) {
732	dev_err(&instance->pdev->dev,
733	"Failed from %s %d\n", __func__, __LINE__);
734	return -ENOMEM;
735	}
736
737	if (!megasas_check_same_4gb_region(instance,
738	start_addr: fusion->reply_frames_desc_phys[0],
739	size: (fusion->reply_alloc_sz * count))) {
740	dma_pool_free(pool: fusion->reply_frames_desc_pool,
741	vaddr: fusion->reply_frames_desc[0],
742	addr: fusion->reply_frames_desc_phys[0]);
743	fusion->reply_frames_desc[0] = NULL;
744	dma_pool_destroy(pool: fusion->reply_frames_desc_pool);
745
746	fusion->reply_frames_desc_pool =
747	dma_pool_create(name: "mr_reply_align",
748	dev: &instance->pdev->dev,
749	size: fusion->reply_alloc_sz * count,
750	roundup_pow_of_two(fusion->reply_alloc_sz * count),
751	allocation: 0);
752
753	if (!fusion->reply_frames_desc_pool) {
754	dev_err(&instance->pdev->dev,
755	"Failed from %s %d\n", __func__, __LINE__);
756	return -ENOMEM;
757	}
758
759	fusion->reply_frames_desc[0] =
760	dma_pool_alloc(pool: fusion->reply_frames_desc_pool,
761	GFP_KERNEL,
762	handle: &fusion->reply_frames_desc_phys[0]);
763
764	if (!fusion->reply_frames_desc[0]) {
765	dev_err(&instance->pdev->dev,
766	"Failed from %s %d\n", __func__, __LINE__);
767	return -ENOMEM;
768	}
769	}
770
771	reply_desc = fusion->reply_frames_desc[0];
772	for (i = 0; i < fusion->reply_q_depth * count; i++, reply_desc++)
773	reply_desc->Words = cpu_to_le64(ULLONG_MAX);
774
775	/* This is not a rdpq mode, but driver still populate
776	* reply_frame_desc array to use same msix index in ISR path.
777	*/
778	for (i = 0; i < (count - 1); i++)
779	fusion->reply_frames_desc[i + 1] =
780	fusion->reply_frames_desc[i] +
781	(fusion->reply_alloc_sz)/sizeof(union MPI2_REPLY_DESCRIPTORS_UNION);
782
783	return 0;
784	}
785
786	static int
787	megasas_alloc_rdpq_fusion(struct megasas_instance *instance)
788	{
789	int i, j, k, msix_count;
790	struct fusion_context *fusion;
791	union MPI2_REPLY_DESCRIPTORS_UNION *reply_desc;
792	union MPI2_REPLY_DESCRIPTORS_UNION *rdpq_chunk_virt[RDPQ_MAX_CHUNK_COUNT];
793	dma_addr_t rdpq_chunk_phys[RDPQ_MAX_CHUNK_COUNT];
794	u8 dma_alloc_count, abs_index;
795	u32 chunk_size, array_size, offset;
796
797	fusion = instance->ctrl_context;
798	chunk_size = fusion->reply_alloc_sz * RDPQ_MAX_INDEX_IN_ONE_CHUNK;
799	array_size = sizeof(struct MPI2_IOC_INIT_RDPQ_ARRAY_ENTRY) *
800	MAX_MSIX_QUEUES_FUSION;
801
802	fusion->rdpq_virt = dma_alloc_coherent(dev: &instance->pdev->dev,
803	size: array_size, dma_handle: &fusion->rdpq_phys,
804	GFP_KERNEL);
805	if (!fusion->rdpq_virt) {
806	dev_err(&instance->pdev->dev,
807	"Failed from %s %d\n", __func__, __LINE__);
808	return -ENOMEM;
809	}
810
811	msix_count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
812	msix_count += instance->iopoll_q_count;
813
814	fusion->reply_frames_desc_pool = dma_pool_create(name: "mr_rdpq",
815	dev: &instance->pdev->dev,
816	size: chunk_size, align: 16, allocation: 0);
817	fusion->reply_frames_desc_pool_align =
818	dma_pool_create(name: "mr_rdpq_align",
819	dev: &instance->pdev->dev,
820	size: chunk_size,
821	roundup_pow_of_two(chunk_size),
822	allocation: 0);
823
824	if (!fusion->reply_frames_desc_pool ||
825	!fusion->reply_frames_desc_pool_align) {
826	dev_err(&instance->pdev->dev,
827	"Failed from %s %d\n", __func__, __LINE__);
828	return -ENOMEM;
829	}
830
831	/*
832	* For INVADER_SERIES each set of 8 reply queues(0-7, 8-15, ..) and
833	* VENTURA_SERIES each set of 16 reply queues(0-15, 16-31, ..) should be
834	* within 4GB boundary and also reply queues in a set must have same
835	* upper 32-bits in their memory address. so here driver is allocating the
836	* DMA'able memory for reply queues according. Driver uses limitation of
837	* VENTURA_SERIES to manage INVADER_SERIES as well.
838	*/
839	dma_alloc_count = DIV_ROUND_UP(msix_count, RDPQ_MAX_INDEX_IN_ONE_CHUNK);
840
841	for (i = 0; i < dma_alloc_count; i++) {
842	rdpq_chunk_virt[i] =
843	dma_pool_alloc(pool: fusion->reply_frames_desc_pool,
844	GFP_KERNEL, handle: &rdpq_chunk_phys[i]);
845	if (!rdpq_chunk_virt[i]) {
846	dev_err(&instance->pdev->dev,
847	"Failed from %s %d\n", __func__, __LINE__);
848	return -ENOMEM;
849	}
850	/* reply desc pool requires to be in same 4 gb region.
851	* Below function will check this.
852	* In case of failure, new pci pool will be created with updated
853	* alignment.
854	* For RDPQ buffers, driver always allocate two separate pci pool.
855	* Alignment will be used such a way that next allocation if
856	* success, will always meet same 4gb region requirement.
857	* rdpq_tracker keep track of each buffer's physical,
858	* virtual address and pci pool descriptor. It will help driver
859	* while freeing the resources.
860	*
861	*/
862	if (!megasas_check_same_4gb_region(instance, start_addr: rdpq_chunk_phys[i],
863	size: chunk_size)) {
864	dma_pool_free(pool: fusion->reply_frames_desc_pool,
865	vaddr: rdpq_chunk_virt[i],
866	addr: rdpq_chunk_phys[i]);
867
868	rdpq_chunk_virt[i] =
869	dma_pool_alloc(pool: fusion->reply_frames_desc_pool_align,
870	GFP_KERNEL, handle: &rdpq_chunk_phys[i]);
871	if (!rdpq_chunk_virt[i]) {
872	dev_err(&instance->pdev->dev,
873	"Failed from %s %d\n",
874	__func__, __LINE__);
875	return -ENOMEM;
876	}
877	fusion->rdpq_tracker[i].dma_pool_ptr =
878	fusion->reply_frames_desc_pool_align;
879	} else {
880	fusion->rdpq_tracker[i].dma_pool_ptr =
881	fusion->reply_frames_desc_pool;
882	}
883
884	fusion->rdpq_tracker[i].pool_entry_phys = rdpq_chunk_phys[i];
885	fusion->rdpq_tracker[i].pool_entry_virt = rdpq_chunk_virt[i];
886	}
887
888	for (k = 0; k < dma_alloc_count; k++) {
889	for (i = 0; i < RDPQ_MAX_INDEX_IN_ONE_CHUNK; i++) {
890	abs_index = (k * RDPQ_MAX_INDEX_IN_ONE_CHUNK) + i;
891
892	if (abs_index == msix_count)
893	break;
894	offset = fusion->reply_alloc_sz * i;
895	fusion->rdpq_virt[abs_index].RDPQBaseAddress =
896	cpu_to_le64(rdpq_chunk_phys[k] + offset);
897	fusion->reply_frames_desc_phys[abs_index] =
898	rdpq_chunk_phys[k] + offset;
899	fusion->reply_frames_desc[abs_index] =
900	(union MPI2_REPLY_DESCRIPTORS_UNION *)((u8 *)rdpq_chunk_virt[k] + offset);
901
902	reply_desc = fusion->reply_frames_desc[abs_index];
903	for (j = 0; j < fusion->reply_q_depth; j++, reply_desc++)
904	reply_desc->Words = ULLONG_MAX;
905	}
906	}
907
908	return 0;
909	}
910
911	static void
912	megasas_free_rdpq_fusion(struct megasas_instance *instance) {
913
914	int i;
915	struct fusion_context *fusion;
916
917	fusion = instance->ctrl_context;
918
919	for (i = 0; i < RDPQ_MAX_CHUNK_COUNT; i++) {
920	if (fusion->rdpq_tracker[i].pool_entry_virt)
921	dma_pool_free(pool: fusion->rdpq_tracker[i].dma_pool_ptr,
922	vaddr: fusion->rdpq_tracker[i].pool_entry_virt,
923	addr: fusion->rdpq_tracker[i].pool_entry_phys);
924
925	}
926
927	dma_pool_destroy(pool: fusion->reply_frames_desc_pool);
928	dma_pool_destroy(pool: fusion->reply_frames_desc_pool_align);
929
930	if (fusion->rdpq_virt)
931	dma_free_coherent(dev: &instance->pdev->dev,
932	size: sizeof(struct MPI2_IOC_INIT_RDPQ_ARRAY_ENTRY) * MAX_MSIX_QUEUES_FUSION,
933	cpu_addr: fusion->rdpq_virt, dma_handle: fusion->rdpq_phys);
934	}
935
936	static void
937	megasas_free_reply_fusion(struct megasas_instance *instance) {
938
939	struct fusion_context *fusion;
940
941	fusion = instance->ctrl_context;
942
943	if (fusion->reply_frames_desc[0])
944	dma_pool_free(pool: fusion->reply_frames_desc_pool,
945	vaddr: fusion->reply_frames_desc[0],
946	addr: fusion->reply_frames_desc_phys[0]);
947
948	dma_pool_destroy(pool: fusion->reply_frames_desc_pool);
949
950	}
951
952
953	/**
954	* megasas_alloc_cmds_fusion - Allocates the command packets
955	* @instance: Adapter soft state
956	*
957	*
958	* Each frame has a 32-bit field called context. This context is used to get
959	* back the megasas_cmd_fusion from the frame when a frame gets completed
960	* In this driver, the 32 bit values are the indices into an array cmd_list.
961	* This array is used only to look up the megasas_cmd_fusion given the context.
962	* The free commands themselves are maintained in a linked list called cmd_pool.
963	*
964	* cmds are formed in the io_request and sg_frame members of the
965	* megasas_cmd_fusion. The context field is used to get a request descriptor
966	* and is used as SMID of the cmd.
967	* SMID value range is from 1 to max_fw_cmds.
968	*/
969	static int
970	megasas_alloc_cmds_fusion(struct megasas_instance *instance)
971	{
972	int i;
973	struct fusion_context *fusion;
974	struct megasas_cmd_fusion *cmd;
975	u32 offset;
976	dma_addr_t io_req_base_phys;
977	u8 *io_req_base;
978
979
980	fusion = instance->ctrl_context;
981
982	if (megasas_alloc_request_fusion(instance))
983	goto fail_exit;
984
985	if (instance->is_rdpq) {
986	if (megasas_alloc_rdpq_fusion(instance))
987	goto fail_exit;
988	} else
989	if (megasas_alloc_reply_fusion(instance))
990	goto fail_exit;
991
992	if (megasas_alloc_cmdlist_fusion(instance))
993	goto fail_exit;
994
995	/* The first 256 bytes (SMID 0) is not used. Don't add to the cmd list */
996	io_req_base = fusion->io_request_frames + MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE;
997	io_req_base_phys = fusion->io_request_frames_phys + MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE;
998
999	/*
1000	* Add all the commands to command pool (fusion->cmd_pool)
1001	*/
1002
1003	/* SMID 0 is reserved. Set SMID/index from 1 */
1004	for (i = 0; i < instance->max_mpt_cmds; i++) {
1005	cmd = fusion->cmd_list[i];
1006	offset = MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE * i;
1007	memset(cmd, 0, sizeof(struct megasas_cmd_fusion));
1008	cmd->index = i + 1;
1009	cmd->scmd = NULL;
1010	cmd->sync_cmd_idx =
1011	(i >= instance->max_scsi_cmds && i < instance->max_fw_cmds) ?
1012	(i - instance->max_scsi_cmds) :
1013	(u32)ULONG_MAX; /* Set to Invalid */
1014	cmd->instance = instance;
1015	cmd->io_request =
1016	(struct MPI2_RAID_SCSI_IO_REQUEST *)
1017	(io_req_base + offset);
1018	memset(cmd->io_request, 0,
1019	sizeof(struct MPI2_RAID_SCSI_IO_REQUEST));
1020	cmd->io_request_phys_addr = io_req_base_phys + offset;
1021	cmd->r1_alt_dev_handle = MR_DEVHANDLE_INVALID;
1022	}
1023
1024	if (megasas_create_sg_sense_fusion(instance))
1025	goto fail_exit;
1026
1027	return 0;
1028
1029	fail_exit:
1030	megasas_free_cmds_fusion(instance);
1031	return -ENOMEM;
1032	}
1033
1034	/**
1035	* wait_and_poll - Issues a polling command
1036	* @instance: Adapter soft state
1037	* @cmd: Command packet to be issued
1038	* @seconds: Maximum poll time
1039	*
1040	* For polling, MFI requires the cmd_status to be set to 0xFF before posting.
1041	*/
1042	int
1043	wait_and_poll(struct megasas_instance *instance, struct megasas_cmd *cmd,
1044	int seconds)
1045	{
1046	int i;
1047	struct megasas_header *frame_hdr = &cmd->frame->hdr;
1048	u32 status_reg;
1049
1050	u32 msecs = seconds * 1000;
1051
1052	/*
1053	* Wait for cmd_status to change
1054	*/
1055	for (i = 0; (i < msecs) && (frame_hdr->cmd_status == 0xff); i += 20) {
1056	rmb();
1057	msleep(msecs: 20);
1058	if (!(i % 5000)) {
1059	status_reg = instance->instancet->read_fw_status_reg(instance)
1060	& MFI_STATE_MASK;
1061	if (status_reg == MFI_STATE_FAULT)
1062	break;
1063	}
1064	}
1065
1066	if (frame_hdr->cmd_status == MFI_STAT_INVALID_STATUS)
1067	return DCMD_TIMEOUT;
1068	else if (frame_hdr->cmd_status == MFI_STAT_OK)
1069	return DCMD_SUCCESS;
1070	else
1071	return DCMD_FAILED;
1072	}
1073
1074	/**
1075	* megasas_ioc_init_fusion - Initializes the FW
1076	* @instance: Adapter soft state
1077	*
1078	* Issues the IOC Init cmd
1079	*/
1080	int
1081	megasas_ioc_init_fusion(struct megasas_instance *instance)
1082	{
1083	struct megasas_init_frame *init_frame;
1084	struct MPI2_IOC_INIT_REQUEST *IOCInitMessage = NULL;
1085	dma_addr_t ioc_init_handle;
1086	struct megasas_cmd *cmd;
1087	u8 ret, cur_rdpq_mode;
1088	struct fusion_context *fusion;
1089	union MEGASAS_REQUEST_DESCRIPTOR_UNION req_desc;
1090	int i;
1091	struct megasas_header *frame_hdr;
1092	const char *sys_info;
1093	MFI_CAPABILITIES *drv_ops;
1094	u32 scratch_pad_1;
1095	ktime_t time;
1096	bool cur_fw_64bit_dma_capable;
1097	bool cur_intr_coalescing;
1098
1099	fusion = instance->ctrl_context;
1100
1101	ioc_init_handle = fusion->ioc_init_request_phys;
1102	IOCInitMessage = fusion->ioc_init_request;
1103
1104	cmd = fusion->ioc_init_cmd;
1105
1106	scratch_pad_1 = megasas_readl
1107	(instance, addr: &instance->reg_set->outbound_scratch_pad_1);
1108
1109	cur_rdpq_mode = (scratch_pad_1 & MR_RDPQ_MODE_OFFSET) ? 1 : 0;
1110
1111	if (instance->adapter_type == INVADER_SERIES) {
1112	cur_fw_64bit_dma_capable =
1113	(scratch_pad_1 & MR_CAN_HANDLE_64_BIT_DMA_OFFSET) ? true : false;
1114
1115	if (instance->consistent_mask_64bit && !cur_fw_64bit_dma_capable) {
1116	dev_err(&instance->pdev->dev, "Driver was operating on 64bit "
1117	"DMA mask, but upcoming FW does not support 64bit DMA mask\n");
1118	megaraid_sas_kill_hba(instance);
1119	ret = 1;
1120	goto fail_fw_init;
1121	}
1122	}
1123
1124	if (instance->is_rdpq && !cur_rdpq_mode) {
1125	dev_err(&instance->pdev->dev, "Firmware downgrade *NOT SUPPORTED*"
1126	" from RDPQ mode to non RDPQ mode\n");
1127	ret = 1;
1128	goto fail_fw_init;
1129	}
1130
1131	cur_intr_coalescing = (scratch_pad_1 & MR_INTR_COALESCING_SUPPORT_OFFSET) ?
1132	true : false;
1133
1134	if ((instance->low_latency_index_start ==
1135	MR_HIGH_IOPS_QUEUE_COUNT) && cur_intr_coalescing)
1136	instance->perf_mode = MR_BALANCED_PERF_MODE;
1137
1138	dev_info(&instance->pdev->dev, "Performance mode :%s (latency index = %d)\n",
1139	MEGASAS_PERF_MODE_2STR(instance->perf_mode),
1140	instance->low_latency_index_start);
1141
1142	instance->fw_sync_cache_support = (scratch_pad_1 &
1143	MR_CAN_HANDLE_SYNC_CACHE_OFFSET) ? 1 : 0;
1144	dev_info(&instance->pdev->dev, "FW supports sync cache\t: %s\n",
1145	instance->fw_sync_cache_support ? "Yes" : "No");
1146
1147	memset(IOCInitMessage, 0, sizeof(struct MPI2_IOC_INIT_REQUEST));
1148
1149	IOCInitMessage->Function = MPI2_FUNCTION_IOC_INIT;
1150	IOCInitMessage->WhoInit = MPI2_WHOINIT_HOST_DRIVER;
1151	IOCInitMessage->MsgVersion = cpu_to_le16(MPI2_VERSION);
1152	IOCInitMessage->HeaderVersion = cpu_to_le16(MPI2_HEADER_VERSION);
1153	IOCInitMessage->SystemRequestFrameSize = cpu_to_le16(MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE / 4);
1154
1155	IOCInitMessage->ReplyDescriptorPostQueueDepth = cpu_to_le16(fusion->reply_q_depth);
1156	IOCInitMessage->ReplyDescriptorPostQueueAddress = instance->is_rdpq ?
1157	cpu_to_le64(fusion->rdpq_phys) :
1158	cpu_to_le64(fusion->reply_frames_desc_phys[0]);
1159	IOCInitMessage->MsgFlags = instance->is_rdpq ?
1160	MPI2_IOCINIT_MSGFLAG_RDPQ_ARRAY_MODE : 0;
1161	IOCInitMessage->SystemRequestFrameBaseAddress = cpu_to_le64(fusion->io_request_frames_phys);
1162	IOCInitMessage->SenseBufferAddressHigh = cpu_to_le32(upper_32_bits(fusion->sense_phys_addr));
1163	IOCInitMessage->HostMSIxVectors = instance->msix_vectors + instance->iopoll_q_count;
1164	IOCInitMessage->HostPageSize = MR_DEFAULT_NVME_PAGE_SHIFT;
1165
1166	time = ktime_get_real();
1167	/* Convert to milliseconds as per FW requirement */
1168	IOCInitMessage->TimeStamp = cpu_to_le64(ktime_to_ms(time));
1169
1170	init_frame = (struct megasas_init_frame *)cmd->frame;
1171	memset(init_frame, 0, IOC_INIT_FRAME_SIZE);
1172
1173	frame_hdr = &cmd->frame->hdr;
1174	frame_hdr->cmd_status = 0xFF;
1175	frame_hdr->flags |= cpu_to_le16(MFI_FRAME_DONT_POST_IN_REPLY_QUEUE);
1176
1177	init_frame->cmd = MFI_CMD_INIT;
1178	init_frame->cmd_status = 0xFF;
1179
1180	drv_ops = (MFI_CAPABILITIES *) &(init_frame->driver_operations);
1181
1182	/* driver support Extended MSIX */
1183	if (instance->adapter_type >= INVADER_SERIES)
1184	drv_ops->mfi_capabilities.support_additional_msix = 1;
1185	/* driver supports HA / Remote LUN over Fast Path interface */
1186	drv_ops->mfi_capabilities.support_fp_remote_lun = 1;
1187
1188	drv_ops->mfi_capabilities.support_max_255lds = 1;
1189	drv_ops->mfi_capabilities.support_ndrive_r1_lb = 1;
1190	drv_ops->mfi_capabilities.security_protocol_cmds_fw = 1;
1191
1192	if (instance->max_chain_frame_sz > MEGASAS_CHAIN_FRAME_SZ_MIN)
1193	drv_ops->mfi_capabilities.support_ext_io_size = 1;
1194
1195	drv_ops->mfi_capabilities.support_fp_rlbypass = 1;
1196	if (!dual_qdepth_disable)
1197	drv_ops->mfi_capabilities.support_ext_queue_depth = 1;
1198
1199	drv_ops->mfi_capabilities.support_qd_throttling = 1;
1200	drv_ops->mfi_capabilities.support_pd_map_target_id = 1;
1201	drv_ops->mfi_capabilities.support_nvme_passthru = 1;
1202	drv_ops->mfi_capabilities.support_fw_exposed_dev_list = 1;
1203
1204	if (reset_devices)
1205	drv_ops->mfi_capabilities.support_memdump = 1;
1206
1207	if (instance->consistent_mask_64bit)
1208	drv_ops->mfi_capabilities.support_64bit_mode = 1;
1209
1210	/* Convert capability to LE32 */
1211	cpu_to_le32s((u32 *)&init_frame->driver_operations.mfi_capabilities);
1212
1213	sys_info = dmi_get_system_info(field: DMI_PRODUCT_UUID);
1214	if (instance->system_info_buf && sys_info) {
1215	memcpy(instance->system_info_buf->systemId, sys_info,
1216	strlen(sys_info) > 64 ? 64 : strlen(sys_info));
1217	instance->system_info_buf->systemIdLength =
1218	strlen(sys_info) > 64 ? 64 : strlen(sys_info);
1219	init_frame->system_info_lo = cpu_to_le32(lower_32_bits(instance->system_info_h));
1220	init_frame->system_info_hi = cpu_to_le32(upper_32_bits(instance->system_info_h));
1221	}
1222
1223	init_frame->queue_info_new_phys_addr_hi =
1224	cpu_to_le32(upper_32_bits(ioc_init_handle));
1225	init_frame->queue_info_new_phys_addr_lo =
1226	cpu_to_le32(lower_32_bits(ioc_init_handle));
1227	init_frame->data_xfer_len = cpu_to_le32(sizeof(struct MPI2_IOC_INIT_REQUEST));
1228
1229	/*
1230	* Each bit in replyqueue_mask represents one group of MSI-x vectors
1231	* (each group has 8 vectors)
1232	*/
1233	switch (instance->perf_mode) {
1234	case MR_BALANCED_PERF_MODE:
1235	init_frame->replyqueue_mask =
1236	cpu_to_le16(~(~0 << instance->low_latency_index_start/8));
1237	break;
1238	case MR_IOPS_PERF_MODE:
1239	init_frame->replyqueue_mask =
1240	cpu_to_le16(~(~0 << instance->msix_vectors/8));
1241	break;
1242	}
1243
1244
1245	req_desc.u.low = cpu_to_le32(lower_32_bits(cmd->frame_phys_addr));
1246	req_desc.u.high = cpu_to_le32(upper_32_bits(cmd->frame_phys_addr));
1247	req_desc.MFAIo.RequestFlags =
1248	(MEGASAS_REQ_DESCRIPT_FLAGS_MFA <<
1249	MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
1250
1251	/*
1252	* disable the intr before firing the init frame
1253	*/
1254	instance->instancet->disable_intr(instance);
1255
1256	for (i = 0; i < (10 * 1000); i += 20) {
1257	if (megasas_readl(instance, addr: &instance->reg_set->doorbell) & 1)
1258	msleep(msecs: 20);
1259	else
1260	break;
1261	}
1262
1263	/* For AERO also, IOC_INIT requires 64 bit descriptor write */
1264	megasas_write_64bit_req_desc(instance, req_desc: &req_desc);
1265
1266	wait_and_poll(instance, cmd, MFI_IO_TIMEOUT_SECS);
1267
1268	frame_hdr = &cmd->frame->hdr;
1269	if (frame_hdr->cmd_status != 0) {
1270	ret = 1;
1271	goto fail_fw_init;
1272	}
1273
1274	if (instance->adapter_type >= AERO_SERIES) {
1275	scratch_pad_1 = megasas_readl
1276	(instance, addr: &instance->reg_set->outbound_scratch_pad_1);
1277
1278	instance->atomic_desc_support =
1279	(scratch_pad_1 & MR_ATOMIC_DESCRIPTOR_SUPPORT_OFFSET) ? 1 : 0;
1280
1281	dev_info(&instance->pdev->dev, "FW supports atomic descriptor\t: %s\n",
1282	instance->atomic_desc_support ? "Yes" : "No");
1283	}
1284
1285	return 0;
1286
1287	fail_fw_init:
1288	dev_err(&instance->pdev->dev,
1289	"Init cmd return status FAILED for SCSI host %d\n",
1290	instance->host->host_no);
1291
1292	return ret;
1293	}
1294
1295	/**
1296	* megasas_sync_pd_seq_num - JBOD SEQ MAP
1297	* @instance: Adapter soft state
1298	* @pend: set to 1, if it is pended jbod map.
1299	*
1300	* Issue Jbod map to the firmware. If it is pended command,
1301	* issue command and return. If it is first instance of jbod map
1302	* issue and receive command.
1303	*/
1304	int
1305	megasas_sync_pd_seq_num(struct megasas_instance *instance, bool pend) {
1306	int ret = 0;
1307	size_t pd_seq_map_sz;
1308	struct megasas_cmd *cmd;
1309	struct megasas_dcmd_frame *dcmd;
1310	struct fusion_context *fusion = instance->ctrl_context;
1311	struct MR_PD_CFG_SEQ_NUM_SYNC *pd_sync;
1312	dma_addr_t pd_seq_h;
1313
1314	pd_sync = (void *)fusion->pd_seq_sync[(instance->pd_seq_map_id & 1)];
1315	pd_seq_h = fusion->pd_seq_phys[(instance->pd_seq_map_id & 1)];
1316	pd_seq_map_sz = struct_size(pd_sync, seq, MAX_PHYSICAL_DEVICES);
1317
1318	cmd = megasas_get_cmd(instance);
1319	if (!cmd) {
1320	dev_err(&instance->pdev->dev,
1321	"Could not get mfi cmd. Fail from %s %d\n",
1322	__func__, __LINE__);
1323	return -ENOMEM;
1324	}
1325
1326	dcmd = &cmd->frame->dcmd;
1327
1328	memset(pd_sync, 0, pd_seq_map_sz);
1329	memset(dcmd->mbox.b, 0, MFI_MBOX_SIZE);
1330
1331	if (pend) {
1332	dcmd->mbox.b[0] = MEGASAS_DCMD_MBOX_PEND_FLAG;
1333	dcmd->flags = MFI_FRAME_DIR_WRITE;
1334	instance->jbod_seq_cmd = cmd;
1335	} else {
1336	dcmd->flags = MFI_FRAME_DIR_READ;
1337	}
1338
1339	dcmd->cmd = MFI_CMD_DCMD;
1340	dcmd->cmd_status = 0xFF;
1341	dcmd->sge_count = 1;
1342	dcmd->timeout = 0;
1343	dcmd->pad_0 = 0;
1344	dcmd->data_xfer_len = cpu_to_le32(pd_seq_map_sz);
1345	dcmd->opcode = cpu_to_le32(MR_DCMD_SYSTEM_PD_MAP_GET_INFO);
1346
1347	megasas_set_dma_settings(instance, dcmd, dma_addr: pd_seq_h, dma_len: pd_seq_map_sz);
1348
1349	if (pend) {
1350	instance->instancet->issue_dcmd(instance, cmd);
1351	return 0;
1352	}
1353
1354	/* Below code is only for non pended DCMD */
1355	if (!instance->mask_interrupts)
1356	ret = megasas_issue_blocked_cmd(instance, cmd,
1357	MFI_IO_TIMEOUT_SECS);
1358	else
1359	ret = megasas_issue_polled(instance, cmd);
1360
1361	if (le32_to_cpu(pd_sync->count) > MAX_PHYSICAL_DEVICES) {
1362	dev_warn(&instance->pdev->dev,
1363	"driver supports max %d JBOD, but FW reports %d\n",
1364	MAX_PHYSICAL_DEVICES, le32_to_cpu(pd_sync->count));
1365	ret = -EINVAL;
1366	}
1367
1368	if (ret == DCMD_TIMEOUT)
1369	dev_warn(&instance->pdev->dev,
1370	"%s DCMD timed out, continue without JBOD sequence map\n",
1371	__func__);
1372
1373	if (ret == DCMD_SUCCESS)
1374	instance->pd_seq_map_id++;
1375
1376	megasas_return_cmd(instance, cmd);
1377	return ret;
1378	}
1379
1380	/*
1381	* megasas_get_ld_map_info - Returns FW's ld_map structure
1382	* @instance: Adapter soft state
1383	* @pend: Pend the command or not
1384	* Issues an internal command (DCMD) to get the FW's controller PD
1385	* list structure. This information is mainly used to find out SYSTEM
1386	* supported by the FW.
1387	* dcmd.mbox value setting for MR_DCMD_LD_MAP_GET_INFO
1388	* dcmd.mbox.b[0] - number of LDs being sync'd
1389	* dcmd.mbox.b[1] - 0 - complete command immediately.
1390	* - 1 - pend till config change
1391	* dcmd.mbox.b[2] - 0 - supports max 64 lds and uses legacy MR_FW_RAID_MAP
1392	* - 1 - supports max MAX_LOGICAL_DRIVES_EXT lds and
1393	* uses extended struct MR_FW_RAID_MAP_EXT
1394	*/
1395	static int
1396	megasas_get_ld_map_info(struct megasas_instance *instance)
1397	{
1398	int ret = 0;
1399	struct megasas_cmd *cmd;
1400	struct megasas_dcmd_frame *dcmd;
1401	void *ci;
1402	dma_addr_t ci_h = 0;
1403	u32 size_map_info;
1404	struct fusion_context *fusion;
1405
1406	cmd = megasas_get_cmd(instance);
1407
1408	if (!cmd) {
1409	dev_printk(KERN_DEBUG, &instance->pdev->dev, "Failed to get cmd for map info\n");
1410	return -ENOMEM;
1411	}
1412
1413	fusion = instance->ctrl_context;
1414
1415	if (!fusion) {
1416	megasas_return_cmd(instance, cmd);
1417	return -ENXIO;
1418	}
1419
1420	dcmd = &cmd->frame->dcmd;
1421
1422	size_map_info = fusion->current_map_sz;
1423
1424	ci = (void *) fusion->ld_map[(instance->map_id & 1)];
1425	ci_h = fusion->ld_map_phys[(instance->map_id & 1)];
1426
1427	if (!ci) {
1428	dev_printk(KERN_DEBUG, &instance->pdev->dev, "Failed to alloc mem for ld_map_info\n");
1429	megasas_return_cmd(instance, cmd);
1430	return -ENOMEM;
1431	}
1432
1433	memset(ci, 0, fusion->max_map_sz);
1434	memset(dcmd->mbox.b, 0, MFI_MBOX_SIZE);
1435	dcmd->cmd = MFI_CMD_DCMD;
1436	dcmd->cmd_status = 0xFF;
1437	dcmd->sge_count = 1;
1438	dcmd->flags = MFI_FRAME_DIR_READ;
1439	dcmd->timeout = 0;
1440	dcmd->pad_0 = 0;
1441	dcmd->data_xfer_len = cpu_to_le32(size_map_info);
1442	dcmd->opcode = cpu_to_le32(MR_DCMD_LD_MAP_GET_INFO);
1443
1444	megasas_set_dma_settings(instance, dcmd, dma_addr: ci_h, dma_len: size_map_info);
1445
1446	if (!instance->mask_interrupts)
1447	ret = megasas_issue_blocked_cmd(instance, cmd,
1448	MFI_IO_TIMEOUT_SECS);
1449	else
1450	ret = megasas_issue_polled(instance, cmd);
1451
1452	if (ret == DCMD_TIMEOUT)
1453	dev_warn(&instance->pdev->dev,
1454	"%s DCMD timed out, RAID map is disabled\n",
1455	__func__);
1456
1457	megasas_return_cmd(instance, cmd);
1458
1459	return ret;
1460	}
1461
1462	u8
1463	megasas_get_map_info(struct megasas_instance *instance)
1464	{
1465	struct fusion_context *fusion = instance->ctrl_context;
1466
1467	fusion->fast_path_io = 0;
1468	if (!megasas_get_ld_map_info(instance)) {
1469	if (MR_ValidateMapInfo(instance, map_id: instance->map_id)) {
1470	fusion->fast_path_io = 1;
1471	return 0;
1472	}
1473	}
1474	return 1;
1475	}
1476
1477	/*
1478	* megasas_sync_map_info - Returns FW's ld_map structure
1479	* @instance: Adapter soft state
1480	*
1481	* Issues an internal command (DCMD) to get the FW's controller PD
1482	* list structure. This information is mainly used to find out SYSTEM
1483	* supported by the FW.
1484	*/
1485	int
1486	megasas_sync_map_info(struct megasas_instance *instance)
1487	{
1488	int i;
1489	struct megasas_cmd *cmd;
1490	struct megasas_dcmd_frame *dcmd;
1491	u16 num_lds;
1492	struct fusion_context *fusion;
1493	struct MR_LD_TARGET_SYNC *ci = NULL;
1494	struct MR_DRV_RAID_MAP_ALL *map;
1495	struct MR_LD_RAID *raid;
1496	struct MR_LD_TARGET_SYNC *ld_sync;
1497	dma_addr_t ci_h = 0;
1498	u32 size_map_info;
1499
1500	cmd = megasas_get_cmd(instance);
1501
1502	if (!cmd) {
1503	dev_printk(KERN_DEBUG, &instance->pdev->dev, "Failed to get cmd for sync info\n");
1504	return -ENOMEM;
1505	}
1506
1507	fusion = instance->ctrl_context;
1508
1509	if (!fusion) {
1510	megasas_return_cmd(instance, cmd);
1511	return 1;
1512	}
1513
1514	map = fusion->ld_drv_map[instance->map_id & 1];
1515
1516	num_lds = le16_to_cpu(map->raidMap.ldCount);
1517
1518	dcmd = &cmd->frame->dcmd;
1519
1520	memset(dcmd->mbox.b, 0, MFI_MBOX_SIZE);
1521
1522	ci = (struct MR_LD_TARGET_SYNC *)
1523	fusion->ld_map[(instance->map_id - 1) & 1];
1524	memset(ci, 0, fusion->max_map_sz);
1525
1526	ci_h = fusion->ld_map_phys[(instance->map_id - 1) & 1];
1527
1528	ld_sync = (struct MR_LD_TARGET_SYNC *)ci;
1529
1530	for (i = 0; i < num_lds; i++, ld_sync++) {
1531	raid = MR_LdRaidGet(ld: i, map);
1532	ld_sync->targetId = MR_GetLDTgtId(ld: i, map);
1533	ld_sync->seqNum = raid->seqNum;
1534	}
1535
1536	size_map_info = fusion->current_map_sz;
1537
1538	dcmd->cmd = MFI_CMD_DCMD;
1539	dcmd->cmd_status = 0xFF;
1540	dcmd->sge_count = 1;
1541	dcmd->flags = MFI_FRAME_DIR_WRITE;
1542	dcmd->timeout = 0;
1543	dcmd->pad_0 = 0;
1544	dcmd->data_xfer_len = cpu_to_le32(size_map_info);
1545	dcmd->mbox.b[0] = num_lds;
1546	dcmd->mbox.b[1] = MEGASAS_DCMD_MBOX_PEND_FLAG;
1547	dcmd->opcode = cpu_to_le32(MR_DCMD_LD_MAP_GET_INFO);
1548
1549	megasas_set_dma_settings(instance, dcmd, dma_addr: ci_h, dma_len: size_map_info);
1550
1551	instance->map_update_cmd = cmd;
1552
1553	instance->instancet->issue_dcmd(instance, cmd);
1554
1555	return 0;
1556	}
1557
1558	/*
1559	* meagasas_display_intel_branding - Display branding string
1560	* @instance: per adapter object
1561	*
1562	* Return nothing.
1563	*/
1564	static void
1565	megasas_display_intel_branding(struct megasas_instance *instance)
1566	{
1567	if (instance->pdev->subsystem_vendor != PCI_VENDOR_ID_INTEL)
1568	return;
1569
1570	switch (instance->pdev->device) {
1571	case PCI_DEVICE_ID_LSI_INVADER:
1572	switch (instance->pdev->subsystem_device) {
1573	case MEGARAID_INTEL_RS3DC080_SSDID:
1574	dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1575	instance->host->host_no,
1576	MEGARAID_INTEL_RS3DC080_BRANDING);
1577	break;
1578	case MEGARAID_INTEL_RS3DC040_SSDID:
1579	dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1580	instance->host->host_no,
1581	MEGARAID_INTEL_RS3DC040_BRANDING);
1582	break;
1583	case MEGARAID_INTEL_RS3SC008_SSDID:
1584	dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1585	instance->host->host_no,
1586	MEGARAID_INTEL_RS3SC008_BRANDING);
1587	break;
1588	case MEGARAID_INTEL_RS3MC044_SSDID:
1589	dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1590	instance->host->host_no,
1591	MEGARAID_INTEL_RS3MC044_BRANDING);
1592	break;
1593	default:
1594	break;
1595	}
1596	break;
1597	case PCI_DEVICE_ID_LSI_FURY:
1598	switch (instance->pdev->subsystem_device) {
1599	case MEGARAID_INTEL_RS3WC080_SSDID:
1600	dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1601	instance->host->host_no,
1602	MEGARAID_INTEL_RS3WC080_BRANDING);
1603	break;
1604	case MEGARAID_INTEL_RS3WC040_SSDID:
1605	dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1606	instance->host->host_no,
1607	MEGARAID_INTEL_RS3WC040_BRANDING);
1608	break;
1609	default:
1610	break;
1611	}
1612	break;
1613	case PCI_DEVICE_ID_LSI_CUTLASS_52:
1614	case PCI_DEVICE_ID_LSI_CUTLASS_53:
1615	switch (instance->pdev->subsystem_device) {
1616	case MEGARAID_INTEL_RMS3BC160_SSDID:
1617	dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1618	instance->host->host_no,
1619	MEGARAID_INTEL_RMS3BC160_BRANDING);
1620	break;
1621	default:
1622	break;
1623	}
1624	break;
1625	default:
1626	break;
1627	}
1628	}
1629
1630	/**
1631	* megasas_allocate_raid_maps - Allocate memory for RAID maps
1632	* @instance: Adapter soft state
1633	*
1634	* return: if success: return 0
1635	* failed: return -ENOMEM
1636	*/
1637	static inline int megasas_allocate_raid_maps(struct megasas_instance *instance)
1638	{
1639	struct fusion_context *fusion;
1640	int i = 0;
1641
1642	fusion = instance->ctrl_context;
1643
1644	fusion->drv_map_pages = get_order(size: fusion->drv_map_sz);
1645
1646	for (i = 0; i < 2; i++) {
1647	fusion->ld_map[i] = NULL;
1648
1649	fusion->ld_drv_map[i] = (void *)
1650	__get_free_pages(__GFP_ZERO | GFP_KERNEL,
1651	order: fusion->drv_map_pages);
1652
1653	if (!fusion->ld_drv_map[i]) {
1654	fusion->ld_drv_map[i] = vzalloc(size: fusion->drv_map_sz);
1655
1656	if (!fusion->ld_drv_map[i]) {
1657	dev_err(&instance->pdev->dev,
1658	"Could not allocate memory for local map"
1659	" size requested: %d\n",
1660	fusion->drv_map_sz);
1661	goto ld_drv_map_alloc_fail;
1662	}
1663	}
1664	}
1665
1666	for (i = 0; i < 2; i++) {
1667	fusion->ld_map[i] = dma_alloc_coherent(dev: &instance->pdev->dev,
1668	size: fusion->max_map_sz,
1669	dma_handle: &fusion->ld_map_phys[i],
1670	GFP_KERNEL);
1671	if (!fusion->ld_map[i]) {
1672	dev_err(&instance->pdev->dev,
1673	"Could not allocate memory for map info %s:%d\n",
1674	__func__, __LINE__);
1675	goto ld_map_alloc_fail;
1676	}
1677	}
1678
1679	return 0;
1680
1681	ld_map_alloc_fail:
1682	for (i = 0; i < 2; i++) {
1683	if (fusion->ld_map[i])
1684	dma_free_coherent(dev: &instance->pdev->dev,
1685	size: fusion->max_map_sz,
1686	cpu_addr: fusion->ld_map[i],
1687	dma_handle: fusion->ld_map_phys[i]);
1688	}
1689
1690	ld_drv_map_alloc_fail:
1691	for (i = 0; i < 2; i++) {
1692	if (fusion->ld_drv_map[i]) {
1693	if (is_vmalloc_addr(x: fusion->ld_drv_map[i]))
1694	vfree(addr: fusion->ld_drv_map[i]);
1695	else
1696	free_pages(addr: (ulong)fusion->ld_drv_map[i],
1697	order: fusion->drv_map_pages);
1698	}
1699	}
1700
1701	return -ENOMEM;
1702	}
1703
1704	/**
1705	* megasas_configure_queue_sizes - Calculate size of request desc queue,
1706	* reply desc queue,
1707	* IO request frame queue, set can_queue.
1708	* @instance: Adapter soft state
1709	* @return: void
1710	*/
1711	static inline
1712	void megasas_configure_queue_sizes(struct megasas_instance *instance)
1713	{
1714	struct fusion_context *fusion;
1715	u16 max_cmd;
1716
1717	fusion = instance->ctrl_context;
1718	max_cmd = instance->max_fw_cmds;
1719
1720	if (instance->adapter_type >= VENTURA_SERIES)
1721	instance->max_mpt_cmds = instance->max_fw_cmds * RAID_1_PEER_CMDS;
1722	else
1723	instance->max_mpt_cmds = instance->max_fw_cmds;
1724
1725	instance->max_scsi_cmds = instance->max_fw_cmds - instance->max_mfi_cmds;
1726	instance->cur_can_queue = instance->max_scsi_cmds;
1727	instance->host->can_queue = instance->cur_can_queue;
1728
1729	fusion->reply_q_depth = 2 * ((max_cmd + 1 + 15) / 16) * 16;
1730
1731	fusion->request_alloc_sz = sizeof(union MEGASAS_REQUEST_DESCRIPTOR_UNION) *
1732	instance->max_mpt_cmds;
1733	fusion->reply_alloc_sz = sizeof(union MPI2_REPLY_DESCRIPTORS_UNION) *
1734	(fusion->reply_q_depth);
1735	fusion->io_frames_alloc_sz = MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE +
1736	(MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE
1737	* (instance->max_mpt_cmds + 1)); /* Extra 1 for SMID 0 */
1738	}
1739
1740	static int megasas_alloc_ioc_init_frame(struct megasas_instance *instance)
1741	{
1742	struct fusion_context *fusion;
1743	struct megasas_cmd *cmd;
1744
1745	fusion = instance->ctrl_context;
1746
1747	cmd = kzalloc(size: sizeof(struct megasas_cmd), GFP_KERNEL);
1748
1749	if (!cmd) {
1750	dev_err(&instance->pdev->dev, "Failed from func: %s line: %d\n",
1751	__func__, __LINE__);
1752	return -ENOMEM;
1753	}
1754
1755	cmd->frame = dma_alloc_coherent(dev: &instance->pdev->dev,
1756	IOC_INIT_FRAME_SIZE,
1757	dma_handle: &cmd->frame_phys_addr, GFP_KERNEL);
1758
1759	if (!cmd->frame) {
1760	dev_err(&instance->pdev->dev, "Failed from func: %s line: %d\n",
1761	__func__, __LINE__);
1762	kfree(objp: cmd);
1763	return -ENOMEM;
1764	}
1765
1766	fusion->ioc_init_cmd = cmd;
1767	return 0;
1768	}
1769
1770	/**
1771	* megasas_free_ioc_init_cmd - Free IOC INIT command frame
1772	* @instance: Adapter soft state
1773	*/
1774	static inline void megasas_free_ioc_init_cmd(struct megasas_instance *instance)
1775	{
1776	struct fusion_context *fusion;
1777
1778	fusion = instance->ctrl_context;
1779
1780	if (fusion->ioc_init_cmd && fusion->ioc_init_cmd->frame)
1781	dma_free_coherent(dev: &instance->pdev->dev,
1782	IOC_INIT_FRAME_SIZE,
1783	cpu_addr: fusion->ioc_init_cmd->frame,
1784	dma_handle: fusion->ioc_init_cmd->frame_phys_addr);
1785
1786	kfree(objp: fusion->ioc_init_cmd);
1787	}
1788
1789	/**
1790	* megasas_init_adapter_fusion - Initializes the FW
1791	* @instance: Adapter soft state
1792	*
1793	* This is the main function for initializing firmware.
1794	*/
1795	static u32
1796	megasas_init_adapter_fusion(struct megasas_instance *instance)
1797	{
1798	struct fusion_context *fusion;
1799	u32 scratch_pad_1;
1800	int i = 0, count;
1801	u32 status_reg;
1802
1803	fusion = instance->ctrl_context;
1804
1805	megasas_fusion_update_can_queue(instance, fw_boot_context: PROBE_CONTEXT);
1806
1807	/*
1808	* Only Driver's internal DCMDs and IOCTL DCMDs needs to have MFI frames
1809	*/
1810	instance->max_mfi_cmds =
1811	MEGASAS_FUSION_INTERNAL_CMDS + MEGASAS_FUSION_IOCTL_CMDS;
1812
1813	megasas_configure_queue_sizes(instance);
1814
1815	scratch_pad_1 = megasas_readl(instance,
1816	addr: &instance->reg_set->outbound_scratch_pad_1);
1817	/* If scratch_pad_1 & MEGASAS_MAX_CHAIN_SIZE_UNITS_MASK is set,
1818	* Firmware support extended IO chain frame which is 4 times more than
1819	* legacy Firmware.
1820	* Legacy Firmware - Frame size is (8 * 128) = 1K
1821	* 1M IO Firmware - Frame size is (8 * 128 * 4) = 4K
1822	*/
1823	if (scratch_pad_1 & MEGASAS_MAX_CHAIN_SIZE_UNITS_MASK)
1824	instance->max_chain_frame_sz =
1825	((scratch_pad_1 & MEGASAS_MAX_CHAIN_SIZE_MASK) >>
1826	MEGASAS_MAX_CHAIN_SHIFT) * MEGASAS_1MB_IO;
1827	else
1828	instance->max_chain_frame_sz =
1829	((scratch_pad_1 & MEGASAS_MAX_CHAIN_SIZE_MASK) >>
1830	MEGASAS_MAX_CHAIN_SHIFT) * MEGASAS_256K_IO;
1831
1832	if (instance->max_chain_frame_sz < MEGASAS_CHAIN_FRAME_SZ_MIN) {
1833	dev_warn(&instance->pdev->dev, "frame size %d invalid, fall back to legacy max frame size %d\n",
1834	instance->max_chain_frame_sz,
1835	MEGASAS_CHAIN_FRAME_SZ_MIN);
1836	instance->max_chain_frame_sz = MEGASAS_CHAIN_FRAME_SZ_MIN;
1837	}
1838
1839	fusion->max_sge_in_main_msg =
1840	(MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE
1841	- offsetof(struct MPI2_RAID_SCSI_IO_REQUEST, SGL))/16;
1842
1843	fusion->max_sge_in_chain =
1844	instance->max_chain_frame_sz
1845	/ sizeof(union MPI2_SGE_IO_UNION);
1846
1847	instance->max_num_sge =
1848	rounddown_pow_of_two(fusion->max_sge_in_main_msg
1849	+ fusion->max_sge_in_chain - 2);
1850
1851	/* Used for pass thru MFI frame (DCMD) */
1852	fusion->chain_offset_mfi_pthru =
1853	offsetof(struct MPI2_RAID_SCSI_IO_REQUEST, SGL)/16;
1854
1855	fusion->chain_offset_io_request =
1856	(MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE -
1857	sizeof(union MPI2_SGE_IO_UNION))/16;
1858
1859	count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
1860	count += instance->iopoll_q_count;
1861
1862	for (i = 0 ; i < count; i++)
1863	fusion->last_reply_idx[i] = 0;
1864
1865	/*
1866	* For fusion adapters, 3 commands for IOCTL and 8 commands
1867	* for driver's internal DCMDs.
1868	*/
1869	instance->max_scsi_cmds = instance->max_fw_cmds -
1870	(MEGASAS_FUSION_INTERNAL_CMDS +
1871	MEGASAS_FUSION_IOCTL_CMDS);
1872	sema_init(sem: &instance->ioctl_sem, MEGASAS_FUSION_IOCTL_CMDS);
1873
1874	for (i = 0; i < MAX_MSIX_QUEUES_FUSION; i++)
1875	atomic_set(v: &fusion->busy_mq_poll[i], i: 0);
1876
1877	if (megasas_alloc_ioc_init_frame(instance))
1878	return 1;
1879
1880	/*
1881	* Allocate memory for descriptors
1882	* Create a pool of commands
1883	*/
1884	if (megasas_alloc_cmds(instance))
1885	goto fail_alloc_mfi_cmds;
1886	if (megasas_alloc_cmds_fusion(instance))
1887	goto fail_alloc_cmds;
1888
1889	if (megasas_ioc_init_fusion(instance)) {
1890	status_reg = instance->instancet->read_fw_status_reg(instance);
1891	if (((status_reg & MFI_STATE_MASK) == MFI_STATE_FAULT) &&
1892	(status_reg & MFI_RESET_ADAPTER)) {
1893	/* Do a chip reset and then retry IOC INIT once */
1894	if (megasas_adp_reset_wait_for_ready
1895	(instance, do_adp_reset: true, ocr_context: 0) == FAILED)
1896	goto fail_ioc_init;
1897
1898	if (megasas_ioc_init_fusion(instance))
1899	goto fail_ioc_init;
1900	} else {
1901	goto fail_ioc_init;
1902	}
1903	}
1904
1905	megasas_display_intel_branding(instance);
1906	if (megasas_get_ctrl_info(instance)) {
1907	dev_err(&instance->pdev->dev,
1908	"Could not get controller info. Fail from %s %d\n",
1909	__func__, __LINE__);
1910	goto fail_ioc_init;
1911	}
1912
1913	instance->flag_ieee = 1;
1914	instance->r1_ldio_hint_default = MR_R1_LDIO_PIGGYBACK_DEFAULT;
1915	instance->threshold_reply_count = instance->max_fw_cmds / 4;
1916	fusion->fast_path_io = 0;
1917
1918	if (megasas_allocate_raid_maps(instance))
1919	goto fail_ioc_init;
1920
1921	if (!megasas_get_map_info(instance))
1922	megasas_sync_map_info(instance);
1923
1924	return 0;
1925
1926	fail_ioc_init:
1927	megasas_free_cmds_fusion(instance);
1928	fail_alloc_cmds:
1929	megasas_free_cmds(instance);
1930	fail_alloc_mfi_cmds:
1931	megasas_free_ioc_init_cmd(instance);
1932	return 1;
1933	}
1934
1935	/**
1936	* megasas_fault_detect_work - Worker function of
1937	* FW fault handling workqueue.
1938	* @work: FW fault work struct
1939	*/
1940	static void
1941	megasas_fault_detect_work(struct work_struct *work)
1942	{
1943	struct megasas_instance *instance =
1944	container_of(work, struct megasas_instance,
1945	fw_fault_work.work);
1946	u32 fw_state, dma_state, status;
1947
1948	/* Check the fw state */
1949	fw_state = instance->instancet->read_fw_status_reg(instance) &
1950	MFI_STATE_MASK;
1951
1952	if (fw_state == MFI_STATE_FAULT) {
1953	dma_state = instance->instancet->read_fw_status_reg(instance) &
1954	MFI_STATE_DMADONE;
1955	/* Start collecting crash, if DMA bit is done */
1956	if (instance->crash_dump_drv_support &&
1957	instance->crash_dump_app_support && dma_state) {
1958	megasas_fusion_crash_dump(instance);
1959	} else {
1960	if (instance->unload == 0) {
1961	status = megasas_reset_fusion(shost: instance->host, reason: 0);
1962	if (status != SUCCESS) {
1963	dev_err(&instance->pdev->dev,
1964	"Failed from %s %d, do not re-arm timer\n",
1965	__func__, __LINE__);
1966	return;
1967	}
1968	}
1969	}
1970	}
1971
1972	if (instance->fw_fault_work_q)
1973	queue_delayed_work(wq: instance->fw_fault_work_q,
1974	dwork: &instance->fw_fault_work,
1975	delay: msecs_to_jiffies(MEGASAS_WATCHDOG_THREAD_INTERVAL));
1976	}
1977
1978	int
1979	megasas_fusion_start_watchdog(struct megasas_instance *instance)
1980	{
1981	/* Check if the Fault WQ is already started */
1982	if (instance->fw_fault_work_q)
1983	return SUCCESS;
1984
1985	INIT_DELAYED_WORK(&instance->fw_fault_work, megasas_fault_detect_work);
1986
1987	snprintf(buf: instance->fault_handler_work_q_name,
1988	size: sizeof(instance->fault_handler_work_q_name),
1989	fmt: "poll_megasas%d_status", instance->host->host_no);
1990
1991	instance->fw_fault_work_q =
1992	create_singlethread_workqueue(instance->fault_handler_work_q_name);
1993	if (!instance->fw_fault_work_q) {
1994	dev_err(&instance->pdev->dev, "Failed from %s %d\n",
1995	__func__, __LINE__);
1996	return FAILED;
1997	}
1998
1999	queue_delayed_work(wq: instance->fw_fault_work_q,
2000	dwork: &instance->fw_fault_work,
2001	delay: msecs_to_jiffies(MEGASAS_WATCHDOG_THREAD_INTERVAL));
2002
2003	return SUCCESS;
2004	}
2005
2006	void
2007	megasas_fusion_stop_watchdog(struct megasas_instance *instance)
2008	{
2009	struct workqueue_struct *wq;
2010
2011	if (instance->fw_fault_work_q) {
2012	wq = instance->fw_fault_work_q;
2013	instance->fw_fault_work_q = NULL;
2014	if (!cancel_delayed_work_sync(dwork: &instance->fw_fault_work))
2015	flush_workqueue(wq);
2016	destroy_workqueue(wq);
2017	}
2018	}
2019
2020	/**
2021	* map_cmd_status - Maps FW cmd status to OS cmd status
2022	* @fusion: fusion context
2023	* @scmd: Pointer to cmd
2024	* @status: status of cmd returned by FW
2025	* @ext_status: ext status of cmd returned by FW
2026	* @data_length: command data length
2027	* @sense: command sense data
2028	*/
2029	static void
2030	map_cmd_status(struct fusion_context *fusion,
2031	struct scsi_cmnd *scmd, u8 status, u8 ext_status,
2032	u32 data_length, u8 *sense)
2033	{
2034	u8 cmd_type;
2035	int resid;
2036
2037	cmd_type = megasas_cmd_type(cmd: scmd);
2038	switch (status) {
2039
2040	case MFI_STAT_OK:
2041	scmd->result = DID_OK << 16;
2042	break;
2043
2044	case MFI_STAT_SCSI_IO_FAILED:
2045	case MFI_STAT_LD_INIT_IN_PROGRESS:
2046	scmd->result = (DID_ERROR << 16) | ext_status;
2047	break;
2048
2049	case MFI_STAT_SCSI_DONE_WITH_ERROR:
2050
2051	scmd->result = (DID_OK << 16) | ext_status;
2052	if (ext_status == SAM_STAT_CHECK_CONDITION) {
2053	memcpy(scmd->sense_buffer, sense,
2054	SCSI_SENSE_BUFFERSIZE);
2055	}
2056
2057	/*
2058	* If the IO request is partially completed, then MR FW will
2059	* update "io_request->DataLength" field with actual number of
2060	* bytes transferred.Driver will set residual bytes count in
2061	* SCSI command structure.
2062	*/
2063	resid = (scsi_bufflen(cmd: scmd) - data_length);
2064	scsi_set_resid(cmd: scmd, resid);
2065
2066	if (resid &&
2067	((cmd_type == READ_WRITE_LDIO) ||
2068	(cmd_type == READ_WRITE_SYSPDIO)))
2069	scmd_printk(KERN_INFO, scmd, "BRCM Debug mfi stat 0x%x, data len"
2070	" requested/completed 0x%x/0x%x\n",
2071	status, scsi_bufflen(cmd: scmd), data_length);
2072	break;
2073
2074	case MFI_STAT_LD_OFFLINE:
2075	case MFI_STAT_DEVICE_NOT_FOUND:
2076	scmd->result = DID_BAD_TARGET << 16;
2077	break;
2078	case MFI_STAT_CONFIG_SEQ_MISMATCH:
2079	scmd->result = DID_IMM_RETRY << 16;
2080	break;
2081	default:
2082	scmd->result = DID_ERROR << 16;
2083	break;
2084	}
2085	}
2086
2087	/**
2088	* megasas_is_prp_possible -
2089	* Checks if native NVMe PRPs can be built for the IO
2090	*
2091	* @instance: Adapter soft state
2092	* @scmd: SCSI command from the mid-layer
2093	* @sge_count: scatter gather element count.
2094	*
2095	* Returns: true: PRPs can be built
2096	* false: IEEE SGLs needs to be built
2097	*/
2098	static bool
2099	megasas_is_prp_possible(struct megasas_instance *instance,
2100	struct scsi_cmnd *scmd, int sge_count)
2101	{
2102	u32 data_length = 0;
2103	struct scatterlist *sg_scmd;
2104	bool build_prp = false;
2105	u32 mr_nvme_pg_size;
2106
2107	mr_nvme_pg_size = max_t(u32, instance->nvme_page_size,
2108	MR_DEFAULT_NVME_PAGE_SIZE);
2109	data_length = scsi_bufflen(cmd: scmd);
2110	sg_scmd = scsi_sglist(cmd: scmd);
2111
2112	/*
2113	* NVMe uses one PRP for each page (or part of a page)
2114	* look at the data length - if 4 pages or less then IEEE is OK
2115	* if > 5 pages then we need to build a native SGL
2116	* if > 4 and <= 5 pages, then check physical address of 1st SG entry
2117	* if this first size in the page is >= the residual beyond 4 pages
2118	* then use IEEE, otherwise use native SGL
2119	*/
2120
2121	if (data_length > (mr_nvme_pg_size * 5)) {
2122	build_prp = true;
2123	} else if ((data_length > (mr_nvme_pg_size * 4)) &&
2124	(data_length <= (mr_nvme_pg_size * 5))) {
2125	/* check if 1st SG entry size is < residual beyond 4 pages */
2126	if (sg_dma_len(sg_scmd) < (data_length - (mr_nvme_pg_size * 4)))
2127	build_prp = true;
2128	}
2129
2130	return build_prp;
2131	}
2132
2133	/**
2134	* megasas_make_prp_nvme -
2135	* Prepare PRPs(Physical Region Page)- SGLs specific to NVMe drives only
2136	*
2137	* @instance: Adapter soft state
2138	* @scmd: SCSI command from the mid-layer
2139	* @sgl_ptr: SGL to be filled in
2140	* @cmd: Fusion command frame
2141	* @sge_count: scatter gather element count.
2142	*
2143	* Returns: true: PRPs are built
2144	* false: IEEE SGLs needs to be built
2145	*/
2146	static bool
2147	megasas_make_prp_nvme(struct megasas_instance *instance, struct scsi_cmnd *scmd,
2148	struct MPI25_IEEE_SGE_CHAIN64 *sgl_ptr,
2149	struct megasas_cmd_fusion *cmd, int sge_count)
2150	{
2151	int sge_len, offset, num_prp_in_chain = 0;
2152	struct MPI25_IEEE_SGE_CHAIN64 *main_chain_element, *ptr_first_sgl;
2153	u64 *ptr_sgl;
2154	dma_addr_t ptr_sgl_phys;
2155	u64 sge_addr;
2156	u32 page_mask, page_mask_result;
2157	struct scatterlist *sg_scmd;
2158	u32 first_prp_len;
2159	bool build_prp = false;
2160	int data_len = scsi_bufflen(cmd: scmd);
2161	u32 mr_nvme_pg_size = max_t(u32, instance->nvme_page_size,
2162	MR_DEFAULT_NVME_PAGE_SIZE);
2163
2164	build_prp = megasas_is_prp_possible(instance, scmd, sge_count);
2165
2166	if (!build_prp)
2167	return false;
2168
2169	/*
2170	* Nvme has a very convoluted prp format. One prp is required
2171	* for each page or partial page. Driver need to split up OS sg_list
2172	* entries if it is longer than one page or cross a page
2173	* boundary. Driver also have to insert a PRP list pointer entry as
2174	* the last entry in each physical page of the PRP list.
2175	*
2176	* NOTE: The first PRP "entry" is actually placed in the first
2177	* SGL entry in the main message as IEEE 64 format. The 2nd
2178	* entry in the main message is the chain element, and the rest
2179	* of the PRP entries are built in the contiguous pcie buffer.
2180	*/
2181	page_mask = mr_nvme_pg_size - 1;
2182	ptr_sgl = (u64 *)cmd->sg_frame;
2183	ptr_sgl_phys = cmd->sg_frame_phys_addr;
2184	memset(ptr_sgl, 0, instance->max_chain_frame_sz);
2185
2186	/* Build chain frame element which holds all prps except first*/
2187	main_chain_element = (struct MPI25_IEEE_SGE_CHAIN64 *)
2188	((u8 *)sgl_ptr + sizeof(struct MPI25_IEEE_SGE_CHAIN64));
2189
2190	main_chain_element->Address = cpu_to_le64(ptr_sgl_phys);
2191	main_chain_element->NextChainOffset = 0;
2192	main_chain_element->Flags = IEEE_SGE_FLAGS_CHAIN_ELEMENT |
2193	IEEE_SGE_FLAGS_SYSTEM_ADDR |
2194	MPI26_IEEE_SGE_FLAGS_NSF_NVME_PRP;
2195
2196	/* Build first prp, sge need not to be page aligned*/
2197	ptr_first_sgl = sgl_ptr;
2198	sg_scmd = scsi_sglist(cmd: scmd);
2199	sge_addr = sg_dma_address(sg_scmd);
2200	sge_len = sg_dma_len(sg_scmd);
2201
2202	offset = (u32)(sge_addr & page_mask);
2203	first_prp_len = mr_nvme_pg_size - offset;
2204
2205	ptr_first_sgl->Address = cpu_to_le64(sge_addr);
2206	ptr_first_sgl->Length = cpu_to_le32(first_prp_len);
2207
2208	data_len -= first_prp_len;
2209
2210	if (sge_len > first_prp_len) {
2211	sge_addr += first_prp_len;
2212	sge_len -= first_prp_len;
2213	} else if (sge_len == first_prp_len) {
2214	sg_scmd = sg_next(sg_scmd);
2215	sge_addr = sg_dma_address(sg_scmd);
2216	sge_len = sg_dma_len(sg_scmd);
2217	}
2218
2219	for (;;) {
2220	offset = (u32)(sge_addr & page_mask);
2221
2222	/* Put PRP pointer due to page boundary*/
2223	page_mask_result = (uintptr_t)(ptr_sgl + 1) & page_mask;
2224	if (unlikely(!page_mask_result)) {
2225	scmd_printk(KERN_NOTICE,
2226	scmd, "page boundary ptr_sgl: 0x%p\n",
2227	ptr_sgl);
2228	ptr_sgl_phys += 8;
2229	*ptr_sgl = cpu_to_le64(ptr_sgl_phys);
2230	ptr_sgl++;
2231	num_prp_in_chain++;
2232	}
2233
2234	*ptr_sgl = cpu_to_le64(sge_addr);
2235	ptr_sgl++;
2236	ptr_sgl_phys += 8;
2237	num_prp_in_chain++;
2238
2239	sge_addr += mr_nvme_pg_size;
2240	sge_len -= mr_nvme_pg_size;
2241	data_len -= mr_nvme_pg_size;
2242
2243	if (data_len <= 0)
2244	break;
2245
2246	if (sge_len > 0)
2247	continue;
2248
2249	sg_scmd = sg_next(sg_scmd);
2250	sge_addr = sg_dma_address(sg_scmd);
2251	sge_len = sg_dma_len(sg_scmd);
2252	}
2253
2254	main_chain_element->Length =
2255	cpu_to_le32(num_prp_in_chain * sizeof(u64));
2256
2257	return build_prp;
2258	}
2259
2260	/**
2261	* megasas_make_sgl_fusion - Prepares 32-bit SGL
2262	* @instance: Adapter soft state
2263	* @scp: SCSI command from the mid-layer
2264	* @sgl_ptr: SGL to be filled in
2265	* @cmd: cmd we are working on
2266	* @sge_count: sge count
2267	*
2268	*/
2269	static void
2270	megasas_make_sgl_fusion(struct megasas_instance *instance,
2271	struct scsi_cmnd *scp,
2272	struct MPI25_IEEE_SGE_CHAIN64 *sgl_ptr,
2273	struct megasas_cmd_fusion *cmd, int sge_count)
2274	{
2275	int i, sg_processed;
2276	struct scatterlist *os_sgl;
2277	struct fusion_context *fusion;
2278
2279	fusion = instance->ctrl_context;
2280
2281	if (instance->adapter_type >= INVADER_SERIES) {
2282	struct MPI25_IEEE_SGE_CHAIN64 *sgl_ptr_end = sgl_ptr;
2283	sgl_ptr_end += fusion->max_sge_in_main_msg - 1;
2284	sgl_ptr_end->Flags = 0;
2285	}
2286
2287	scsi_for_each_sg(scp, os_sgl, sge_count, i) {
2288	sgl_ptr->Length = cpu_to_le32(sg_dma_len(os_sgl));
2289	sgl_ptr->Address = cpu_to_le64(sg_dma_address(os_sgl));
2290	sgl_ptr->Flags = 0;
2291	if (instance->adapter_type >= INVADER_SERIES)
2292	if (i == sge_count - 1)
2293	sgl_ptr->Flags = IEEE_SGE_FLAGS_END_OF_LIST;
2294	sgl_ptr++;
2295	sg_processed = i + 1;
2296
2297	if ((sg_processed == (fusion->max_sge_in_main_msg - 1)) &&
2298	(sge_count > fusion->max_sge_in_main_msg)) {
2299
2300	struct MPI25_IEEE_SGE_CHAIN64 *sg_chain;
2301	if (instance->adapter_type >= INVADER_SERIES) {
2302	if ((le16_to_cpu(cmd->io_request->IoFlags) &
2303	MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH) !=
2304	MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH)
2305	cmd->io_request->ChainOffset =
2306	fusion->
2307	chain_offset_io_request;
2308	else
2309	cmd->io_request->ChainOffset = 0;
2310	} else
2311	cmd->io_request->ChainOffset =
2312	fusion->chain_offset_io_request;
2313
2314	sg_chain = sgl_ptr;
2315	/* Prepare chain element */
2316	sg_chain->NextChainOffset = 0;
2317	if (instance->adapter_type >= INVADER_SERIES)
2318	sg_chain->Flags = IEEE_SGE_FLAGS_CHAIN_ELEMENT;
2319	else
2320	sg_chain->Flags =
2321	(IEEE_SGE_FLAGS_CHAIN_ELEMENT |
2322	MPI2_IEEE_SGE_FLAGS_IOCPLBNTA_ADDR);
2323	sg_chain->Length = cpu_to_le32((sizeof(union MPI2_SGE_IO_UNION) * (sge_count - sg_processed)));
2324	sg_chain->Address = cpu_to_le64(cmd->sg_frame_phys_addr);
2325
2326	sgl_ptr =
2327	(struct MPI25_IEEE_SGE_CHAIN64 *)cmd->sg_frame;
2328	memset(sgl_ptr, 0, instance->max_chain_frame_sz);
2329	}
2330	}
2331	}
2332
2333	/**
2334	* megasas_make_sgl - Build Scatter Gather List(SGLs)
2335	* @scp: SCSI command pointer
2336	* @instance: Soft instance of controller
2337	* @cmd: Fusion command pointer
2338	*
2339	* This function will build sgls based on device type.
2340	* For nvme drives, there is different way of building sgls in nvme native
2341	* format- PRPs(Physical Region Page).
2342	*
2343	* Returns the number of sg lists actually used, zero if the sg lists
2344	* is NULL, or -ENOMEM if the mapping failed
2345	*/
2346	static
2347	int megasas_make_sgl(struct megasas_instance *instance, struct scsi_cmnd *scp,
2348	struct megasas_cmd_fusion *cmd)
2349	{
2350	int sge_count;
2351	bool build_prp = false;
2352	struct MPI25_IEEE_SGE_CHAIN64 *sgl_chain64;
2353
2354	sge_count = scsi_dma_map(cmd: scp);
2355
2356	if ((sge_count > instance->max_num_sge) || (sge_count <= 0))
2357	return sge_count;
2358
2359	sgl_chain64 = (struct MPI25_IEEE_SGE_CHAIN64 *)&cmd->io_request->SGL;
2360	if ((le16_to_cpu(cmd->io_request->IoFlags) &
2361	MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH) &&
2362	(cmd->pd_interface == NVME_PD))
2363	build_prp = megasas_make_prp_nvme(instance, scmd: scp, sgl_ptr: sgl_chain64,
2364	cmd, sge_count);
2365
2366	if (!build_prp)
2367	megasas_make_sgl_fusion(instance, scp, sgl_ptr: sgl_chain64,
2368	cmd, sge_count);
2369
2370	return sge_count;
2371	}
2372
2373	/**
2374	* megasas_set_pd_lba - Sets PD LBA
2375	* @io_request: IO request
2376	* @cdb_len: cdb length
2377	* @io_info: IO information
2378	* @scp: SCSI command
2379	* @local_map_ptr: Raid map
2380	* @ref_tag: Primary reference tag
2381	*
2382	* Used to set the PD LBA in CDB for FP IOs
2383	*/
2384	static void
2385	megasas_set_pd_lba(struct MPI2_RAID_SCSI_IO_REQUEST *io_request, u8 cdb_len,
2386	struct IO_REQUEST_INFO *io_info, struct scsi_cmnd *scp,
2387	struct MR_DRV_RAID_MAP_ALL *local_map_ptr, u32 ref_tag)
2388	{
2389	struct MR_LD_RAID *raid;
2390	u16 ld;
2391	u64 start_blk = io_info->pdBlock;
2392	u8 *cdb = io_request->CDB.CDB32;
2393	u32 num_blocks = io_info->numBlocks;
2394	u8 opcode = 0, flagvals = 0, groupnum = 0, control = 0;
2395
2396	/* Check if T10 PI (DIF) is enabled for this LD */
2397	ld = MR_TargetIdToLdGet(ldTgtId: io_info->ldTgtId, map: local_map_ptr);
2398	raid = MR_LdRaidGet(ld, map: local_map_ptr);
2399	if (raid->capability.ldPiMode == MR_PROT_INFO_TYPE_CONTROLLER) {
2400	memset(cdb, 0, sizeof(io_request->CDB.CDB32));
2401	cdb[0] = MEGASAS_SCSI_VARIABLE_LENGTH_CMD;
2402	cdb[7] = MEGASAS_SCSI_ADDL_CDB_LEN;
2403
2404	if (scp->sc_data_direction == DMA_FROM_DEVICE)
2405	cdb[9] = MEGASAS_SCSI_SERVICE_ACTION_READ32;
2406	else
2407	cdb[9] = MEGASAS_SCSI_SERVICE_ACTION_WRITE32;
2408	cdb[10] = MEGASAS_RD_WR_PROTECT_CHECK_ALL;
2409
2410	/* LBA */
2411	cdb[12] = (u8)((start_blk >> 56) & 0xff);
2412	cdb[13] = (u8)((start_blk >> 48) & 0xff);
2413	cdb[14] = (u8)((start_blk >> 40) & 0xff);
2414	cdb[15] = (u8)((start_blk >> 32) & 0xff);
2415	cdb[16] = (u8)((start_blk >> 24) & 0xff);
2416	cdb[17] = (u8)((start_blk >> 16) & 0xff);
2417	cdb[18] = (u8)((start_blk >> 8) & 0xff);
2418	cdb[19] = (u8)(start_blk & 0xff);
2419
2420	/* Logical block reference tag */
2421	io_request->CDB.EEDP32.PrimaryReferenceTag =
2422	cpu_to_be32(ref_tag);
2423	io_request->CDB.EEDP32.PrimaryApplicationTagMask = cpu_to_be16(0xffff);
2424	io_request->IoFlags = cpu_to_le16(32); /* Specify 32-byte cdb */
2425
2426	/* Transfer length */
2427	cdb[28] = (u8)((num_blocks >> 24) & 0xff);
2428	cdb[29] = (u8)((num_blocks >> 16) & 0xff);
2429	cdb[30] = (u8)((num_blocks >> 8) & 0xff);
2430	cdb[31] = (u8)(num_blocks & 0xff);
2431
2432	/* set SCSI IO EEDPFlags */
2433	if (scp->sc_data_direction == DMA_FROM_DEVICE) {
2434	io_request->EEDPFlags = cpu_to_le16(
2435	MPI2_SCSIIO_EEDPFLAGS_INC_PRI_REFTAG |
2436	MPI2_SCSIIO_EEDPFLAGS_CHECK_REFTAG |
2437	MPI2_SCSIIO_EEDPFLAGS_CHECK_REMOVE_OP |
2438	MPI2_SCSIIO_EEDPFLAGS_CHECK_APPTAG |
2439	MPI25_SCSIIO_EEDPFLAGS_DO_NOT_DISABLE_MODE |
2440	MPI2_SCSIIO_EEDPFLAGS_CHECK_GUARD);
2441	} else {
2442	io_request->EEDPFlags = cpu_to_le16(
2443	MPI2_SCSIIO_EEDPFLAGS_INC_PRI_REFTAG |
2444	MPI2_SCSIIO_EEDPFLAGS_INSERT_OP);
2445	}
2446	io_request->Control |= cpu_to_le32((0x4 << 26));
2447	io_request->EEDPBlockSize = cpu_to_le32(scp->device->sector_size);
2448	} else {
2449	/* Some drives don't support 16/12 byte CDB's, convert to 10 */
2450	if (((cdb_len == 12) || (cdb_len == 16)) &&
2451	(start_blk <= 0xffffffff)) {
2452	if (cdb_len == 16) {
2453	opcode = cdb[0] == READ_16 ? READ_10 : WRITE_10;
2454	flagvals = cdb[1];
2455	groupnum = cdb[14];
2456	control = cdb[15];
2457	} else {
2458	opcode = cdb[0] == READ_12 ? READ_10 : WRITE_10;
2459	flagvals = cdb[1];
2460	groupnum = cdb[10];
2461	control = cdb[11];
2462	}
2463
2464	memset(cdb, 0, sizeof(io_request->CDB.CDB32));
2465
2466	cdb[0] = opcode;
2467	cdb[1] = flagvals;
2468	cdb[6] = groupnum;
2469	cdb[9] = control;
2470
2471	/* Transfer length */
2472	cdb[8] = (u8)(num_blocks & 0xff);
2473	cdb[7] = (u8)((num_blocks >> 8) & 0xff);
2474
2475	io_request->IoFlags = cpu_to_le16(10); /* Specify 10-byte cdb */
2476	cdb_len = 10;
2477	} else if ((cdb_len < 16) && (start_blk > 0xffffffff)) {
2478	/* Convert to 16 byte CDB for large LBA's */
2479	switch (cdb_len) {
2480	case 6:
2481	opcode = cdb[0] == READ_6 ? READ_16 : WRITE_16;
2482	control = cdb[5];
2483	break;
2484	case 10:
2485	opcode =
2486	cdb[0] == READ_10 ? READ_16 : WRITE_16;
2487	flagvals = cdb[1];
2488	groupnum = cdb[6];
2489	control = cdb[9];
2490	break;
2491	case 12:
2492	opcode =
2493	cdb[0] == READ_12 ? READ_16 : WRITE_16;
2494	flagvals = cdb[1];
2495	groupnum = cdb[10];
2496	control = cdb[11];
2497	break;
2498	}
2499
2500	memset(cdb, 0, sizeof(io_request->CDB.CDB32));
2501
2502	cdb[0] = opcode;
2503	cdb[1] = flagvals;
2504	cdb[14] = groupnum;
2505	cdb[15] = control;
2506
2507	/* Transfer length */
2508	cdb[13] = (u8)(num_blocks & 0xff);
2509	cdb[12] = (u8)((num_blocks >> 8) & 0xff);
2510	cdb[11] = (u8)((num_blocks >> 16) & 0xff);
2511	cdb[10] = (u8)((num_blocks >> 24) & 0xff);
2512
2513	io_request->IoFlags = cpu_to_le16(16); /* Specify 16-byte cdb */
2514	cdb_len = 16;
2515	}
2516
2517	/* Normal case, just load LBA here */
2518	switch (cdb_len) {
2519	case 6:
2520	{
2521	u8 val = cdb[1] & 0xE0;
2522	cdb[3] = (u8)(start_blk & 0xff);
2523	cdb[2] = (u8)((start_blk >> 8) & 0xff);
2524	cdb[1] = val | ((u8)(start_blk >> 16) & 0x1f);
2525	break;
2526	}
2527	case 10:
2528	cdb[5] = (u8)(start_blk & 0xff);
2529	cdb[4] = (u8)((start_blk >> 8) & 0xff);
2530	cdb[3] = (u8)((start_blk >> 16) & 0xff);
2531	cdb[2] = (u8)((start_blk >> 24) & 0xff);
2532	break;
2533	case 12:
2534	cdb[5] = (u8)(start_blk & 0xff);
2535	cdb[4] = (u8)((start_blk >> 8) & 0xff);
2536	cdb[3] = (u8)((start_blk >> 16) & 0xff);
2537	cdb[2] = (u8)((start_blk >> 24) & 0xff);
2538	break;
2539	case 16:
2540	cdb[9] = (u8)(start_blk & 0xff);
2541	cdb[8] = (u8)((start_blk >> 8) & 0xff);
2542	cdb[7] = (u8)((start_blk >> 16) & 0xff);
2543	cdb[6] = (u8)((start_blk >> 24) & 0xff);
2544	cdb[5] = (u8)((start_blk >> 32) & 0xff);
2545	cdb[4] = (u8)((start_blk >> 40) & 0xff);
2546	cdb[3] = (u8)((start_blk >> 48) & 0xff);
2547	cdb[2] = (u8)((start_blk >> 56) & 0xff);
2548	break;
2549	}
2550	}
2551	}
2552
2553	/**
2554	* megasas_stream_detect - stream detection on read and and write IOs
2555	* @instance: Adapter soft state
2556	* @cmd: Command to be prepared
2557	* @io_info: IO Request info
2558	*
2559	*/
2560
2561	/** stream detection on read and and write IOs */
2562	static void megasas_stream_detect(struct megasas_instance *instance,
2563	struct megasas_cmd_fusion *cmd,
2564	struct IO_REQUEST_INFO *io_info)
2565	{
2566	struct fusion_context *fusion = instance->ctrl_context;
2567	u32 device_id = io_info->ldTgtId;
2568	struct LD_STREAM_DETECT *current_ld_sd
2569	= fusion->stream_detect_by_ld[device_id];
2570	u32 *track_stream = &current_ld_sd->mru_bit_map, stream_num;
2571	u32 shifted_values, unshifted_values;
2572	u32 index_value_mask, shifted_values_mask;
2573	int i;
2574	bool is_read_ahead = false;
2575	struct STREAM_DETECT *current_sd;
2576	/* find possible stream */
2577	for (i = 0; i < MAX_STREAMS_TRACKED; ++i) {
2578	stream_num = (*track_stream >>
2579	(i * BITS_PER_INDEX_STREAM)) &
2580	STREAM_MASK;
2581	current_sd = &current_ld_sd->stream_track[stream_num];
2582	/* if we found a stream, update the raid
2583	* context and also update the mruBitMap
2584	*/
2585	/* boundary condition */
2586	if ((current_sd->next_seq_lba) &&
2587	(io_info->ldStartBlock >= current_sd->next_seq_lba) &&
2588	(io_info->ldStartBlock <= (current_sd->next_seq_lba + 32)) &&
2589	(current_sd->is_read == io_info->isRead)) {
2590
2591	if ((io_info->ldStartBlock != current_sd->next_seq_lba) &&
2592	((!io_info->isRead) || (!is_read_ahead)))
2593	/*
2594	* Once the API is available we need to change this.
2595	* At this point we are not allowing any gap
2596	*/
2597	continue;
2598
2599	SET_STREAM_DETECTED(cmd->io_request->RaidContext.raid_context_g35);
2600	current_sd->next_seq_lba =
2601	io_info->ldStartBlock + io_info->numBlocks;
2602	/*
2603	* update the mruBitMap LRU
2604	*/
2605	shifted_values_mask =
2606	(1 << i * BITS_PER_INDEX_STREAM) - 1;
2607	shifted_values = ((*track_stream & shifted_values_mask)
2608	<< BITS_PER_INDEX_STREAM);
2609	index_value_mask =
2610	STREAM_MASK << i * BITS_PER_INDEX_STREAM;
2611	unshifted_values =
2612	*track_stream & ~(shifted_values_mask |
2613	index_value_mask);
2614	*track_stream =
2615	unshifted_values | shifted_values | stream_num;
2616	return;
2617	}
2618	}
2619	/*
2620	* if we did not find any stream, create a new one
2621	* from the least recently used
2622	*/
2623	stream_num = (*track_stream >>
2624	((MAX_STREAMS_TRACKED - 1) * BITS_PER_INDEX_STREAM)) &
2625	STREAM_MASK;
2626	current_sd = &current_ld_sd->stream_track[stream_num];
2627	current_sd->is_read = io_info->isRead;
2628	current_sd->next_seq_lba = io_info->ldStartBlock + io_info->numBlocks;
2629	*track_stream = (((*track_stream & ZERO_LAST_STREAM) << 4) | stream_num);
2630	return;
2631	}
2632
2633	/**
2634	* megasas_set_raidflag_cpu_affinity - This function sets the cpu
2635	* affinity (cpu of the controller) and raid_flags in the raid context
2636	* based on IO type.
2637	*
2638	* @fusion: Fusion context
2639	* @praid_context: IO RAID context
2640	* @raid: LD raid map
2641	* @fp_possible: Is fast path possible?
2642	* @is_read: Is read IO?
2643	* @scsi_buff_len: SCSI command buffer length
2644	*
2645	*/
2646	static void
2647	megasas_set_raidflag_cpu_affinity(struct fusion_context *fusion,
2648	union RAID_CONTEXT_UNION *praid_context,
2649	struct MR_LD_RAID *raid, bool fp_possible,
2650	u8 is_read, u32 scsi_buff_len)
2651	{
2652	u8 cpu_sel = MR_RAID_CTX_CPUSEL_0;
2653	struct RAID_CONTEXT_G35 *rctx_g35;
2654
2655	rctx_g35 = &praid_context->raid_context_g35;
2656	if (fp_possible) {
2657	if (is_read) {
2658	if ((raid->cpuAffinity.pdRead.cpu0) &&
2659	(raid->cpuAffinity.pdRead.cpu1))
2660	cpu_sel = MR_RAID_CTX_CPUSEL_FCFS;
2661	else if (raid->cpuAffinity.pdRead.cpu1)
2662	cpu_sel = MR_RAID_CTX_CPUSEL_1;
2663	} else {
2664	if ((raid->cpuAffinity.pdWrite.cpu0) &&
2665	(raid->cpuAffinity.pdWrite.cpu1))
2666	cpu_sel = MR_RAID_CTX_CPUSEL_FCFS;
2667	else if (raid->cpuAffinity.pdWrite.cpu1)
2668	cpu_sel = MR_RAID_CTX_CPUSEL_1;
2669	/* Fast path cache by pass capable R0/R1 VD */
2670	if ((raid->level <= 1) &&
2671	(raid->capability.fp_cache_bypass_capable)) {
2672	rctx_g35->routing_flags |=
2673	(1 << MR_RAID_CTX_ROUTINGFLAGS_SLD_SHIFT);
2674	rctx_g35->raid_flags =
2675	(MR_RAID_FLAGS_IO_SUB_TYPE_CACHE_BYPASS
2676	<< MR_RAID_CTX_RAID_FLAGS_IO_SUB_TYPE_SHIFT);
2677	}
2678	}
2679	} else {
2680	if (is_read) {
2681	if ((raid->cpuAffinity.ldRead.cpu0) &&
2682	(raid->cpuAffinity.ldRead.cpu1))
2683	cpu_sel = MR_RAID_CTX_CPUSEL_FCFS;
2684	else if (raid->cpuAffinity.ldRead.cpu1)
2685	cpu_sel = MR_RAID_CTX_CPUSEL_1;
2686	} else {
2687	if ((raid->cpuAffinity.ldWrite.cpu0) &&
2688	(raid->cpuAffinity.ldWrite.cpu1))
2689	cpu_sel = MR_RAID_CTX_CPUSEL_FCFS;
2690	else if (raid->cpuAffinity.ldWrite.cpu1)
2691	cpu_sel = MR_RAID_CTX_CPUSEL_1;
2692
2693	if (is_stream_detected(rctx_g35) &&
2694	((raid->level == 5) || (raid->level == 6)) &&
2695	(raid->writeMode == MR_RL_WRITE_THROUGH_MODE) &&
2696	(cpu_sel == MR_RAID_CTX_CPUSEL_FCFS))
2697	cpu_sel = MR_RAID_CTX_CPUSEL_0;
2698	}
2699	}
2700
2701	rctx_g35->routing_flags |=
2702	(cpu_sel << MR_RAID_CTX_ROUTINGFLAGS_CPUSEL_SHIFT);
2703
2704	/* Always give priority to MR_RAID_FLAGS_IO_SUB_TYPE_LDIO_BW_LIMIT
2705	* vs MR_RAID_FLAGS_IO_SUB_TYPE_CACHE_BYPASS.
2706	* IO Subtype is not bitmap.
2707	*/
2708	if ((fusion->pcie_bw_limitation) && (raid->level == 1) && (!is_read) &&
2709	(scsi_buff_len > MR_LARGE_IO_MIN_SIZE)) {
2710	praid_context->raid_context_g35.raid_flags =
2711	(MR_RAID_FLAGS_IO_SUB_TYPE_LDIO_BW_LIMIT
2712	<< MR_RAID_CTX_RAID_FLAGS_IO_SUB_TYPE_SHIFT);
2713	}
2714	}
2715
2716	/**
2717	* megasas_build_ldio_fusion - Prepares IOs to devices
2718	* @instance: Adapter soft state
2719	* @scp: SCSI command
2720	* @cmd: Command to be prepared
2721	*
2722	* Prepares the io_request and chain elements (sg_frame) for IO
2723	* The IO can be for PD (Fast Path) or LD
2724	*/
2725	static void
2726	megasas_build_ldio_fusion(struct megasas_instance *instance,
2727	struct scsi_cmnd *scp,
2728	struct megasas_cmd_fusion *cmd)
2729	{
2730	bool fp_possible;
2731	u16 ld;
2732	u32 start_lba_lo, start_lba_hi, device_id, datalength = 0;
2733	u32 scsi_buff_len;
2734	struct MPI2_RAID_SCSI_IO_REQUEST *io_request;
2735	struct IO_REQUEST_INFO io_info;
2736	struct fusion_context *fusion;
2737	struct MR_DRV_RAID_MAP_ALL *local_map_ptr;
2738	u8 *raidLUN;
2739	unsigned long spinlock_flags;
2740	struct MR_LD_RAID *raid = NULL;
2741	struct MR_PRIV_DEVICE *mrdev_priv;
2742	struct RAID_CONTEXT *rctx;
2743	struct RAID_CONTEXT_G35 *rctx_g35;
2744
2745	device_id = MEGASAS_DEV_INDEX(scp);
2746
2747	fusion = instance->ctrl_context;
2748
2749	io_request = cmd->io_request;
2750	rctx = &io_request->RaidContext.raid_context;
2751	rctx_g35 = &io_request->RaidContext.raid_context_g35;
2752
2753	rctx->virtual_disk_tgt_id = cpu_to_le16(device_id);
2754	rctx->status = 0;
2755	rctx->ex_status = 0;
2756
2757	start_lba_lo = 0;
2758	start_lba_hi = 0;
2759	fp_possible = false;
2760
2761	/*
2762	* 6-byte READ(0x08) or WRITE(0x0A) cdb
2763	*/
2764	if (scp->cmd_len == 6) {
2765	datalength = (u32) scp->cmnd[4];
2766	start_lba_lo = ((u32) scp->cmnd[1] << 16) |
2767	((u32) scp->cmnd[2] << 8) | (u32) scp->cmnd[3];
2768
2769	start_lba_lo &= 0x1FFFFF;
2770	}
2771
2772	/*
2773	* 10-byte READ(0x28) or WRITE(0x2A) cdb
2774	*/
2775	else if (scp->cmd_len == 10) {
2776	datalength = (u32) scp->cmnd[8] |
2777	((u32) scp->cmnd[7] << 8);
2778	start_lba_lo = ((u32) scp->cmnd[2] << 24) |
2779	((u32) scp->cmnd[3] << 16) |
2780	((u32) scp->cmnd[4] << 8) | (u32) scp->cmnd[5];
2781	}
2782
2783	/*
2784	* 12-byte READ(0xA8) or WRITE(0xAA) cdb
2785	*/
2786	else if (scp->cmd_len == 12) {
2787	datalength = ((u32) scp->cmnd[6] << 24) |
2788	((u32) scp->cmnd[7] << 16) |
2789	((u32) scp->cmnd[8] << 8) | (u32) scp->cmnd[9];
2790	start_lba_lo = ((u32) scp->cmnd[2] << 24) |
2791	((u32) scp->cmnd[3] << 16) |
2792	((u32) scp->cmnd[4] << 8) | (u32) scp->cmnd[5];
2793	}
2794
2795	/*
2796	* 16-byte READ(0x88) or WRITE(0x8A) cdb
2797	*/
2798	else if (scp->cmd_len == 16) {
2799	datalength = ((u32) scp->cmnd[10] << 24) |
2800	((u32) scp->cmnd[11] << 16) |
2801	((u32) scp->cmnd[12] << 8) | (u32) scp->cmnd[13];
2802	start_lba_lo = ((u32) scp->cmnd[6] << 24) |
2803	((u32) scp->cmnd[7] << 16) |
2804	((u32) scp->cmnd[8] << 8) | (u32) scp->cmnd[9];
2805
2806	start_lba_hi = ((u32) scp->cmnd[2] << 24) |
2807	((u32) scp->cmnd[3] << 16) |
2808	((u32) scp->cmnd[4] << 8) | (u32) scp->cmnd[5];
2809	}
2810
2811	memset(&io_info, 0, sizeof(struct IO_REQUEST_INFO));
2812	io_info.ldStartBlock = ((u64)start_lba_hi << 32) | start_lba_lo;
2813	io_info.numBlocks = datalength;
2814	io_info.ldTgtId = device_id;
2815	io_info.r1_alt_dev_handle = MR_DEVHANDLE_INVALID;
2816	scsi_buff_len = scsi_bufflen(cmd: scp);
2817	io_request->DataLength = cpu_to_le32(scsi_buff_len);
2818	io_info.data_arms = 1;
2819
2820	if (scp->sc_data_direction == DMA_FROM_DEVICE)
2821	io_info.isRead = 1;
2822
2823	local_map_ptr = fusion->ld_drv_map[(instance->map_id & 1)];
2824	ld = MR_TargetIdToLdGet(ldTgtId: device_id, map: local_map_ptr);
2825
2826	if (ld < instance->fw_supported_vd_count)
2827	raid = MR_LdRaidGet(ld, map: local_map_ptr);
2828
2829	if (!raid || (!fusion->fast_path_io)) {
2830	rctx->reg_lock_flags = 0;
2831	fp_possible = false;
2832	} else {
2833	if (MR_BuildRaidContext(instance, io_info: &io_info, pRAID_Context: rctx,
2834	map: local_map_ptr, raidLUN: &raidLUN))
2835	fp_possible = (io_info.fpOkForIo > 0) ? true : false;
2836	}
2837
2838	megasas_get_msix_index(instance, scmd: scp, cmd, data_arms: io_info.data_arms);
2839
2840	if (instance->adapter_type >= VENTURA_SERIES) {
2841	/* FP for Optimal raid level 1.
2842	* All large RAID-1 writes (> 32 KiB, both WT and WB modes)
2843	* are built by the driver as LD I/Os.
2844	* All small RAID-1 WT writes (<= 32 KiB) are built as FP I/Os
2845	* (there is never a reason to process these as buffered writes)
2846	* All small RAID-1 WB writes (<= 32 KiB) are built as FP I/Os
2847	* with the SLD bit asserted.
2848	*/
2849	if (io_info.r1_alt_dev_handle != MR_DEVHANDLE_INVALID) {
2850	mrdev_priv = scp->device->hostdata;
2851
2852	if (atomic_inc_return(v: &instance->fw_outstanding) >
2853	(instance->host->can_queue)) {
2854	fp_possible = false;
2855	atomic_dec(v: &instance->fw_outstanding);
2856	} else if (fusion->pcie_bw_limitation &&
2857	((scsi_buff_len > MR_LARGE_IO_MIN_SIZE) ||
2858	(atomic_dec_if_positive(v: &mrdev_priv->r1_ldio_hint) > 0))) {
2859	fp_possible = false;
2860	atomic_dec(v: &instance->fw_outstanding);
2861	if (scsi_buff_len > MR_LARGE_IO_MIN_SIZE)
2862	atomic_set(v: &mrdev_priv->r1_ldio_hint,
2863	i: instance->r1_ldio_hint_default);
2864	}
2865	}
2866
2867	if (!fp_possible ||
2868	(io_info.isRead && io_info.ra_capable)) {
2869	spin_lock_irqsave(&instance->stream_lock,
2870	spinlock_flags);
2871	megasas_stream_detect(instance, cmd, io_info: &io_info);
2872	spin_unlock_irqrestore(lock: &instance->stream_lock,
2873	flags: spinlock_flags);
2874	/* In ventura if stream detected for a read and it is
2875	* read ahead capable make this IO as LDIO
2876	*/
2877	if (is_stream_detected(rctx_g35))
2878	fp_possible = false;
2879	}
2880
2881	/* If raid is NULL, set CPU affinity to default CPU0 */
2882	if (raid)
2883	megasas_set_raidflag_cpu_affinity(fusion, praid_context: &io_request->RaidContext,
2884	raid, fp_possible, is_read: io_info.isRead,
2885	scsi_buff_len);
2886	else
2887	rctx_g35->routing_flags |=
2888	(MR_RAID_CTX_CPUSEL_0 << MR_RAID_CTX_ROUTINGFLAGS_CPUSEL_SHIFT);
2889	}
2890
2891	if (fp_possible) {
2892	megasas_set_pd_lba(io_request, cdb_len: scp->cmd_len, io_info: &io_info, scp,
2893	local_map_ptr, ref_tag: start_lba_lo);
2894	io_request->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
2895	cmd->request_desc->SCSIIO.RequestFlags =
2896	(MPI2_REQ_DESCRIPT_FLAGS_FP_IO
2897	<< MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
2898	if (instance->adapter_type == INVADER_SERIES) {
2899	rctx->type = MPI2_TYPE_CUDA;
2900	rctx->nseg = 0x1;
2901	io_request->IoFlags |= cpu_to_le16(MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH);
2902	rctx->reg_lock_flags |=
2903	(MR_RL_FLAGS_GRANT_DESTINATION_CUDA |
2904	MR_RL_FLAGS_SEQ_NUM_ENABLE);
2905	} else if (instance->adapter_type >= VENTURA_SERIES) {
2906	rctx_g35->nseg_type |= (1 << RAID_CONTEXT_NSEG_SHIFT);
2907	rctx_g35->nseg_type |= (MPI2_TYPE_CUDA << RAID_CONTEXT_TYPE_SHIFT);
2908	rctx_g35->routing_flags |= (1 << MR_RAID_CTX_ROUTINGFLAGS_SQN_SHIFT);
2909	io_request->IoFlags |=
2910	cpu_to_le16(MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH);
2911	}
2912	if (fusion->load_balance_info &&
2913	(fusion->load_balance_info[device_id].loadBalanceFlag) &&
2914	(io_info.isRead)) {
2915	io_info.devHandle =
2916	get_updated_dev_handle(instance,
2917	lbInfo: &fusion->load_balance_info[device_id],
2918	in_info: &io_info, drv_map: local_map_ptr);
2919	megasas_priv(cmd: scp)->status |= MEGASAS_LOAD_BALANCE_FLAG;
2920	cmd->pd_r1_lb = io_info.pd_after_lb;
2921	if (instance->adapter_type >= VENTURA_SERIES)
2922	rctx_g35->span_arm = io_info.span_arm;
2923	else
2924	rctx->span_arm = io_info.span_arm;
2925
2926	} else
2927	megasas_priv(cmd: scp)->status &= ~MEGASAS_LOAD_BALANCE_FLAG;
2928
2929	if (instance->adapter_type >= VENTURA_SERIES)
2930	cmd->r1_alt_dev_handle = io_info.r1_alt_dev_handle;
2931	else
2932	cmd->r1_alt_dev_handle = MR_DEVHANDLE_INVALID;
2933
2934	if ((raidLUN[0] == 1) &&
2935	(local_map_ptr->raidMap.devHndlInfo[io_info.pd_after_lb].validHandles > 1)) {
2936	instance->dev_handle = !(instance->dev_handle);
2937	io_info.devHandle =
2938	local_map_ptr->raidMap.devHndlInfo[io_info.pd_after_lb].devHandle[instance->dev_handle];
2939	}
2940
2941	cmd->request_desc->SCSIIO.DevHandle = io_info.devHandle;
2942	io_request->DevHandle = io_info.devHandle;
2943	cmd->pd_interface = io_info.pd_interface;
2944	/* populate the LUN field */
2945	memcpy(io_request->LUN, raidLUN, 8);
2946	} else {
2947	rctx->timeout_value =
2948	cpu_to_le16(local_map_ptr->raidMap.fpPdIoTimeoutSec);
2949	cmd->request_desc->SCSIIO.RequestFlags =
2950	(MEGASAS_REQ_DESCRIPT_FLAGS_LD_IO
2951	<< MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
2952	if (instance->adapter_type == INVADER_SERIES) {
2953	if (io_info.do_fp_rlbypass ||
2954	(rctx->reg_lock_flags == REGION_TYPE_UNUSED))
2955	cmd->request_desc->SCSIIO.RequestFlags =
2956	(MEGASAS_REQ_DESCRIPT_FLAGS_NO_LOCK <<
2957	MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
2958	rctx->type = MPI2_TYPE_CUDA;
2959	rctx->reg_lock_flags |=
2960	(MR_RL_FLAGS_GRANT_DESTINATION_CPU0 |
2961	MR_RL_FLAGS_SEQ_NUM_ENABLE);
2962	rctx->nseg = 0x1;
2963	} else if (instance->adapter_type >= VENTURA_SERIES) {
2964	rctx_g35->routing_flags |= (1 << MR_RAID_CTX_ROUTINGFLAGS_SQN_SHIFT);
2965	rctx_g35->nseg_type |= (1 << RAID_CONTEXT_NSEG_SHIFT);
2966	rctx_g35->nseg_type |= (MPI2_TYPE_CUDA << RAID_CONTEXT_TYPE_SHIFT);
2967	}
2968	io_request->Function = MEGASAS_MPI2_FUNCTION_LD_IO_REQUEST;
2969	io_request->DevHandle = cpu_to_le16(device_id);
2970
2971	} /* Not FP */
2972	}
2973
2974	/**
2975	* megasas_build_ld_nonrw_fusion - prepares non rw ios for virtual disk
2976	* @instance: Adapter soft state
2977	* @scmd: SCSI command
2978	* @cmd: Command to be prepared
2979	*
2980	* Prepares the io_request frame for non-rw io cmds for vd.
2981	*/
2982	static void megasas_build_ld_nonrw_fusion(struct megasas_instance *instance,
2983	struct scsi_cmnd *scmd, struct megasas_cmd_fusion *cmd)
2984	{
2985	u32 device_id;
2986	struct MPI2_RAID_SCSI_IO_REQUEST *io_request;
2987	u16 ld;
2988	struct MR_DRV_RAID_MAP_ALL *local_map_ptr;
2989	struct fusion_context *fusion = instance->ctrl_context;
2990	u8 span, physArm;
2991	__le16 devHandle;
2992	u32 arRef, pd;
2993	struct MR_LD_RAID *raid;
2994	struct RAID_CONTEXT *pRAID_Context;
2995	u8 fp_possible = 1;
2996
2997	io_request = cmd->io_request;
2998	device_id = MEGASAS_DEV_INDEX(scmd);
2999	local_map_ptr = fusion->ld_drv_map[(instance->map_id & 1)];
3000	io_request->DataLength = cpu_to_le32(scsi_bufflen(scmd));
3001	/* get RAID_Context pointer */
3002	pRAID_Context = &io_request->RaidContext.raid_context;
3003	/* Check with FW team */
3004	pRAID_Context->virtual_disk_tgt_id = cpu_to_le16(device_id);
3005	pRAID_Context->reg_lock_row_lba = 0;
3006	pRAID_Context->reg_lock_length = 0;
3007
3008	if (fusion->fast_path_io && (
3009	device_id < instance->fw_supported_vd_count)) {
3010
3011	ld = MR_TargetIdToLdGet(ldTgtId: device_id, map: local_map_ptr);
3012	if (ld >= instance->fw_supported_vd_count - 1)
3013	fp_possible = 0;
3014	else {
3015	raid = MR_LdRaidGet(ld, map: local_map_ptr);
3016	if (!(raid->capability.fpNonRWCapable))
3017	fp_possible = 0;
3018	}
3019	} else
3020	fp_possible = 0;
3021
3022	if (!fp_possible) {
3023	io_request->Function = MEGASAS_MPI2_FUNCTION_LD_IO_REQUEST;
3024	io_request->DevHandle = cpu_to_le16(device_id);
3025	io_request->LUN[1] = scmd->device->lun;
3026	pRAID_Context->timeout_value =
3027	cpu_to_le16(scsi_cmd_to_rq(scmd)->timeout / HZ);
3028	cmd->request_desc->SCSIIO.RequestFlags =
3029	(MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO <<
3030	MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
3031	} else {
3032
3033	/* set RAID context values */
3034	pRAID_Context->config_seq_num = raid->seqNum;
3035	if (instance->adapter_type < VENTURA_SERIES)
3036	pRAID_Context->reg_lock_flags = REGION_TYPE_SHARED_READ;
3037	pRAID_Context->timeout_value =
3038	cpu_to_le16(raid->fpIoTimeoutForLd);
3039
3040	/* get the DevHandle for the PD (since this is
3041	fpNonRWCapable, this is a single disk RAID0) */
3042	span = physArm = 0;
3043	arRef = MR_LdSpanArrayGet(ld, span, map: local_map_ptr);
3044	pd = MR_ArPdGet(ar: arRef, arm: physArm, map: local_map_ptr);
3045	devHandle = MR_PdDevHandleGet(pd, map: local_map_ptr);
3046
3047	/* build request descriptor */
3048	cmd->request_desc->SCSIIO.RequestFlags =
3049	(MPI2_REQ_DESCRIPT_FLAGS_FP_IO <<
3050	MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
3051	cmd->request_desc->SCSIIO.DevHandle = devHandle;
3052
3053	/* populate the LUN field */
3054	memcpy(io_request->LUN, raid->LUN, 8);
3055
3056	/* build the raidScsiIO structure */
3057	io_request->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
3058	io_request->DevHandle = devHandle;
3059	}
3060	}
3061
3062	/**
3063	* megasas_build_syspd_fusion - prepares rw/non-rw ios for syspd
3064	* @instance: Adapter soft state
3065	* @scmd: SCSI command
3066	* @cmd: Command to be prepared
3067	* @fp_possible: parameter to detect fast path or firmware path io.
3068	*
3069	* Prepares the io_request frame for rw/non-rw io cmds for syspds
3070	*/
3071	static void
3072	megasas_build_syspd_fusion(struct megasas_instance *instance,
3073	struct scsi_cmnd *scmd, struct megasas_cmd_fusion *cmd,
3074	bool fp_possible)
3075	{
3076	u32 device_id;
3077	struct MPI2_RAID_SCSI_IO_REQUEST *io_request;
3078	u16 pd_index = 0;
3079	u16 os_timeout_value;
3080	u16 timeout_limit;
3081	struct MR_DRV_RAID_MAP_ALL *local_map_ptr;
3082	struct RAID_CONTEXT *pRAID_Context;
3083	struct MR_PD_CFG_SEQ_NUM_SYNC *pd_sync;
3084	struct MR_PRIV_DEVICE *mr_device_priv_data;
3085	struct fusion_context *fusion = instance->ctrl_context;
3086	pd_sync = (void *)fusion->pd_seq_sync[(instance->pd_seq_map_id - 1) & 1];
3087
3088	device_id = MEGASAS_DEV_INDEX(scmd);
3089	pd_index = MEGASAS_PD_INDEX(scmd);
3090	os_timeout_value = scsi_cmd_to_rq(scmd)->timeout / HZ;
3091	mr_device_priv_data = scmd->device->hostdata;
3092	cmd->pd_interface = mr_device_priv_data->interface_type;
3093
3094	io_request = cmd->io_request;
3095	/* get RAID_Context pointer */
3096	pRAID_Context = &io_request->RaidContext.raid_context;
3097	pRAID_Context->reg_lock_flags = 0;
3098	pRAID_Context->reg_lock_row_lba = 0;
3099	pRAID_Context->reg_lock_length = 0;
3100	io_request->DataLength = cpu_to_le32(scsi_bufflen(scmd));
3101	io_request->LUN[1] = scmd->device->lun;
3102	pRAID_Context->raid_flags = MR_RAID_FLAGS_IO_SUB_TYPE_SYSTEM_PD
3103	<< MR_RAID_CTX_RAID_FLAGS_IO_SUB_TYPE_SHIFT;
3104
3105	/* If FW supports PD sequence number */
3106	if (instance->support_seqnum_jbod_fp) {
3107	if (instance->use_seqnum_jbod_fp &&
3108	instance->pd_list[pd_index].driveType == TYPE_DISK) {
3109
3110	/* More than 256 PD/JBOD support for Ventura */
3111	if (instance->support_morethan256jbod)
3112	pRAID_Context->virtual_disk_tgt_id =
3113	pd_sync->seq[pd_index].pd_target_id;
3114	else
3115	pRAID_Context->virtual_disk_tgt_id =
3116	cpu_to_le16(device_id +
3117	(MAX_PHYSICAL_DEVICES - 1));
3118	pRAID_Context->config_seq_num =
3119	pd_sync->seq[pd_index].seqNum;
3120	io_request->DevHandle =
3121	pd_sync->seq[pd_index].devHandle;
3122	if (instance->adapter_type >= VENTURA_SERIES) {
3123	io_request->RaidContext.raid_context_g35.routing_flags |=
3124	(1 << MR_RAID_CTX_ROUTINGFLAGS_SQN_SHIFT);
3125	io_request->RaidContext.raid_context_g35.nseg_type |=
3126	(1 << RAID_CONTEXT_NSEG_SHIFT);
3127	io_request->RaidContext.raid_context_g35.nseg_type |=
3128	(MPI2_TYPE_CUDA << RAID_CONTEXT_TYPE_SHIFT);
3129	} else {
3130	pRAID_Context->type = MPI2_TYPE_CUDA;
3131	pRAID_Context->nseg = 0x1;
3132	pRAID_Context->reg_lock_flags |=
3133	(MR_RL_FLAGS_SEQ_NUM_ENABLE |
3134	MR_RL_FLAGS_GRANT_DESTINATION_CUDA);
3135	}
3136	} else {
3137	pRAID_Context->virtual_disk_tgt_id =
3138	cpu_to_le16(device_id +
3139	(MAX_PHYSICAL_DEVICES - 1));
3140	pRAID_Context->config_seq_num = 0;
3141	io_request->DevHandle = cpu_to_le16(0xFFFF);
3142	}
3143	} else {
3144	pRAID_Context->virtual_disk_tgt_id = cpu_to_le16(device_id);
3145	pRAID_Context->config_seq_num = 0;
3146
3147	if (fusion->fast_path_io) {
3148	local_map_ptr =
3149	fusion->ld_drv_map[(instance->map_id & 1)];
3150	io_request->DevHandle =
3151	local_map_ptr->raidMap.devHndlInfo[device_id].curDevHdl;
3152	} else {
3153	io_request->DevHandle = cpu_to_le16(0xFFFF);
3154	}
3155	}
3156
3157	cmd->request_desc->SCSIIO.DevHandle = io_request->DevHandle;
3158
3159	megasas_get_msix_index(instance, scmd, cmd, data_arms: 1);
3160
3161	if (!fp_possible) {
3162	/* system pd firmware path */
3163	io_request->Function = MEGASAS_MPI2_FUNCTION_LD_IO_REQUEST;
3164	cmd->request_desc->SCSIIO.RequestFlags =
3165	(MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO <<
3166	MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
3167	pRAID_Context->timeout_value = cpu_to_le16(os_timeout_value);
3168	pRAID_Context->virtual_disk_tgt_id = cpu_to_le16(device_id);
3169	} else {
3170	if (os_timeout_value)
3171	os_timeout_value++;
3172
3173	/* system pd Fast Path */
3174	io_request->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
3175	timeout_limit = (scmd->device->type == TYPE_DISK) ?
3176	255 : 0xFFFF;
3177	pRAID_Context->timeout_value =
3178	cpu_to_le16((os_timeout_value > timeout_limit) ?
3179	timeout_limit : os_timeout_value);
3180	if (instance->adapter_type >= INVADER_SERIES)
3181	io_request->IoFlags |=
3182	cpu_to_le16(MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH);
3183
3184	cmd->request_desc->SCSIIO.RequestFlags =
3185	(MPI2_REQ_DESCRIPT_FLAGS_FP_IO <<
3186	MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
3187	}
3188	}
3189
3190	/**
3191	* megasas_build_io_fusion - Prepares IOs to devices
3192	* @instance: Adapter soft state
3193	* @scp: SCSI command
3194	* @cmd: Command to be prepared
3195	*
3196	* Invokes helper functions to prepare request frames
3197	* and sets flags appropriate for IO/Non-IO cmd
3198	*/
3199	static int
3200	megasas_build_io_fusion(struct megasas_instance *instance,
3201	struct scsi_cmnd *scp,
3202	struct megasas_cmd_fusion *cmd)
3203	{
3204	int sge_count;
3205	u16 pd_index = 0;
3206	u8 drive_type = 0;
3207	struct MPI2_RAID_SCSI_IO_REQUEST *io_request = cmd->io_request;
3208	struct MR_PRIV_DEVICE *mr_device_priv_data;
3209	mr_device_priv_data = scp->device->hostdata;
3210
3211	/* Zero out some fields so they don't get reused */
3212	memset(io_request->LUN, 0x0, 8);
3213	io_request->CDB.EEDP32.PrimaryReferenceTag = 0;
3214	io_request->CDB.EEDP32.PrimaryApplicationTagMask = 0;
3215	io_request->EEDPFlags = 0;
3216	io_request->Control = 0;
3217	io_request->EEDPBlockSize = 0;
3218	io_request->ChainOffset = 0;
3219	io_request->RaidContext.raid_context.raid_flags = 0;
3220	io_request->RaidContext.raid_context.type = 0;
3221	io_request->RaidContext.raid_context.nseg = 0;
3222
3223	memcpy(io_request->CDB.CDB32, scp->cmnd, scp->cmd_len);
3224	/*
3225	* Just the CDB length,rest of the Flags are zero
3226	* This will be modified for FP in build_ldio_fusion
3227	*/
3228	io_request->IoFlags = cpu_to_le16(scp->cmd_len);
3229
3230	switch (megasas_cmd_type(cmd: scp)) {
3231	case READ_WRITE_LDIO:
3232	megasas_build_ldio_fusion(instance, scp, cmd);
3233	break;
3234	case NON_READ_WRITE_LDIO:
3235	megasas_build_ld_nonrw_fusion(instance, scmd: scp, cmd);
3236	break;
3237	case READ_WRITE_SYSPDIO:
3238	megasas_build_syspd_fusion(instance, scmd: scp, cmd, fp_possible: true);
3239	break;
3240	case NON_READ_WRITE_SYSPDIO:
3241	pd_index = MEGASAS_PD_INDEX(scp);
3242	drive_type = instance->pd_list[pd_index].driveType;
3243	if ((instance->secure_jbod_support ||
3244	mr_device_priv_data->is_tm_capable) ||
3245	(instance->adapter_type >= VENTURA_SERIES &&
3246	drive_type == TYPE_ENCLOSURE))
3247	megasas_build_syspd_fusion(instance, scmd: scp, cmd, fp_possible: false);
3248	else
3249	megasas_build_syspd_fusion(instance, scmd: scp, cmd, fp_possible: true);
3250	break;
3251	default:
3252	break;
3253	}
3254
3255	/*
3256	* Construct SGL
3257	*/
3258
3259	sge_count = megasas_make_sgl(instance, scp, cmd);
3260
3261	if (sge_count > instance->max_num_sge || (sge_count < 0)) {
3262	dev_err(&instance->pdev->dev,
3263	"%s %d sge_count (%d) is out of range. Range is: 0-%d\n",
3264	__func__, __LINE__, sge_count, instance->max_num_sge);
3265	return 1;
3266	}
3267
3268	if (instance->adapter_type >= VENTURA_SERIES) {
3269	set_num_sge(rctx_g35: &io_request->RaidContext.raid_context_g35, sge_count);
3270	cpu_to_le16s(&io_request->RaidContext.raid_context_g35.routing_flags);
3271	cpu_to_le16s(&io_request->RaidContext.raid_context_g35.nseg_type);
3272	} else {
3273	/* numSGE store lower 8 bit of sge_count.
3274	* numSGEExt store higher 8 bit of sge_count
3275	*/
3276	io_request->RaidContext.raid_context.num_sge = sge_count;
3277	io_request->RaidContext.raid_context.num_sge_ext =
3278	(u8)(sge_count >> 8);
3279	}
3280
3281	io_request->SGLFlags = cpu_to_le16(MPI2_SGE_FLAGS_64_BIT_ADDRESSING);
3282
3283	if (scp->sc_data_direction == DMA_TO_DEVICE)
3284	io_request->Control |= cpu_to_le32(MPI2_SCSIIO_CONTROL_WRITE);
3285	else if (scp->sc_data_direction == DMA_FROM_DEVICE)
3286	io_request->Control |= cpu_to_le32(MPI2_SCSIIO_CONTROL_READ);
3287
3288	io_request->SGLOffset0 =
3289	offsetof(struct MPI2_RAID_SCSI_IO_REQUEST, SGL) / 4;
3290
3291	io_request->SenseBufferLowAddress =
3292	cpu_to_le32(lower_32_bits(cmd->sense_phys_addr));
3293	io_request->SenseBufferLength = SCSI_SENSE_BUFFERSIZE;
3294
3295	cmd->scmd = scp;
3296	megasas_priv(cmd: scp)->cmd_priv = cmd;
3297
3298	return 0;
3299	}
3300
3301	static union MEGASAS_REQUEST_DESCRIPTOR_UNION *
3302	megasas_get_request_descriptor(struct megasas_instance *instance, u16 index)
3303	{
3304	u8 *p;
3305	struct fusion_context *fusion;
3306
3307	fusion = instance->ctrl_context;
3308	p = fusion->req_frames_desc +
3309	sizeof(union MEGASAS_REQUEST_DESCRIPTOR_UNION) * index;
3310
3311	return (union MEGASAS_REQUEST_DESCRIPTOR_UNION *)p;
3312	}
3313
3314
3315	/* megasas_prepate_secondRaid1_IO
3316	* It prepares the raid 1 second IO
3317	*/
3318	static void megasas_prepare_secondRaid1_IO(struct megasas_instance *instance,
3319	struct megasas_cmd_fusion *cmd,
3320	struct megasas_cmd_fusion *r1_cmd)
3321	{
3322	union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc, *req_desc2 = NULL;
3323	struct fusion_context *fusion;
3324	fusion = instance->ctrl_context;
3325	req_desc = cmd->request_desc;
3326	/* copy the io request frame as well as 8 SGEs data for r1 command*/
3327	memcpy(r1_cmd->io_request, cmd->io_request,
3328	(sizeof(struct MPI2_RAID_SCSI_IO_REQUEST)));
3329	memcpy(r1_cmd->io_request->SGLs, cmd->io_request->SGLs,
3330	(fusion->max_sge_in_main_msg * sizeof(union MPI2_SGE_IO_UNION)));
3331	/*sense buffer is different for r1 command*/
3332	r1_cmd->io_request->SenseBufferLowAddress =
3333	cpu_to_le32(lower_32_bits(r1_cmd->sense_phys_addr));
3334	r1_cmd->scmd = cmd->scmd;
3335	req_desc2 = megasas_get_request_descriptor(instance,
3336	index: (r1_cmd->index - 1));
3337	req_desc2->Words = 0;
3338	r1_cmd->request_desc = req_desc2;
3339	req_desc2->SCSIIO.SMID = cpu_to_le16(r1_cmd->index);
3340	req_desc2->SCSIIO.RequestFlags = req_desc->SCSIIO.RequestFlags;
3341	r1_cmd->request_desc->SCSIIO.DevHandle = cmd->r1_alt_dev_handle;
3342	r1_cmd->io_request->DevHandle = cmd->r1_alt_dev_handle;
3343	r1_cmd->r1_alt_dev_handle = cmd->io_request->DevHandle;
3344	cmd->io_request->RaidContext.raid_context_g35.flow_specific.peer_smid =
3345	cpu_to_le16(r1_cmd->index);
3346	r1_cmd->io_request->RaidContext.raid_context_g35.flow_specific.peer_smid =
3347	cpu_to_le16(cmd->index);
3348	/*MSIxIndex of both commands request descriptors should be same*/
3349	r1_cmd->request_desc->SCSIIO.MSIxIndex =
3350	cmd->request_desc->SCSIIO.MSIxIndex;
3351	/*span arm is different for r1 cmd*/
3352	r1_cmd->io_request->RaidContext.raid_context_g35.span_arm =
3353	cmd->io_request->RaidContext.raid_context_g35.span_arm + 1;
3354	}
3355
3356	/**
3357	* megasas_build_and_issue_cmd_fusion -Main routine for building and
3358	* issuing non IOCTL cmd
3359	* @instance: Adapter soft state
3360	* @scmd: pointer to scsi cmd from OS
3361	*/
3362	static u32
3363	megasas_build_and_issue_cmd_fusion(struct megasas_instance *instance,
3364	struct scsi_cmnd *scmd)
3365	{
3366	struct megasas_cmd_fusion *cmd, *r1_cmd = NULL;
3367	union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc;
3368	u32 index;
3369
3370	if ((megasas_cmd_type(cmd: scmd) == READ_WRITE_LDIO) &&
3371	instance->ldio_threshold &&
3372	(atomic_inc_return(v: &instance->ldio_outstanding) >
3373	instance->ldio_threshold)) {
3374	atomic_dec(v: &instance->ldio_outstanding);
3375	return SCSI_MLQUEUE_DEVICE_BUSY;
3376	}
3377
3378	if (atomic_inc_return(v: &instance->fw_outstanding) >
3379	instance->host->can_queue) {
3380	atomic_dec(v: &instance->fw_outstanding);
3381	return SCSI_MLQUEUE_HOST_BUSY;
3382	}
3383
3384	cmd = megasas_get_cmd_fusion(instance, blk_tag: scsi_cmd_to_rq(scmd)->tag);
3385
3386	if (!cmd) {
3387	atomic_dec(v: &instance->fw_outstanding);
3388	return SCSI_MLQUEUE_HOST_BUSY;
3389	}
3390
3391	index = cmd->index;
3392
3393	req_desc = megasas_get_request_descriptor(instance, index: index-1);
3394
3395	req_desc->Words = 0;
3396	cmd->request_desc = req_desc;
3397
3398	if (megasas_build_io_fusion(instance, scp: scmd, cmd)) {
3399	megasas_return_cmd_fusion(instance, cmd);
3400	dev_err(&instance->pdev->dev, "Error building command\n");
3401	cmd->request_desc = NULL;
3402	atomic_dec(v: &instance->fw_outstanding);
3403	return SCSI_MLQUEUE_HOST_BUSY;
3404	}
3405
3406	req_desc = cmd->request_desc;
3407	req_desc->SCSIIO.SMID = cpu_to_le16(index);
3408
3409	if (cmd->io_request->ChainOffset != 0 &&
3410	cmd->io_request->ChainOffset != 0xF)
3411	dev_err(&instance->pdev->dev, "The chain offset value is not "
3412	"correct : %x\n", cmd->io_request->ChainOffset);
3413	/*
3414	* if it is raid 1/10 fp write capable.
3415	* try to get second command from pool and construct it.
3416	* From FW, it has confirmed that lba values of two PDs
3417	* corresponds to single R1/10 LD are always same
3418	*
3419	*/
3420	/* driver side count always should be less than max_fw_cmds
3421	* to get new command
3422	*/
3423	if (cmd->r1_alt_dev_handle != MR_DEVHANDLE_INVALID) {
3424	r1_cmd = megasas_get_cmd_fusion(instance,
3425	blk_tag: scsi_cmd_to_rq(scmd)->tag + instance->max_fw_cmds);
3426	megasas_prepare_secondRaid1_IO(instance, cmd, r1_cmd);
3427	}
3428
3429
3430	/*
3431	* Issue the command to the FW
3432	*/
3433
3434	megasas_sdev_busy_inc(instance, scmd);
3435	megasas_fire_cmd_fusion(instance, req_desc);
3436
3437	if (r1_cmd)
3438	megasas_fire_cmd_fusion(instance, req_desc: r1_cmd->request_desc);
3439
3440
3441	return 0;
3442	}
3443
3444	/**
3445	* megasas_complete_r1_command -
3446	* completes R1 FP write commands which has valid peer smid
3447	* @instance: Adapter soft state
3448	* @cmd: MPT command frame
3449	*
3450	*/
3451	static inline void
3452	megasas_complete_r1_command(struct megasas_instance *instance,
3453	struct megasas_cmd_fusion *cmd)
3454	{
3455	u8 *sense, status, ex_status;
3456	u32 data_length;
3457	u16 peer_smid;
3458	struct fusion_context *fusion;
3459	struct megasas_cmd_fusion *r1_cmd = NULL;
3460	struct scsi_cmnd *scmd_local = NULL;
3461	struct RAID_CONTEXT_G35 *rctx_g35;
3462
3463	rctx_g35 = &cmd->io_request->RaidContext.raid_context_g35;
3464	fusion = instance->ctrl_context;
3465	peer_smid = le16_to_cpu(rctx_g35->flow_specific.peer_smid);
3466
3467	r1_cmd = fusion->cmd_list[peer_smid - 1];
3468	scmd_local = cmd->scmd;
3469	status = rctx_g35->status;
3470	ex_status = rctx_g35->ex_status;
3471	data_length = cmd->io_request->DataLength;
3472	sense = cmd->sense;
3473
3474	cmd->cmd_completed = true;
3475
3476	/* Check if peer command is completed or not*/
3477	if (r1_cmd->cmd_completed) {
3478	rctx_g35 = &r1_cmd->io_request->RaidContext.raid_context_g35;
3479	if (rctx_g35->status != MFI_STAT_OK) {
3480	status = rctx_g35->status;
3481	ex_status = rctx_g35->ex_status;
3482	data_length = r1_cmd->io_request->DataLength;
3483	sense = r1_cmd->sense;
3484	}
3485
3486	megasas_return_cmd_fusion(instance, cmd: r1_cmd);
3487	map_cmd_status(fusion, scmd: scmd_local, status, ext_status: ex_status,
3488	le32_to_cpu(data_length), sense);
3489	if (instance->ldio_threshold &&
3490	megasas_cmd_type(cmd: scmd_local) == READ_WRITE_LDIO)
3491	atomic_dec(v: &instance->ldio_outstanding);
3492	megasas_priv(cmd: scmd_local)->cmd_priv = NULL;
3493	megasas_return_cmd_fusion(instance, cmd);
3494	scsi_dma_unmap(cmd: scmd_local);
3495	megasas_sdev_busy_dec(instance, scmd: scmd_local);
3496	scsi_done(cmd: scmd_local);
3497	}
3498	}
3499
3500	/**
3501	* access_irq_context: Access to reply processing
3502	* @irq_context: IRQ context
3503	*
3504	* Synchronize access to reply processing.
3505	*
3506	* Return: true on success, false on failure.
3507	*/
3508	static inline
3509	bool access_irq_context(struct megasas_irq_context *irq_context)
3510	{
3511	if (!irq_context)
3512	return true;
3513
3514	if (atomic_add_unless(v: &irq_context->in_used, a: 1, u: 1))
3515	return true;
3516
3517	return false;
3518	}
3519
3520	/**
3521	* release_irq_context: Release reply processing
3522	* @irq_context: IRQ context
3523	*
3524	* Release access of reply processing.
3525	*
3526	* Return: Nothing.
3527	*/
3528	static inline
3529	void release_irq_context(struct megasas_irq_context *irq_context)
3530	{
3531	if (irq_context)
3532	atomic_dec(v: &irq_context->in_used);
3533	}
3534
3535	/**
3536	* complete_cmd_fusion - Completes command
3537	* @instance: Adapter soft state
3538	* @MSIxIndex: MSI number
3539	* @irq_context: IRQ context
3540	*
3541	* Completes all commands that is in reply descriptor queue
3542	*/
3543	static int
3544	complete_cmd_fusion(struct megasas_instance *instance, u32 MSIxIndex,
3545	struct megasas_irq_context *irq_context)
3546	{
3547	union MPI2_REPLY_DESCRIPTORS_UNION *desc;
3548	struct MPI2_SCSI_IO_SUCCESS_REPLY_DESCRIPTOR *reply_desc;
3549	struct MPI2_RAID_SCSI_IO_REQUEST *scsi_io_req;
3550	struct fusion_context *fusion;
3551	struct megasas_cmd *cmd_mfi;
3552	struct megasas_cmd_fusion *cmd_fusion;
3553	u16 smid, num_completed;
3554	u8 reply_descript_type, *sense, status, extStatus;
3555	u32 device_id, data_length;
3556	union desc_value d_val;
3557	struct LD_LOAD_BALANCE_INFO *lbinfo;
3558	int threshold_reply_count = 0;
3559	struct scsi_cmnd *scmd_local = NULL;
3560	struct MR_TASK_MANAGE_REQUEST *mr_tm_req;
3561	struct MPI2_SCSI_TASK_MANAGE_REQUEST *mpi_tm_req;
3562
3563	fusion = instance->ctrl_context;
3564
3565	if (atomic_read(v: &instance->adprecovery) == MEGASAS_HW_CRITICAL_ERROR)
3566	return IRQ_HANDLED;
3567
3568	if (!access_irq_context(irq_context))
3569	return 0;
3570
3571	desc = fusion->reply_frames_desc[MSIxIndex] +
3572	fusion->last_reply_idx[MSIxIndex];
3573
3574	reply_desc = (struct MPI2_SCSI_IO_SUCCESS_REPLY_DESCRIPTOR *)desc;
3575
3576	d_val.word = desc->Words;
3577
3578	reply_descript_type = reply_desc->ReplyFlags &
3579	MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK;
3580
3581	if (reply_descript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED) {
3582	release_irq_context(irq_context);
3583	return IRQ_NONE;
3584	}
3585
3586	num_completed = 0;
3587
3588	while (d_val.u.low != cpu_to_le32(UINT_MAX) &&
3589	d_val.u.high != cpu_to_le32(UINT_MAX)) {
3590
3591	smid = le16_to_cpu(reply_desc->SMID);
3592	cmd_fusion = fusion->cmd_list[smid - 1];
3593	scsi_io_req = (struct MPI2_RAID_SCSI_IO_REQUEST *)
3594	cmd_fusion->io_request;
3595
3596	scmd_local = cmd_fusion->scmd;
3597	status = scsi_io_req->RaidContext.raid_context.status;
3598	extStatus = scsi_io_req->RaidContext.raid_context.ex_status;
3599	sense = cmd_fusion->sense;
3600	data_length = scsi_io_req->DataLength;
3601
3602	switch (scsi_io_req->Function) {
3603	case MPI2_FUNCTION_SCSI_TASK_MGMT:
3604	mr_tm_req = (struct MR_TASK_MANAGE_REQUEST *)
3605	cmd_fusion->io_request;
3606	mpi_tm_req = (struct MPI2_SCSI_TASK_MANAGE_REQUEST *)
3607	&mr_tm_req->TmRequest;
3608	dev_dbg(&instance->pdev->dev, "TM completion:"
3609	"type: 0x%x TaskMID: 0x%x\n",
3610	mpi_tm_req->TaskType, mpi_tm_req->TaskMID);
3611	complete(&cmd_fusion->done);
3612	break;
3613	case MPI2_FUNCTION_SCSI_IO_REQUEST: /*Fast Path IO.*/
3614	/* Update load balancing info */
3615	if (fusion->load_balance_info &&
3616	(megasas_priv(cmd: cmd_fusion->scmd)->status &
3617	MEGASAS_LOAD_BALANCE_FLAG)) {
3618	device_id = MEGASAS_DEV_INDEX(scmd_local);
3619	lbinfo = &fusion->load_balance_info[device_id];
3620	atomic_dec(v: &lbinfo->scsi_pending_cmds[cmd_fusion->pd_r1_lb]);
3621	megasas_priv(cmd: cmd_fusion->scmd)->status &=
3622	~MEGASAS_LOAD_BALANCE_FLAG;
3623	}
3624	fallthrough; /* and complete IO */
3625	case MEGASAS_MPI2_FUNCTION_LD_IO_REQUEST: /* LD-IO Path */
3626	atomic_dec(v: &instance->fw_outstanding);
3627	if (cmd_fusion->r1_alt_dev_handle == MR_DEVHANDLE_INVALID) {
3628	map_cmd_status(fusion, scmd: scmd_local, status,
3629	ext_status: extStatus, le32_to_cpu(data_length),
3630	sense);
3631	if (instance->ldio_threshold &&
3632	(megasas_cmd_type(cmd: scmd_local) == READ_WRITE_LDIO))
3633	atomic_dec(v: &instance->ldio_outstanding);
3634	megasas_priv(cmd: scmd_local)->cmd_priv = NULL;
3635	megasas_return_cmd_fusion(instance, cmd: cmd_fusion);
3636	scsi_dma_unmap(cmd: scmd_local);
3637	megasas_sdev_busy_dec(instance, scmd: scmd_local);
3638	scsi_done(cmd: scmd_local);
3639	} else /* Optimal VD - R1 FP command completion. */
3640	megasas_complete_r1_command(instance, cmd: cmd_fusion);
3641	break;
3642	case MEGASAS_MPI2_FUNCTION_PASSTHRU_IO_REQUEST: /*MFI command */
3643	cmd_mfi = instance->cmd_list[cmd_fusion->sync_cmd_idx];
3644	/* Poll mode. Dummy free.
3645	* In case of Interrupt mode, caller has reverse check.
3646	*/
3647	if (cmd_mfi->flags & DRV_DCMD_POLLED_MODE) {
3648	cmd_mfi->flags &= ~DRV_DCMD_POLLED_MODE;
3649	megasas_return_cmd(instance, cmd: cmd_mfi);
3650	} else
3651	megasas_complete_cmd(instance, cmd: cmd_mfi, alt_status: DID_OK);
3652	break;
3653	}
3654
3655	fusion->last_reply_idx[MSIxIndex]++;
3656	if (fusion->last_reply_idx[MSIxIndex] >=
3657	fusion->reply_q_depth)
3658	fusion->last_reply_idx[MSIxIndex] = 0;
3659
3660	desc->Words = cpu_to_le64(ULLONG_MAX);
3661	num_completed++;
3662	threshold_reply_count++;
3663
3664	/* Get the next reply descriptor */
3665	if (!fusion->last_reply_idx[MSIxIndex])
3666	desc = fusion->reply_frames_desc[MSIxIndex];
3667	else
3668	desc++;
3669
3670	reply_desc =
3671	(struct MPI2_SCSI_IO_SUCCESS_REPLY_DESCRIPTOR *)desc;
3672
3673	d_val.word = desc->Words;
3674
3675	reply_descript_type = reply_desc->ReplyFlags &
3676	MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK;
3677
3678	if (reply_descript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED)
3679	break;
3680	/*
3681	* Write to reply post host index register after completing threshold
3682	* number of reply counts and still there are more replies in reply queue
3683	* pending to be completed
3684	*/
3685	if (threshold_reply_count >= instance->threshold_reply_count) {
3686	if (instance->msix_combined)
3687	writel(val: ((MSIxIndex & 0x7) << 24) |
3688	fusion->last_reply_idx[MSIxIndex],
3689	addr: instance->reply_post_host_index_addr[MSIxIndex/8]);
3690	else
3691	writel(val: (MSIxIndex << 24) |
3692	fusion->last_reply_idx[MSIxIndex],
3693	addr: instance->reply_post_host_index_addr[0]);
3694	threshold_reply_count = 0;
3695	if (irq_context) {
3696	if (!irq_context->irq_poll_scheduled) {
3697	irq_context->irq_poll_scheduled = true;
3698	irq_context->irq_line_enable = true;
3699	irq_poll_sched(&irq_context->irqpoll);
3700	}
3701	release_irq_context(irq_context);
3702	return num_completed;
3703	}
3704	}
3705	}
3706
3707	if (num_completed) {
3708	wmb();
3709	if (instance->msix_combined)
3710	writel(val: ((MSIxIndex & 0x7) << 24) |
3711	fusion->last_reply_idx[MSIxIndex],
3712	addr: instance->reply_post_host_index_addr[MSIxIndex/8]);
3713	else
3714	writel(val: (MSIxIndex << 24) |
3715	fusion->last_reply_idx[MSIxIndex],
3716	addr: instance->reply_post_host_index_addr[0]);
3717	megasas_check_and_restore_queue_depth(instance);
3718	}
3719
3720	release_irq_context(irq_context);
3721
3722	return num_completed;
3723	}
3724
3725	int megasas_blk_mq_poll(struct Scsi_Host *shost, unsigned int queue_num)
3726	{
3727
3728	struct megasas_instance *instance;
3729	int num_entries = 0;
3730	struct fusion_context *fusion;
3731
3732	instance = (struct megasas_instance *)shost->hostdata;
3733
3734	fusion = instance->ctrl_context;
3735
3736	queue_num = queue_num + instance->low_latency_index_start;
3737
3738	if (!atomic_add_unless(v: &fusion->busy_mq_poll[queue_num], a: 1, u: 1))
3739	return 0;
3740
3741	num_entries = complete_cmd_fusion(instance, MSIxIndex: queue_num, NULL);
3742	atomic_dec(v: &fusion->busy_mq_poll[queue_num]);
3743
3744	return num_entries;
3745	}
3746
3747	/**
3748	* megasas_enable_irq_poll() - enable irqpoll
3749	* @instance: Adapter soft state
3750	*/
3751	static void megasas_enable_irq_poll(struct megasas_instance *instance)
3752	{
3753	u32 count, i;
3754	struct megasas_irq_context *irq_ctx;
3755
3756	count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
3757
3758	for (i = 0; i < count; i++) {
3759	irq_ctx = &instance->irq_context[i];
3760	irq_poll_enable(&irq_ctx->irqpoll);
3761	}
3762	}
3763
3764	/**
3765	* megasas_sync_irqs - Synchronizes all IRQs owned by adapter
3766	* @instance_addr: Adapter soft state address
3767	*/
3768	static void megasas_sync_irqs(unsigned long instance_addr)
3769	{
3770	u32 count, i;
3771	struct megasas_instance *instance =
3772	(struct megasas_instance *)instance_addr;
3773	struct megasas_irq_context *irq_ctx;
3774
3775	count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
3776
3777	for (i = 0; i < count; i++) {
3778	synchronize_irq(irq: pci_irq_vector(dev: instance->pdev, nr: i));
3779	irq_ctx = &instance->irq_context[i];
3780	irq_poll_disable(&irq_ctx->irqpoll);
3781	if (irq_ctx->irq_poll_scheduled) {
3782	irq_ctx->irq_poll_scheduled = false;
3783	enable_irq(irq: irq_ctx->os_irq);
3784	complete_cmd_fusion(instance, MSIxIndex: irq_ctx->MSIxIndex, irq_context: irq_ctx);
3785	}
3786	}
3787	}
3788
3789	/**
3790	* megasas_irqpoll() - process a queue for completed reply descriptors
3791	* @irqpoll: IRQ poll structure associated with queue to poll.
3792	* @budget: Threshold of reply descriptors to process per poll.
3793	*
3794	* Return: The number of entries processed.
3795	*/
3796
3797	int megasas_irqpoll(struct irq_poll *irqpoll, int budget)
3798	{
3799	struct megasas_irq_context *irq_ctx;
3800	struct megasas_instance *instance;
3801	int num_entries;
3802
3803	irq_ctx = container_of(irqpoll, struct megasas_irq_context, irqpoll);
3804	instance = irq_ctx->instance;
3805
3806	if (irq_ctx->irq_line_enable) {
3807	disable_irq_nosync(irq: irq_ctx->os_irq);
3808	irq_ctx->irq_line_enable = false;
3809	}
3810
3811	num_entries = complete_cmd_fusion(instance, MSIxIndex: irq_ctx->MSIxIndex, irq_context: irq_ctx);
3812	if (num_entries < budget) {
3813	irq_poll_complete(irqpoll);
3814	irq_ctx->irq_poll_scheduled = false;
3815	enable_irq(irq: irq_ctx->os_irq);
3816	complete_cmd_fusion(instance, MSIxIndex: irq_ctx->MSIxIndex, irq_context: irq_ctx);
3817	}
3818
3819	return num_entries;
3820	}
3821
3822	/**
3823	* megasas_complete_cmd_dpc_fusion - Completes command
3824	* @instance_addr: Adapter soft state address
3825	*
3826	* Tasklet to complete cmds
3827	*/
3828	static void
3829	megasas_complete_cmd_dpc_fusion(unsigned long instance_addr)
3830	{
3831	struct megasas_instance *instance =
3832	(struct megasas_instance *)instance_addr;
3833	struct megasas_irq_context *irq_ctx = NULL;
3834	u32 count, MSIxIndex;
3835
3836	count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
3837
3838	/* If we have already declared adapter dead, donot complete cmds */
3839	if (atomic_read(v: &instance->adprecovery) == MEGASAS_HW_CRITICAL_ERROR)
3840	return;
3841
3842	for (MSIxIndex = 0 ; MSIxIndex < count; MSIxIndex++) {
3843	irq_ctx = &instance->irq_context[MSIxIndex];
3844	complete_cmd_fusion(instance, MSIxIndex, irq_context: irq_ctx);
3845	}
3846	}
3847
3848	/**
3849	* megasas_isr_fusion - isr entry point
3850	* @irq: IRQ number
3851	* @devp: IRQ context
3852	*/
3853	static irqreturn_t megasas_isr_fusion(int irq, void *devp)
3854	{
3855	struct megasas_irq_context *irq_context = devp;
3856	struct megasas_instance *instance = irq_context->instance;
3857	u32 mfiStatus;
3858
3859	if (instance->mask_interrupts)
3860	return IRQ_NONE;
3861
3862	if (irq_context->irq_poll_scheduled)
3863	return IRQ_HANDLED;
3864
3865	if (!instance->msix_vectors) {
3866	mfiStatus = instance->instancet->clear_intr(instance);
3867	if (!mfiStatus)
3868	return IRQ_NONE;
3869	}
3870
3871	/* If we are resetting, bail */
3872	if (test_bit(MEGASAS_FUSION_IN_RESET, &instance->reset_flags)) {
3873	instance->instancet->clear_intr(instance);
3874	return IRQ_HANDLED;
3875	}
3876
3877	return complete_cmd_fusion(instance, MSIxIndex: irq_context->MSIxIndex, irq_context)
3878	? IRQ_HANDLED : IRQ_NONE;
3879	}
3880
3881	/**
3882	* build_mpt_mfi_pass_thru - builds a cmd fo MFI Pass thru
3883	* @instance: Adapter soft state
3884	* @mfi_cmd: megasas_cmd pointer
3885	*
3886	*/
3887	static void
3888	build_mpt_mfi_pass_thru(struct megasas_instance *instance,
3889	struct megasas_cmd *mfi_cmd)
3890	{
3891	struct MPI25_IEEE_SGE_CHAIN64 *mpi25_ieee_chain;
3892	struct MPI2_RAID_SCSI_IO_REQUEST *io_req;
3893	struct megasas_cmd_fusion *cmd;
3894	struct fusion_context *fusion;
3895	struct megasas_header *frame_hdr = &mfi_cmd->frame->hdr;
3896
3897	fusion = instance->ctrl_context;
3898
3899	cmd = megasas_get_cmd_fusion(instance,
3900	blk_tag: instance->max_scsi_cmds + mfi_cmd->index);
3901
3902	/* Save the smid. To be used for returning the cmd */
3903	mfi_cmd->context.smid = cmd->index;
3904
3905	/*
3906	* For cmds where the flag is set, store the flag and check
3907	* on completion. For cmds with this flag, don't call
3908	* megasas_complete_cmd
3909	*/
3910
3911	if (frame_hdr->flags & cpu_to_le16(MFI_FRAME_DONT_POST_IN_REPLY_QUEUE))
3912	mfi_cmd->flags |= DRV_DCMD_POLLED_MODE;
3913
3914	io_req = cmd->io_request;
3915
3916	if (instance->adapter_type >= INVADER_SERIES) {
3917	struct MPI25_IEEE_SGE_CHAIN64 *sgl_ptr_end =
3918	(struct MPI25_IEEE_SGE_CHAIN64 *)&io_req->SGL;
3919	sgl_ptr_end += fusion->max_sge_in_main_msg - 1;
3920	sgl_ptr_end->Flags = 0;
3921	}
3922
3923	mpi25_ieee_chain =
3924	(struct MPI25_IEEE_SGE_CHAIN64 *)&io_req->SGL.IeeeChain;
3925
3926	io_req->Function = MEGASAS_MPI2_FUNCTION_PASSTHRU_IO_REQUEST;
3927	io_req->SGLOffset0 = offsetof(struct MPI2_RAID_SCSI_IO_REQUEST,
3928	SGL) / 4;
3929	io_req->ChainOffset = fusion->chain_offset_mfi_pthru;
3930
3931	mpi25_ieee_chain->Address = cpu_to_le64(mfi_cmd->frame_phys_addr);
3932
3933	mpi25_ieee_chain->Flags = IEEE_SGE_FLAGS_CHAIN_ELEMENT |
3934	MPI2_IEEE_SGE_FLAGS_IOCPLBNTA_ADDR;
3935
3936	mpi25_ieee_chain->Length = cpu_to_le32(instance->mfi_frame_size);
3937	}
3938
3939	/**
3940	* build_mpt_cmd - Calls helper function to build a cmd MFI Pass thru cmd
3941	* @instance: Adapter soft state
3942	* @cmd: mfi cmd to build
3943	*
3944	*/
3945	static union MEGASAS_REQUEST_DESCRIPTOR_UNION *
3946	build_mpt_cmd(struct megasas_instance *instance, struct megasas_cmd *cmd)
3947	{
3948	union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc = NULL;
3949	u16 index;
3950
3951	build_mpt_mfi_pass_thru(instance, mfi_cmd: cmd);
3952	index = cmd->context.smid;
3953
3954	req_desc = megasas_get_request_descriptor(instance, index: index - 1);
3955
3956	req_desc->Words = 0;
3957	req_desc->SCSIIO.RequestFlags = (MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO <<
3958	MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
3959
3960	req_desc->SCSIIO.SMID = cpu_to_le16(index);
3961
3962	return req_desc;
3963	}
3964
3965	/**
3966	* megasas_issue_dcmd_fusion - Issues a MFI Pass thru cmd
3967	* @instance: Adapter soft state
3968	* @cmd: mfi cmd pointer
3969	*
3970	*/
3971	static void
3972	megasas_issue_dcmd_fusion(struct megasas_instance *instance,
3973	struct megasas_cmd *cmd)
3974	{
3975	union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc;
3976
3977	req_desc = build_mpt_cmd(instance, cmd);
3978
3979	megasas_fire_cmd_fusion(instance, req_desc);
3980	return;
3981	}
3982
3983	/**
3984	* megasas_release_fusion - Reverses the FW initialization
3985	* @instance: Adapter soft state
3986	*/
3987	void
3988	megasas_release_fusion(struct megasas_instance *instance)
3989	{
3990	megasas_free_ioc_init_cmd(instance);
3991	megasas_free_cmds(instance);
3992	megasas_free_cmds_fusion(instance);
3993
3994	iounmap(addr: instance->reg_set);
3995
3996	pci_release_selected_regions(instance->pdev, 1<<instance->bar);
3997	}
3998
3999	/**
4000	* megasas_read_fw_status_reg_fusion - returns the current FW status value
4001	* @instance: Adapter soft state
4002	*/
4003	static u32
4004	megasas_read_fw_status_reg_fusion(struct megasas_instance *instance)
4005	{
4006	return megasas_readl(instance, addr: &instance->reg_set->outbound_scratch_pad_0);
4007	}
4008
4009	/**
4010	* megasas_alloc_host_crash_buffer - Host buffers for Crash dump collection from Firmware
4011	* @instance: Controller's soft instance
4012	* @return: Number of allocated host crash buffers
4013	*/
4014	static void
4015	megasas_alloc_host_crash_buffer(struct megasas_instance *instance)
4016	{
4017	unsigned int i;
4018
4019	for (i = 0; i < MAX_CRASH_DUMP_SIZE; i++) {
4020	instance->crash_buf[i] = vzalloc(CRASH_DMA_BUF_SIZE);
4021	if (!instance->crash_buf[i]) {
4022	dev_info(&instance->pdev->dev, "Firmware crash dump "
4023	"memory allocation failed at index %d\n", i);
4024	break;
4025	}
4026	}
4027	instance->drv_buf_alloc = i;
4028	}
4029
4030	/**
4031	* megasas_free_host_crash_buffer - Host buffers for Crash dump collection from Firmware
4032	* @instance: Controller's soft instance
4033	*/
4034	void
4035	megasas_free_host_crash_buffer(struct megasas_instance *instance)
4036	{
4037	unsigned int i;
4038	for (i = 0; i < instance->drv_buf_alloc; i++) {
4039	vfree(addr: instance->crash_buf[i]);
4040	}
4041	instance->drv_buf_index = 0;
4042	instance->drv_buf_alloc = 0;
4043	instance->fw_crash_state = UNAVAILABLE;
4044	instance->fw_crash_buffer_size = 0;
4045	}
4046
4047	/**
4048	* megasas_adp_reset_fusion - For controller reset
4049	* @instance: Controller's soft instance
4050	* @regs: MFI register set
4051	*/
4052	static int
4053	megasas_adp_reset_fusion(struct megasas_instance *instance,
4054	struct megasas_register_set __iomem *regs)
4055	{
4056	u32 host_diag, abs_state, retry;
4057
4058	/* Now try to reset the chip */
4059	writel(MPI2_WRSEQ_FLUSH_KEY_VALUE, addr: &instance->reg_set->fusion_seq_offset);
4060	writel(MPI2_WRSEQ_1ST_KEY_VALUE, addr: &instance->reg_set->fusion_seq_offset);
4061	writel(MPI2_WRSEQ_2ND_KEY_VALUE, addr: &instance->reg_set->fusion_seq_offset);
4062	writel(MPI2_WRSEQ_3RD_KEY_VALUE, addr: &instance->reg_set->fusion_seq_offset);
4063	writel(MPI2_WRSEQ_4TH_KEY_VALUE, addr: &instance->reg_set->fusion_seq_offset);
4064	writel(MPI2_WRSEQ_5TH_KEY_VALUE, addr: &instance->reg_set->fusion_seq_offset);
4065	writel(MPI2_WRSEQ_6TH_KEY_VALUE, addr: &instance->reg_set->fusion_seq_offset);
4066
4067	/* Check that the diag write enable (DRWE) bit is on */
4068	host_diag = megasas_readl(instance, addr: &instance->reg_set->fusion_host_diag);
4069	retry = 0;
4070	while (!(host_diag & HOST_DIAG_WRITE_ENABLE)) {
4071	msleep(msecs: 100);
4072	host_diag = megasas_readl(instance,
4073	addr: &instance->reg_set->fusion_host_diag);
4074	if (retry++ == 100) {
4075	dev_warn(&instance->pdev->dev,
4076	"Host diag unlock failed from %s %d\n",
4077	__func__, __LINE__);
4078	break;
4079	}
4080	}
4081	if (!(host_diag & HOST_DIAG_WRITE_ENABLE))
4082	return -1;
4083
4084	/* Send chip reset command */
4085	writel(val: host_diag | HOST_DIAG_RESET_ADAPTER,
4086	addr: &instance->reg_set->fusion_host_diag);
4087	msleep(msecs: 3000);
4088
4089	/* Make sure reset adapter bit is cleared */
4090	host_diag = megasas_readl(instance, addr: &instance->reg_set->fusion_host_diag);
4091	retry = 0;
4092	while (host_diag & HOST_DIAG_RESET_ADAPTER) {
4093	msleep(msecs: 100);
4094	host_diag = megasas_readl(instance,
4095	addr: &instance->reg_set->fusion_host_diag);
4096	if (retry++ == 1000) {
4097	dev_warn(&instance->pdev->dev,
4098	"Diag reset adapter never cleared %s %d\n",
4099	__func__, __LINE__);
4100	break;
4101	}
4102	}
4103	if (host_diag & HOST_DIAG_RESET_ADAPTER)
4104	return -1;
4105
4106	abs_state = instance->instancet->read_fw_status_reg(instance)
4107	& MFI_STATE_MASK;
4108	retry = 0;
4109
4110	while ((abs_state <= MFI_STATE_FW_INIT) && (retry++ < 1000)) {
4111	msleep(msecs: 100);
4112	abs_state = instance->instancet->
4113	read_fw_status_reg(instance) & MFI_STATE_MASK;
4114	}
4115	if (abs_state <= MFI_STATE_FW_INIT) {
4116	dev_warn(&instance->pdev->dev,
4117	"fw state < MFI_STATE_FW_INIT, state = 0x%x %s %d\n",
4118	abs_state, __func__, __LINE__);
4119	return -1;
4120	}
4121
4122	return 0;
4123	}
4124
4125	/**
4126	* megasas_check_reset_fusion - For controller reset check
4127	* @instance: Controller's soft instance
4128	* @regs: MFI register set
4129	*/
4130	static int
4131	megasas_check_reset_fusion(struct megasas_instance *instance,
4132	struct megasas_register_set __iomem *regs)
4133	{
4134	return 0;
4135	}
4136
4137	/**
4138	* megasas_trigger_snap_dump - Trigger snap dump in FW
4139	* @instance: Soft instance of adapter
4140	*/
4141	static inline void megasas_trigger_snap_dump(struct megasas_instance *instance)
4142	{
4143	int j;
4144	u32 fw_state, abs_state;
4145
4146	if (!instance->disableOnlineCtrlReset) {
4147	dev_info(&instance->pdev->dev, "Trigger snap dump\n");
4148	writel(MFI_ADP_TRIGGER_SNAP_DUMP,
4149	addr: &instance->reg_set->doorbell);
4150	readl(addr: &instance->reg_set->doorbell);
4151	}
4152
4153	for (j = 0; j < instance->snapdump_wait_time; j++) {
4154	abs_state = instance->instancet->read_fw_status_reg(instance);
4155	fw_state = abs_state & MFI_STATE_MASK;
4156	if (fw_state == MFI_STATE_FAULT) {
4157	dev_printk(KERN_ERR, &instance->pdev->dev,
4158	"FW in FAULT state Fault code:0x%x subcode:0x%x func:%s\n",
4159	abs_state & MFI_STATE_FAULT_CODE,
4160	abs_state & MFI_STATE_FAULT_SUBCODE, __func__);
4161	return;
4162	}
4163	msleep(msecs: 1000);
4164	}
4165	}
4166
4167	/* This function waits for outstanding commands on fusion to complete */
4168	static int
4169	megasas_wait_for_outstanding_fusion(struct megasas_instance *instance,
4170	int reason, int *convert)
4171	{
4172	int i, outstanding, retval = 0, hb_seconds_missed = 0;
4173	u32 fw_state, abs_state;
4174	u32 waittime_for_io_completion;
4175
4176	waittime_for_io_completion =
4177	min_t(u32, resetwaittime,
4178	(resetwaittime - instance->snapdump_wait_time));
4179
4180	if (reason == MFI_IO_TIMEOUT_OCR) {
4181	dev_info(&instance->pdev->dev,
4182	"MFI command is timed out\n");
4183	megasas_complete_cmd_dpc_fusion(instance_addr: (unsigned long)instance);
4184	if (instance->snapdump_wait_time)
4185	megasas_trigger_snap_dump(instance);
4186	retval = 1;
4187	goto out;
4188	}
4189
4190	for (i = 0; i < waittime_for_io_completion; i++) {
4191	/* Check if firmware is in fault state */
4192	abs_state = instance->instancet->read_fw_status_reg(instance);
4193	fw_state = abs_state & MFI_STATE_MASK;
4194	if (fw_state == MFI_STATE_FAULT) {
4195	dev_printk(KERN_ERR, &instance->pdev->dev,
4196	"FW in FAULT state Fault code:0x%x subcode:0x%x func:%s\n",
4197	abs_state & MFI_STATE_FAULT_CODE,
4198	abs_state & MFI_STATE_FAULT_SUBCODE, __func__);
4199	megasas_complete_cmd_dpc_fusion(instance_addr: (unsigned long)instance);
4200	if (instance->requestorId && reason) {
4201	dev_warn(&instance->pdev->dev, "SR-IOV Found FW in FAULT"
4202	" state while polling during"
4203	" I/O timeout handling for %d\n",
4204	instance->host->host_no);
4205	*convert = 1;
4206	}
4207
4208	retval = 1;
4209	goto out;
4210	}
4211
4212
4213	/* If SR-IOV VF mode & heartbeat timeout, don't wait */
4214	if (instance->requestorId && !reason) {
4215	retval = 1;
4216	goto out;
4217	}
4218
4219	/* If SR-IOV VF mode & I/O timeout, check for HB timeout */
4220	if (instance->requestorId && (reason == SCSIIO_TIMEOUT_OCR)) {
4221	if (instance->hb_host_mem->HB.fwCounter !=
4222	instance->hb_host_mem->HB.driverCounter) {
4223	instance->hb_host_mem->HB.driverCounter =
4224	instance->hb_host_mem->HB.fwCounter;
4225	hb_seconds_missed = 0;
4226	} else {
4227	hb_seconds_missed++;
4228	if (hb_seconds_missed ==
4229	(MEGASAS_SRIOV_HEARTBEAT_INTERVAL_VF/HZ)) {
4230	dev_warn(&instance->pdev->dev, "SR-IOV:"
4231	" Heartbeat never completed "
4232	" while polling during I/O "
4233	" timeout handling for "
4234	"scsi%d.\n",
4235	instance->host->host_no);
4236	*convert = 1;
4237	retval = 1;
4238	goto out;
4239	}
4240	}
4241	}
4242
4243	megasas_complete_cmd_dpc_fusion(instance_addr: (unsigned long)instance);
4244	outstanding = atomic_read(v: &instance->fw_outstanding);
4245	if (!outstanding)
4246	goto out;
4247
4248	if (!(i % MEGASAS_RESET_NOTICE_INTERVAL)) {
4249	dev_notice(&instance->pdev->dev, "[%2d]waiting for %d "
4250	"commands to complete for scsi%d\n", i,
4251	outstanding, instance->host->host_no);
4252	}
4253	msleep(msecs: 1000);
4254	}
4255
4256	if (instance->snapdump_wait_time) {
4257	megasas_trigger_snap_dump(instance);
4258	retval = 1;
4259	goto out;
4260	}
4261
4262	if (atomic_read(v: &instance->fw_outstanding)) {
4263	dev_err(&instance->pdev->dev, "pending commands remain after waiting, "
4264	"will reset adapter scsi%d.\n",
4265	instance->host->host_no);
4266	*convert = 1;
4267	retval = 1;
4268	}
4269
4270	out:
4271	if (!retval && reason == SCSIIO_TIMEOUT_OCR)
4272	dev_info(&instance->pdev->dev, "IO is completed, no OCR is required\n");
4273
4274	return retval;
4275	}
4276
4277	void megasas_reset_reply_desc(struct megasas_instance *instance)
4278	{
4279	int i, j, count;
4280	struct fusion_context *fusion;
4281	union MPI2_REPLY_DESCRIPTORS_UNION *reply_desc;
4282
4283	fusion = instance->ctrl_context;
4284	count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
4285	count += instance->iopoll_q_count;
4286
4287	for (i = 0 ; i < count ; i++) {
4288	fusion->last_reply_idx[i] = 0;
4289	reply_desc = fusion->reply_frames_desc[i];
4290	for (j = 0 ; j < fusion->reply_q_depth; j++, reply_desc++)
4291	reply_desc->Words = cpu_to_le64(ULLONG_MAX);
4292	}
4293	}
4294
4295	/*
4296	* megasas_refire_mgmt_cmd : Re-fire management commands
4297	* @instance: Controller's soft instance
4298	*/
4299	static void megasas_refire_mgmt_cmd(struct megasas_instance *instance,
4300	bool return_ioctl)
4301	{
4302	int j;
4303	struct megasas_cmd_fusion *cmd_fusion;
4304	struct fusion_context *fusion;
4305	struct megasas_cmd *cmd_mfi;
4306	union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc;
4307	struct MPI2_RAID_SCSI_IO_REQUEST *scsi_io_req;
4308	u16 smid;
4309	bool refire_cmd = false;
4310	u8 result;
4311	u32 opcode = 0;
4312
4313	fusion = instance->ctrl_context;
4314
4315	/* Re-fire management commands.
4316	* Do not traverse complet MPT frame pool. Start from max_scsi_cmds.
4317	*/
4318	for (j = instance->max_scsi_cmds ; j < instance->max_fw_cmds; j++) {
4319	cmd_fusion = fusion->cmd_list[j];
4320	cmd_mfi = instance->cmd_list[cmd_fusion->sync_cmd_idx];
4321	smid = le16_to_cpu(cmd_mfi->context.smid);
4322	result = REFIRE_CMD;
4323
4324	if (!smid)
4325	continue;
4326
4327	req_desc = megasas_get_request_descriptor(instance, index: smid - 1);
4328
4329	switch (cmd_mfi->frame->hdr.cmd) {
4330	case MFI_CMD_DCMD:
4331	opcode = le32_to_cpu(cmd_mfi->frame->dcmd.opcode);
4332	/* Do not refire shutdown command */
4333	if (opcode == MR_DCMD_CTRL_SHUTDOWN) {
4334	cmd_mfi->frame->dcmd.cmd_status = MFI_STAT_OK;
4335	result = COMPLETE_CMD;
4336	break;
4337	}
4338
4339	refire_cmd = ((opcode != MR_DCMD_LD_MAP_GET_INFO)) &&
4340	(opcode != MR_DCMD_SYSTEM_PD_MAP_GET_INFO) &&
4341	!(cmd_mfi->flags & DRV_DCMD_SKIP_REFIRE);
4342
4343	if (!refire_cmd)
4344	result = RETURN_CMD;
4345
4346	break;
4347	case MFI_CMD_NVME:
4348	if (!instance->support_nvme_passthru) {
4349	cmd_mfi->frame->hdr.cmd_status = MFI_STAT_INVALID_CMD;
4350	result = COMPLETE_CMD;
4351	}
4352
4353	break;
4354	case MFI_CMD_TOOLBOX:
4355	if (!instance->support_pci_lane_margining) {
4356	cmd_mfi->frame->hdr.cmd_status = MFI_STAT_INVALID_CMD;
4357	result = COMPLETE_CMD;
4358	}
4359
4360	break;
4361	default:
4362	break;
4363	}
4364
4365	if (return_ioctl && cmd_mfi->sync_cmd &&
4366	cmd_mfi->frame->hdr.cmd != MFI_CMD_ABORT) {
4367	dev_err(&instance->pdev->dev,
4368	"return -EBUSY from %s %d cmd 0x%x opcode 0x%x\n",
4369	__func__, __LINE__, cmd_mfi->frame->hdr.cmd,
4370	le32_to_cpu(cmd_mfi->frame->dcmd.opcode));
4371	cmd_mfi->cmd_status_drv = DCMD_BUSY;
4372	result = COMPLETE_CMD;
4373	}
4374
4375	scsi_io_req = (struct MPI2_RAID_SCSI_IO_REQUEST *)
4376	cmd_fusion->io_request;
4377	if (scsi_io_req->Function == MPI2_FUNCTION_SCSI_TASK_MGMT)
4378	result = RETURN_CMD;
4379
4380	switch (result) {
4381	case REFIRE_CMD:
4382	megasas_fire_cmd_fusion(instance, req_desc);
4383	break;
4384	case RETURN_CMD:
4385	megasas_return_cmd(instance, cmd: cmd_mfi);
4386	break;
4387	case COMPLETE_CMD:
4388	megasas_complete_cmd(instance, cmd: cmd_mfi, alt_status: DID_OK);
4389	break;
4390	}
4391	}
4392	}
4393
4394	/*
4395	* megasas_return_polled_cmds: Return polled mode commands back to the pool
4396	* before initiating an OCR.
4397	* @instance: Controller's soft instance
4398	*/
4399	static void
4400	megasas_return_polled_cmds(struct megasas_instance *instance)
4401	{
4402	int i;
4403	struct megasas_cmd_fusion *cmd_fusion;
4404	struct fusion_context *fusion;
4405	struct megasas_cmd *cmd_mfi;
4406
4407	fusion = instance->ctrl_context;
4408
4409	for (i = instance->max_scsi_cmds; i < instance->max_fw_cmds; i++) {
4410	cmd_fusion = fusion->cmd_list[i];
4411	cmd_mfi = instance->cmd_list[cmd_fusion->sync_cmd_idx];
4412
4413	if (cmd_mfi->flags & DRV_DCMD_POLLED_MODE) {
4414	if (megasas_dbg_lvl & OCR_DEBUG)
4415	dev_info(&instance->pdev->dev,
4416	"%s %d return cmd 0x%x opcode 0x%x\n",
4417	__func__, __LINE__, cmd_mfi->frame->hdr.cmd,
4418	le32_to_cpu(cmd_mfi->frame->dcmd.opcode));
4419	cmd_mfi->flags &= ~DRV_DCMD_POLLED_MODE;
4420	megasas_return_cmd(instance, cmd: cmd_mfi);
4421	}
4422	}
4423	}
4424
4425	/*
4426	* megasas_track_scsiio : Track SCSI IOs outstanding to a SCSI device
4427	* @instance: per adapter struct
4428	* @channel: the channel assigned by the OS
4429	* @id: the id assigned by the OS
4430	*
4431	* Returns SUCCESS if no IOs pending to SCSI device, else return FAILED
4432	*/
4433
4434	static int megasas_track_scsiio(struct megasas_instance *instance,
4435	int id, int channel)
4436	{
4437	int i, found = 0;
4438	struct megasas_cmd_fusion *cmd_fusion;
4439	struct fusion_context *fusion;
4440	fusion = instance->ctrl_context;
4441
4442	for (i = 0 ; i < instance->max_scsi_cmds; i++) {
4443	cmd_fusion = fusion->cmd_list[i];
4444	if (cmd_fusion->scmd &&
4445	(cmd_fusion->scmd->device->id == id &&
4446	cmd_fusion->scmd->device->channel == channel)) {
4447	dev_info(&instance->pdev->dev,
4448	"SCSI commands pending to target"
4449	"channel %d id %d \tSMID: 0x%x\n",
4450	channel, id, cmd_fusion->index);
4451	scsi_print_command(cmd_fusion->scmd);
4452	found = 1;
4453	break;
4454	}
4455	}
4456
4457	return found ? FAILED : SUCCESS;
4458	}
4459
4460	/**
4461	* megasas_tm_response_code - translation of device response code
4462	* @instance: Controller's soft instance
4463	* @mpi_reply: MPI reply returned by firmware
4464	*
4465	* Return nothing.
4466	*/
4467	static void
4468	megasas_tm_response_code(struct megasas_instance *instance,
4469	struct MPI2_SCSI_TASK_MANAGE_REPLY *mpi_reply)
4470	{
4471	char *desc;
4472
4473	switch (mpi_reply->ResponseCode) {
4474	case MPI2_SCSITASKMGMT_RSP_TM_COMPLETE:
4475	desc = "task management request completed";
4476	break;
4477	case MPI2_SCSITASKMGMT_RSP_INVALID_FRAME:
4478	desc = "invalid frame";
4479	break;
4480	case MPI2_SCSITASKMGMT_RSP_TM_NOT_SUPPORTED:
4481	desc = "task management request not supported";
4482	break;
4483	case MPI2_SCSITASKMGMT_RSP_TM_FAILED:
4484	desc = "task management request failed";
4485	break;
4486	case MPI2_SCSITASKMGMT_RSP_TM_SUCCEEDED:
4487	desc = "task management request succeeded";
4488	break;
4489	case MPI2_SCSITASKMGMT_RSP_TM_INVALID_LUN:
4490	desc = "invalid lun";
4491	break;
4492	case 0xA:
4493	desc = "overlapped tag attempted";
4494	break;
4495	case MPI2_SCSITASKMGMT_RSP_IO_QUEUED_ON_IOC:
4496	desc = "task queued, however not sent to target";
4497	break;
4498	default:
4499	desc = "unknown";
4500	break;
4501	}
4502	dev_dbg(&instance->pdev->dev, "response_code(%01x): %s\n",
4503	mpi_reply->ResponseCode, desc);
4504	dev_dbg(&instance->pdev->dev,
4505	"TerminationCount/DevHandle/Function/TaskType/IOCStat/IOCLoginfo"
4506	" 0x%x/0x%x/0x%x/0x%x/0x%x/0x%x\n",
4507	mpi_reply->TerminationCount, mpi_reply->DevHandle,
4508	mpi_reply->Function, mpi_reply->TaskType,
4509	mpi_reply->IOCStatus, mpi_reply->IOCLogInfo);
4510	}
4511
4512	/**
4513	* megasas_issue_tm - main routine for sending tm requests
4514	* @instance: per adapter struct
4515	* @device_handle: device handle
4516	* @channel: the channel assigned by the OS
4517	* @id: the id assigned by the OS
4518	* @smid_task: smid assigned to the task
4519	* @type: MPI2_SCSITASKMGMT_TASKTYPE__XXX (defined in megaraid_sas_fusion.c)
4520	* @mr_device_priv_data: private data
4521	* Context: user
4522	*
4523	* MegaRaid use MPT interface for Task Magement request.
4524	* A generic API for sending task management requests to firmware.
4525	*
4526	* Return SUCCESS or FAILED.
4527	*/
4528	static int
4529	megasas_issue_tm(struct megasas_instance *instance, u16 device_handle,
4530	uint channel, uint id, u16 smid_task, u8 type,
4531	struct MR_PRIV_DEVICE *mr_device_priv_data)
4532	{
4533	struct MR_TASK_MANAGE_REQUEST *mr_request;
4534	struct MPI2_SCSI_TASK_MANAGE_REQUEST *mpi_request;
4535	unsigned long timeleft;
4536	struct megasas_cmd_fusion *cmd_fusion;
4537	struct megasas_cmd *cmd_mfi;
4538	union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc;
4539	struct fusion_context *fusion = NULL;
4540	struct megasas_cmd_fusion *scsi_lookup;
4541	int rc;
4542	int timeout = MEGASAS_DEFAULT_TM_TIMEOUT;
4543	struct MPI2_SCSI_TASK_MANAGE_REPLY *mpi_reply;
4544
4545	fusion = instance->ctrl_context;
4546
4547	cmd_mfi = megasas_get_cmd(instance);
4548
4549	if (!cmd_mfi) {
4550	dev_err(&instance->pdev->dev, "Failed from %s %d\n",
4551	__func__, __LINE__);
4552	return -ENOMEM;
4553	}
4554
4555	cmd_fusion = megasas_get_cmd_fusion(instance,
4556	blk_tag: instance->max_scsi_cmds + cmd_mfi->index);
4557
4558	/* Save the smid. To be used for returning the cmd */
4559	cmd_mfi->context.smid = cmd_fusion->index;
4560
4561	req_desc = megasas_get_request_descriptor(instance,
4562	index: (cmd_fusion->index - 1));
4563
4564	cmd_fusion->request_desc = req_desc;
4565	req_desc->Words = 0;
4566
4567	mr_request = (struct MR_TASK_MANAGE_REQUEST *) cmd_fusion->io_request;
4568	memset(mr_request, 0, sizeof(struct MR_TASK_MANAGE_REQUEST));
4569	mpi_request = (struct MPI2_SCSI_TASK_MANAGE_REQUEST *) &mr_request->TmRequest;
4570	mpi_request->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
4571	mpi_request->DevHandle = cpu_to_le16(device_handle);
4572	mpi_request->TaskType = type;
4573	mpi_request->TaskMID = cpu_to_le16(smid_task);
4574	mpi_request->LUN[1] = 0;
4575
4576
4577	req_desc = cmd_fusion->request_desc;
4578	req_desc->HighPriority.SMID = cpu_to_le16(cmd_fusion->index);
4579	req_desc->HighPriority.RequestFlags =
4580	(MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY <<
4581	MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
4582	req_desc->HighPriority.MSIxIndex = 0;
4583	req_desc->HighPriority.LMID = 0;
4584	req_desc->HighPriority.Reserved1 = 0;
4585
4586	if (channel < MEGASAS_MAX_PD_CHANNELS)
4587	mr_request->tmReqFlags.isTMForPD = 1;
4588	else
4589	mr_request->tmReqFlags.isTMForLD = 1;
4590
4591	init_completion(x: &cmd_fusion->done);
4592	megasas_fire_cmd_fusion(instance, req_desc);
4593
4594	switch (type) {
4595	case MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK:
4596	timeout = mr_device_priv_data->task_abort_tmo;
4597	break;
4598	case MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET:
4599	timeout = mr_device_priv_data->target_reset_tmo;
4600	break;
4601	}
4602
4603	timeleft = wait_for_completion_timeout(x: &cmd_fusion->done, timeout: timeout * HZ);
4604
4605	if (!timeleft) {
4606	dev_err(&instance->pdev->dev,
4607	"task mgmt type 0x%x timed out\n", type);
4608	mutex_unlock(lock: &instance->reset_mutex);
4609	rc = megasas_reset_fusion(shost: instance->host, reason: MFI_IO_TIMEOUT_OCR);
4610	mutex_lock(&instance->reset_mutex);
4611	return rc;
4612	}
4613
4614	mpi_reply = (struct MPI2_SCSI_TASK_MANAGE_REPLY *) &mr_request->TMReply;
4615	megasas_tm_response_code(instance, mpi_reply);
4616
4617	megasas_return_cmd(instance, cmd: cmd_mfi);
4618	rc = SUCCESS;
4619	switch (type) {
4620	case MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK:
4621	scsi_lookup = fusion->cmd_list[smid_task - 1];
4622
4623	if (scsi_lookup->scmd == NULL)
4624	break;
4625	else {
4626	instance->instancet->disable_intr(instance);
4627	megasas_sync_irqs(instance_addr: (unsigned long)instance);
4628	instance->instancet->enable_intr(instance);
4629	megasas_enable_irq_poll(instance);
4630	if (scsi_lookup->scmd == NULL)
4631	break;
4632	}
4633	rc = FAILED;
4634	break;
4635
4636	case MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET:
4637	if ((channel == 0xFFFFFFFF) && (id == 0xFFFFFFFF))
4638	break;
4639	instance->instancet->disable_intr(instance);
4640	megasas_sync_irqs(instance_addr: (unsigned long)instance);
4641	rc = megasas_track_scsiio(instance, id, channel);
4642	instance->instancet->enable_intr(instance);
4643	megasas_enable_irq_poll(instance);
4644
4645	break;
4646	case MPI2_SCSITASKMGMT_TASKTYPE_ABRT_TASK_SET:
4647	case MPI2_SCSITASKMGMT_TASKTYPE_QUERY_TASK:
4648	break;
4649	default:
4650	rc = FAILED;
4651	break;
4652	}
4653
4654	return rc;
4655
4656	}
4657
4658	/*
4659	* megasas_fusion_smid_lookup : Look for fusion command corresponding to SCSI
4660	* @instance: per adapter struct
4661	*
4662	* Return Non Zero index, if SMID found in outstanding commands
4663	*/
4664	static u16 megasas_fusion_smid_lookup(struct scsi_cmnd *scmd)
4665	{
4666	int i, ret = 0;
4667	struct megasas_instance *instance;
4668	struct megasas_cmd_fusion *cmd_fusion;
4669	struct fusion_context *fusion;
4670
4671	instance = (struct megasas_instance *)scmd->device->host->hostdata;
4672
4673	fusion = instance->ctrl_context;
4674
4675	for (i = 0; i < instance->max_scsi_cmds; i++) {
4676	cmd_fusion = fusion->cmd_list[i];
4677	if (cmd_fusion->scmd && (cmd_fusion->scmd == scmd)) {
4678	scmd_printk(KERN_NOTICE, scmd, "Abort request is for"
4679	" SMID: %d\n", cmd_fusion->index);
4680	ret = cmd_fusion->index;
4681	break;
4682	}
4683	}
4684
4685	return ret;
4686	}
4687
4688	/*
4689	* megasas_get_tm_devhandle - Get devhandle for TM request
4690	* @sdev- OS provided scsi device
4691	*
4692	* Returns- devhandle/targetID of SCSI device
4693	*/
4694	static u16 megasas_get_tm_devhandle(struct scsi_device *sdev)
4695	{
4696	u16 pd_index = 0;
4697	u32 device_id;
4698	struct megasas_instance *instance;
4699	struct fusion_context *fusion;
4700	struct MR_PD_CFG_SEQ_NUM_SYNC *pd_sync;
4701	u16 devhandle = (u16)ULONG_MAX;
4702
4703	instance = (struct megasas_instance *)sdev->host->hostdata;
4704	fusion = instance->ctrl_context;
4705
4706	if (!MEGASAS_IS_LOGICAL(sdev)) {
4707	if (instance->use_seqnum_jbod_fp) {
4708	pd_index = (sdev->channel * MEGASAS_MAX_DEV_PER_CHANNEL)
4709	+ sdev->id;
4710	pd_sync = (void *)fusion->pd_seq_sync
4711	[(instance->pd_seq_map_id - 1) & 1];
4712	devhandle = pd_sync->seq[pd_index].devHandle;
4713	} else
4714	sdev_printk(KERN_ERR, sdev, "Firmware expose tmCapable"
4715	" without JBOD MAP support from %s %d\n", __func__, __LINE__);
4716	} else {
4717	device_id = ((sdev->channel % 2) * MEGASAS_MAX_DEV_PER_CHANNEL)
4718	+ sdev->id;
4719	devhandle = device_id;
4720	}
4721
4722	return devhandle;
4723	}
4724
4725	/*
4726	* megasas_task_abort_fusion : SCSI task abort function for fusion adapters
4727	* @scmd : pointer to scsi command object
4728	*
4729	* Return SUCCESS, if command aborted else FAILED
4730	*/
4731
4732	int megasas_task_abort_fusion(struct scsi_cmnd *scmd)
4733	{
4734	struct megasas_instance *instance;
4735	u16 smid, devhandle;
4736	int ret;
4737	struct MR_PRIV_DEVICE *mr_device_priv_data;
4738	mr_device_priv_data = scmd->device->hostdata;
4739
4740	instance = (struct megasas_instance *)scmd->device->host->hostdata;
4741
4742	if (atomic_read(v: &instance->adprecovery) != MEGASAS_HBA_OPERATIONAL) {
4743	dev_err(&instance->pdev->dev, "Controller is not OPERATIONAL,"
4744	"SCSI host:%d\n", instance->host->host_no);
4745	ret = FAILED;
4746	return ret;
4747	}
4748
4749	if (!mr_device_priv_data) {
4750	sdev_printk(KERN_INFO, scmd->device, "device been deleted! "
4751	"scmd(%p)\n", scmd);
4752	scmd->result = DID_NO_CONNECT << 16;
4753	ret = SUCCESS;
4754	goto out;
4755	}
4756
4757	if (!mr_device_priv_data->is_tm_capable) {
4758	ret = FAILED;
4759	goto out;
4760	}
4761
4762	mutex_lock(&instance->reset_mutex);
4763
4764	smid = megasas_fusion_smid_lookup(scmd);
4765
4766	if (!smid) {
4767	ret = SUCCESS;
4768	scmd_printk(KERN_NOTICE, scmd, "Command for which abort is"
4769	" issued is not found in outstanding commands\n");
4770	mutex_unlock(lock: &instance->reset_mutex);
4771	goto out;
4772	}
4773
4774	devhandle = megasas_get_tm_devhandle(sdev: scmd->device);
4775
4776	if (devhandle == (u16)ULONG_MAX) {
4777	ret = FAILED;
4778	sdev_printk(KERN_INFO, scmd->device,
4779	"task abort issued for invalid devhandle\n");
4780	mutex_unlock(lock: &instance->reset_mutex);
4781	goto out;
4782	}
4783	sdev_printk(KERN_INFO, scmd->device,
4784	"attempting task abort! scmd(0x%p) tm_dev_handle 0x%x\n",
4785	scmd, devhandle);
4786
4787	mr_device_priv_data->tm_busy = true;
4788	ret = megasas_issue_tm(instance, device_handle: devhandle,
4789	channel: scmd->device->channel, id: scmd->device->id, smid_task: smid,
4790	MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK,
4791	mr_device_priv_data);
4792	mr_device_priv_data->tm_busy = false;
4793
4794	mutex_unlock(lock: &instance->reset_mutex);
4795	scmd_printk(KERN_INFO, scmd, "task abort %s!! scmd(0x%p)\n",
4796	((ret == SUCCESS) ? "SUCCESS" : "FAILED"), scmd);
4797	out:
4798	scsi_print_command(scmd);
4799	if (megasas_dbg_lvl & TM_DEBUG)
4800	megasas_dump_fusion_io(scmd);
4801
4802	return ret;
4803	}
4804
4805	/*
4806	* megasas_reset_target_fusion : target reset function for fusion adapters
4807	* scmd: SCSI command pointer
4808	*
4809	* Returns SUCCESS if all commands associated with target aborted else FAILED
4810	*/
4811
4812	int megasas_reset_target_fusion(struct scsi_cmnd *scmd)
4813	{
4814
4815	struct megasas_instance *instance;
4816	int ret = FAILED;
4817	u16 devhandle;
4818	struct MR_PRIV_DEVICE *mr_device_priv_data;
4819	mr_device_priv_data = scmd->device->hostdata;
4820
4821	instance = (struct megasas_instance *)scmd->device->host->hostdata;
4822
4823	if (atomic_read(v: &instance->adprecovery) != MEGASAS_HBA_OPERATIONAL) {
4824	dev_err(&instance->pdev->dev, "Controller is not OPERATIONAL,"
4825	"SCSI host:%d\n", instance->host->host_no);
4826	ret = FAILED;
4827	return ret;
4828	}
4829
4830	if (!mr_device_priv_data) {
4831	sdev_printk(KERN_INFO, scmd->device,
4832	"device been deleted! scmd: (0x%p)\n", scmd);
4833	scmd->result = DID_NO_CONNECT << 16;
4834	ret = SUCCESS;
4835	goto out;
4836	}
4837
4838	if (!mr_device_priv_data->is_tm_capable) {
4839	ret = FAILED;
4840	goto out;
4841	}
4842
4843	mutex_lock(&instance->reset_mutex);
4844	devhandle = megasas_get_tm_devhandle(sdev: scmd->device);
4845
4846	if (devhandle == (u16)ULONG_MAX) {
4847	ret = FAILED;
4848	sdev_printk(KERN_INFO, scmd->device,
4849	"target reset issued for invalid devhandle\n");
4850	mutex_unlock(lock: &instance->reset_mutex);
4851	goto out;
4852	}
4853
4854	sdev_printk(KERN_INFO, scmd->device,
4855	"attempting target reset! scmd(0x%p) tm_dev_handle: 0x%x\n",
4856	scmd, devhandle);
4857	mr_device_priv_data->tm_busy = true;
4858	ret = megasas_issue_tm(instance, device_handle: devhandle,
4859	channel: scmd->device->channel, id: scmd->device->id, smid_task: 0,
4860	MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET,
4861	mr_device_priv_data);
4862	mr_device_priv_data->tm_busy = false;
4863	mutex_unlock(lock: &instance->reset_mutex);
4864	scmd_printk(KERN_NOTICE, scmd, "target reset %s!!\n",
4865	(ret == SUCCESS) ? "SUCCESS" : "FAILED");
4866
4867	out:
4868	return ret;
4869	}
4870
4871	/*SRIOV get other instance in cluster if any*/
4872	static struct
4873	megasas_instance *megasas_get_peer_instance(struct megasas_instance *instance)
4874	{
4875	int i;
4876
4877	for (i = 0; i < MAX_MGMT_ADAPTERS; i++) {
4878	if (megasas_mgmt_info.instance[i] &&
4879	(megasas_mgmt_info.instance[i] != instance) &&
4880	megasas_mgmt_info.instance[i]->requestorId &&
4881	megasas_mgmt_info.instance[i]->peerIsPresent &&
4882	(memcmp(p: (megasas_mgmt_info.instance[i]->clusterId),
4883	q: instance->clusterId, MEGASAS_CLUSTER_ID_SIZE) == 0))
4884	return megasas_mgmt_info.instance[i];
4885	}
4886	return NULL;
4887	}
4888
4889	/* Check for a second path that is currently UP */
4890	int megasas_check_mpio_paths(struct megasas_instance *instance,
4891	struct scsi_cmnd *scmd)
4892	{
4893	struct megasas_instance *peer_instance = NULL;
4894	int retval = (DID_REQUEUE << 16);
4895
4896	if (instance->peerIsPresent) {
4897	peer_instance = megasas_get_peer_instance(instance);
4898	if ((peer_instance) &&
4899	(atomic_read(v: &peer_instance->adprecovery) ==
4900	MEGASAS_HBA_OPERATIONAL))
4901	retval = (DID_NO_CONNECT << 16);
4902	}
4903	return retval;
4904	}
4905
4906	/* Core fusion reset function */
4907	int megasas_reset_fusion(struct Scsi_Host *shost, int reason)
4908	{
4909	int retval = SUCCESS, i, j, convert = 0;
4910	struct megasas_instance *instance;
4911	struct megasas_cmd_fusion *cmd_fusion, *r1_cmd;
4912	struct fusion_context *fusion;
4913	u32 abs_state, status_reg, reset_adapter, fpio_count = 0;
4914	u32 io_timeout_in_crash_mode = 0;
4915	struct scsi_cmnd *scmd_local = NULL;
4916	struct scsi_device *sdev;
4917	int ret_target_prop = DCMD_FAILED;
4918	bool is_target_prop = false;
4919	bool do_adp_reset = true;
4920	int max_reset_tries = MEGASAS_FUSION_MAX_RESET_TRIES;
4921
4922	instance = (struct megasas_instance *)shost->hostdata;
4923	fusion = instance->ctrl_context;
4924
4925	mutex_lock(&instance->reset_mutex);
4926
4927	if (atomic_read(v: &instance->adprecovery) == MEGASAS_HW_CRITICAL_ERROR) {
4928	dev_warn(&instance->pdev->dev, "Hardware critical error, "
4929	"returning FAILED for scsi%d.\n",
4930	instance->host->host_no);
4931	mutex_unlock(lock: &instance->reset_mutex);
4932	return FAILED;
4933	}
4934	status_reg = instance->instancet->read_fw_status_reg(instance);
4935	abs_state = status_reg & MFI_STATE_MASK;
4936
4937	/* IO timeout detected, forcibly put FW in FAULT state */
4938	if (abs_state != MFI_STATE_FAULT && instance->crash_dump_buf &&
4939	instance->crash_dump_app_support && reason) {
4940	dev_info(&instance->pdev->dev, "IO/DCMD timeout is detected, "
4941	"forcibly FAULT Firmware\n");
4942	atomic_set(v: &instance->adprecovery, i: MEGASAS_ADPRESET_SM_INFAULT);
4943	status_reg = megasas_readl(instance, addr: &instance->reg_set->doorbell);
4944	writel(val: status_reg | MFI_STATE_FORCE_OCR,
4945	addr: &instance->reg_set->doorbell);
4946	readl(addr: &instance->reg_set->doorbell);
4947	mutex_unlock(lock: &instance->reset_mutex);
4948	do {
4949	ssleep(seconds: 3);
4950	io_timeout_in_crash_mode++;
4951	dev_dbg(&instance->pdev->dev, "waiting for [%d] "
4952	"seconds for crash dump collection and OCR "
4953	"to be done\n", (io_timeout_in_crash_mode * 3));
4954	} while ((atomic_read(v: &instance->adprecovery) != MEGASAS_HBA_OPERATIONAL) &&
4955	(io_timeout_in_crash_mode < 80));
4956
4957	if (atomic_read(v: &instance->adprecovery) == MEGASAS_HBA_OPERATIONAL) {
4958	dev_info(&instance->pdev->dev, "OCR done for IO "
4959	"timeout case\n");
4960	retval = SUCCESS;
4961	} else {
4962	dev_info(&instance->pdev->dev, "Controller is not "
4963	"operational after 240 seconds wait for IO "
4964	"timeout case in FW crash dump mode\n do "
4965	"OCR/kill adapter\n");
4966	retval = megasas_reset_fusion(shost, reason: 0);
4967	}
4968	return retval;
4969	}
4970
4971	if (instance->requestorId && !instance->skip_heartbeat_timer_del)
4972	del_timer_sync(timer: &instance->sriov_heartbeat_timer);
4973	set_bit(MEGASAS_FUSION_IN_RESET, addr: &instance->reset_flags);
4974	set_bit(MEGASAS_FUSION_OCR_NOT_POSSIBLE, addr: &instance->reset_flags);
4975	atomic_set(v: &instance->adprecovery, i: MEGASAS_ADPRESET_SM_POLLING);
4976	instance->instancet->disable_intr(instance);
4977	megasas_sync_irqs(instance_addr: (unsigned long)instance);
4978
4979	/* First try waiting for commands to complete */
4980	if (megasas_wait_for_outstanding_fusion(instance, reason,
4981	convert: &convert)) {
4982	atomic_set(v: &instance->adprecovery, i: MEGASAS_ADPRESET_SM_INFAULT);
4983	dev_warn(&instance->pdev->dev, "resetting fusion "
4984	"adapter scsi%d.\n", instance->host->host_no);
4985	if (convert)
4986	reason = 0;
4987
4988	if (megasas_dbg_lvl & OCR_DEBUG)
4989	dev_info(&instance->pdev->dev, "\nPending SCSI commands:\n");
4990
4991	/* Now return commands back to the OS */
4992	for (i = 0 ; i < instance->max_scsi_cmds; i++) {
4993	cmd_fusion = fusion->cmd_list[i];
4994	/*check for extra commands issued by driver*/
4995	if (instance->adapter_type >= VENTURA_SERIES) {
4996	r1_cmd = fusion->cmd_list[i + instance->max_fw_cmds];
4997	megasas_return_cmd_fusion(instance, cmd: r1_cmd);
4998	}
4999	scmd_local = cmd_fusion->scmd;
5000	if (cmd_fusion->scmd) {
5001	if (megasas_dbg_lvl & OCR_DEBUG) {
5002	sdev_printk(KERN_INFO,
5003	cmd_fusion->scmd->device, "SMID: 0x%x\n",
5004	cmd_fusion->index);
5005	megasas_dump_fusion_io(scmd: cmd_fusion->scmd);
5006	}
5007
5008	if (cmd_fusion->io_request->Function ==
5009	MPI2_FUNCTION_SCSI_IO_REQUEST)
5010	fpio_count++;
5011
5012	scmd_local->result =
5013	megasas_check_mpio_paths(instance,
5014	scmd: scmd_local);
5015	if (instance->ldio_threshold &&
5016	megasas_cmd_type(cmd: scmd_local) == READ_WRITE_LDIO)
5017	atomic_dec(v: &instance->ldio_outstanding);
5018	megasas_return_cmd_fusion(instance, cmd: cmd_fusion);
5019	scsi_dma_unmap(cmd: scmd_local);
5020	scsi_done(cmd: scmd_local);
5021	}
5022	}
5023
5024	dev_info(&instance->pdev->dev, "Outstanding fastpath IOs: %d\n",
5025	fpio_count);
5026
5027	atomic_set(v: &instance->fw_outstanding, i: 0);
5028
5029	status_reg = instance->instancet->read_fw_status_reg(instance);
5030	abs_state = status_reg & MFI_STATE_MASK;
5031	reset_adapter = status_reg & MFI_RESET_ADAPTER;
5032	if (instance->disableOnlineCtrlReset ||
5033	(abs_state == MFI_STATE_FAULT && !reset_adapter)) {
5034	/* Reset not supported, kill adapter */
5035	dev_warn(&instance->pdev->dev, "Reset not supported"
5036	", killing adapter scsi%d.\n",
5037	instance->host->host_no);
5038	goto kill_hba;
5039	}
5040
5041	/* Let SR-IOV VF & PF sync up if there was a HB failure */
5042	if (instance->requestorId && !reason) {
5043	msleep(MEGASAS_OCR_SETTLE_TIME_VF);
5044	do_adp_reset = false;
5045	max_reset_tries = MEGASAS_SRIOV_MAX_RESET_TRIES_VF;
5046	}
5047
5048	/* Now try to reset the chip */
5049	for (i = 0; i < max_reset_tries; i++) {
5050	/*
5051	* Do adp reset and wait for
5052	* controller to transition to ready
5053	*/
5054	if (megasas_adp_reset_wait_for_ready(instance,
5055	do_adp_reset, ocr_context: 1) == FAILED)
5056	continue;
5057
5058	/* Wait for FW to become ready */
5059	if (megasas_transition_to_ready(instance, ocr: 1)) {
5060	dev_warn(&instance->pdev->dev,
5061	"Failed to transition controller to ready for "
5062	"scsi%d.\n", instance->host->host_no);
5063	continue;
5064	}
5065	megasas_reset_reply_desc(instance);
5066	megasas_fusion_update_can_queue(instance, fw_boot_context: OCR_CONTEXT);
5067
5068	if (megasas_ioc_init_fusion(instance)) {
5069	continue;
5070	}
5071
5072	if (megasas_get_ctrl_info(instance)) {
5073	dev_info(&instance->pdev->dev,
5074	"Failed from %s %d\n",
5075	__func__, __LINE__);
5076	goto kill_hba;
5077	}
5078
5079	megasas_refire_mgmt_cmd(instance,
5080	return_ioctl: (i == (MEGASAS_FUSION_MAX_RESET_TRIES - 1)
5081	? 1 : 0));
5082
5083	/* Reset load balance info */
5084	if (fusion->load_balance_info)
5085	memset(fusion->load_balance_info, 0,
5086	(sizeof(struct LD_LOAD_BALANCE_INFO) *
5087	MAX_LOGICAL_DRIVES_EXT));
5088
5089	if (!megasas_get_map_info(instance)) {
5090	megasas_sync_map_info(instance);
5091	} else {
5092	/*
5093	* Return pending polled mode cmds before
5094	* retrying OCR
5095	*/
5096	megasas_return_polled_cmds(instance);
5097	continue;
5098	}
5099
5100	megasas_setup_jbod_map(instance);
5101
5102	/* reset stream detection array */
5103	if (instance->adapter_type >= VENTURA_SERIES) {
5104	for (j = 0; j < MAX_LOGICAL_DRIVES_EXT; ++j) {
5105	memset(fusion->stream_detect_by_ld[j],
5106	0, sizeof(struct LD_STREAM_DETECT));
5107	fusion->stream_detect_by_ld[j]->mru_bit_map
5108	= MR_STREAM_BITMAP;
5109	}
5110	}
5111
5112	clear_bit(MEGASAS_FUSION_IN_RESET,
5113	addr: &instance->reset_flags);
5114	instance->instancet->enable_intr(instance);
5115	megasas_enable_irq_poll(instance);
5116	shost_for_each_device(sdev, shost) {
5117	if ((instance->tgt_prop) &&
5118	(instance->nvme_page_size))
5119	ret_target_prop = megasas_get_target_prop(instance, sdev);
5120
5121	is_target_prop = (ret_target_prop == DCMD_SUCCESS) ? true : false;
5122	megasas_set_dynamic_target_properties(sdev, is_target_prop);
5123	}
5124
5125	status_reg = instance->instancet->read_fw_status_reg
5126	(instance);
5127	abs_state = status_reg & MFI_STATE_MASK;
5128	if (abs_state != MFI_STATE_OPERATIONAL) {
5129	dev_info(&instance->pdev->dev,
5130	"Adapter is not OPERATIONAL, state 0x%x for scsi:%d\n",
5131	abs_state, instance->host->host_no);
5132	goto out;
5133	}
5134	atomic_set(v: &instance->adprecovery, i: MEGASAS_HBA_OPERATIONAL);
5135
5136	dev_info(&instance->pdev->dev,
5137	"Adapter is OPERATIONAL for scsi:%d\n",
5138	instance->host->host_no);
5139
5140	/* Restart SR-IOV heartbeat */
5141	if (instance->requestorId) {
5142	if (!megasas_sriov_start_heartbeat(instance, initial: 0))
5143	megasas_start_timer(instance);
5144	else
5145	instance->skip_heartbeat_timer_del = 1;
5146	}
5147
5148	if (instance->crash_dump_drv_support &&
5149	instance->crash_dump_app_support)
5150	megasas_set_crash_dump_params(instance,
5151	crash_buf_state: MR_CRASH_BUF_TURN_ON);
5152	else
5153	megasas_set_crash_dump_params(instance,
5154	crash_buf_state: MR_CRASH_BUF_TURN_OFF);
5155
5156	if (instance->snapdump_wait_time) {
5157	megasas_get_snapdump_properties(instance);
5158	dev_info(&instance->pdev->dev,
5159	"Snap dump wait time\t: %d\n",
5160	instance->snapdump_wait_time);
5161	}
5162
5163	retval = SUCCESS;
5164
5165	/* Adapter reset completed successfully */
5166	dev_warn(&instance->pdev->dev,
5167	"Reset successful for scsi%d.\n",
5168	instance->host->host_no);
5169
5170	goto out;
5171	}
5172	/* Reset failed, kill the adapter */
5173	dev_warn(&instance->pdev->dev, "Reset failed, killing "
5174	"adapter scsi%d.\n", instance->host->host_no);
5175	goto kill_hba;
5176	} else {
5177	/* For VF: Restart HB timer if we didn't OCR */
5178	if (instance->requestorId) {
5179	megasas_start_timer(instance);
5180	}
5181	clear_bit(MEGASAS_FUSION_IN_RESET, addr: &instance->reset_flags);
5182	instance->instancet->enable_intr(instance);
5183	megasas_enable_irq_poll(instance);
5184	atomic_set(v: &instance->adprecovery, i: MEGASAS_HBA_OPERATIONAL);
5185	goto out;
5186	}
5187	kill_hba:
5188	megaraid_sas_kill_hba(instance);
5189	megasas_enable_irq_poll(instance);
5190	instance->skip_heartbeat_timer_del = 1;
5191	retval = FAILED;
5192	out:
5193	clear_bit(MEGASAS_FUSION_OCR_NOT_POSSIBLE, addr: &instance->reset_flags);
5194	mutex_unlock(lock: &instance->reset_mutex);
5195	return retval;
5196	}
5197
5198	/* Fusion Crash dump collection */
5199	static void megasas_fusion_crash_dump(struct megasas_instance *instance)
5200	{
5201	u32 status_reg;
5202	u8 partial_copy = 0;
5203	int wait = 0;
5204
5205
5206	status_reg = instance->instancet->read_fw_status_reg(instance);
5207
5208	/*
5209	* Allocate host crash buffers to copy data from 1 MB DMA crash buffer
5210	* to host crash buffers
5211	*/
5212	if (instance->drv_buf_index == 0) {
5213	/* Buffer is already allocated for old Crash dump.
5214	* Do OCR and do not wait for crash dump collection
5215	*/
5216	if (instance->drv_buf_alloc) {
5217	dev_info(&instance->pdev->dev, "earlier crash dump is "
5218	"not yet copied by application, ignoring this "
5219	"crash dump and initiating OCR\n");
5220	status_reg |= MFI_STATE_CRASH_DUMP_DONE;
5221	writel(val: status_reg,
5222	addr: &instance->reg_set->outbound_scratch_pad_0);
5223	readl(addr: &instance->reg_set->outbound_scratch_pad_0);
5224	return;
5225	}
5226	megasas_alloc_host_crash_buffer(instance);
5227	dev_info(&instance->pdev->dev, "Number of host crash buffers "
5228	"allocated: %d\n", instance->drv_buf_alloc);
5229	}
5230
5231	while (!(status_reg & MFI_STATE_CRASH_DUMP_DONE) &&
5232	(wait < MEGASAS_WATCHDOG_WAIT_COUNT)) {
5233	if (!(status_reg & MFI_STATE_DMADONE)) {
5234	/*
5235	* Next crash dump buffer is not yet DMA'd by FW
5236	* Check after 10ms. Wait for 1 second for FW to
5237	* post the next buffer. If not bail out.
5238	*/
5239	wait++;
5240	msleep(MEGASAS_WAIT_FOR_NEXT_DMA_MSECS);
5241	status_reg = instance->instancet->read_fw_status_reg(
5242	instance);
5243	continue;
5244	}
5245
5246	wait = 0;
5247	if (instance->drv_buf_index >= instance->drv_buf_alloc) {
5248	dev_info(&instance->pdev->dev,
5249	"Driver is done copying the buffer: %d\n",
5250	instance->drv_buf_alloc);
5251	status_reg |= MFI_STATE_CRASH_DUMP_DONE;
5252	partial_copy = 1;
5253	break;
5254	} else {
5255	memcpy(instance->crash_buf[instance->drv_buf_index],
5256	instance->crash_dump_buf, CRASH_DMA_BUF_SIZE);
5257	instance->drv_buf_index++;
5258	status_reg &= ~MFI_STATE_DMADONE;
5259	}
5260
5261	writel(val: status_reg, addr: &instance->reg_set->outbound_scratch_pad_0);
5262	readl(addr: &instance->reg_set->outbound_scratch_pad_0);
5263
5264	msleep(MEGASAS_WAIT_FOR_NEXT_DMA_MSECS);
5265	status_reg = instance->instancet->read_fw_status_reg(instance);
5266	}
5267
5268	if (status_reg & MFI_STATE_CRASH_DUMP_DONE) {
5269	dev_info(&instance->pdev->dev, "Crash Dump is available,number "
5270	"of copied buffers: %d\n", instance->drv_buf_index);
5271	instance->fw_crash_buffer_size = instance->drv_buf_index;
5272	instance->fw_crash_state = AVAILABLE;
5273	instance->drv_buf_index = 0;
5274	writel(val: status_reg, addr: &instance->reg_set->outbound_scratch_pad_0);
5275	readl(addr: &instance->reg_set->outbound_scratch_pad_0);
5276	if (!partial_copy)
5277	megasas_reset_fusion(shost: instance->host, reason: 0);
5278	}
5279	}
5280
5281
5282	/* Fusion OCR work queue */
5283	void megasas_fusion_ocr_wq(struct work_struct *work)
5284	{
5285	struct megasas_instance *instance =
5286	container_of(work, struct megasas_instance, work_init);
5287
5288	megasas_reset_fusion(shost: instance->host, reason: 0);
5289	}
5290
5291	/* Allocate fusion context */
5292	int
5293	megasas_alloc_fusion_context(struct megasas_instance *instance)
5294	{
5295	struct fusion_context *fusion;
5296
5297	instance->ctrl_context = kzalloc(size: sizeof(struct fusion_context),
5298	GFP_KERNEL);
5299	if (!instance->ctrl_context) {
5300	dev_err(&instance->pdev->dev, "Failed from %s %d\n",
5301	__func__, __LINE__);
5302	return -ENOMEM;
5303	}
5304
5305	fusion = instance->ctrl_context;
5306
5307	fusion->log_to_span_pages = get_order(MAX_LOGICAL_DRIVES_EXT *
5308	sizeof(LD_SPAN_INFO));
5309	fusion->log_to_span =
5310	(PLD_SPAN_INFO)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
5311	order: fusion->log_to_span_pages);
5312	if (!fusion->log_to_span) {
5313	fusion->log_to_span =
5314	vzalloc(array_size(MAX_LOGICAL_DRIVES_EXT,
5315	sizeof(LD_SPAN_INFO)));
5316	if (!fusion->log_to_span) {
5317	dev_err(&instance->pdev->dev, "Failed from %s %d\n",
5318	__func__, __LINE__);
5319	return -ENOMEM;
5320	}
5321	}
5322
5323	fusion->load_balance_info_pages = get_order(MAX_LOGICAL_DRIVES_EXT *
5324	sizeof(struct LD_LOAD_BALANCE_INFO));
5325	fusion->load_balance_info =
5326	(struct LD_LOAD_BALANCE_INFO *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
5327	order: fusion->load_balance_info_pages);
5328	if (!fusion->load_balance_info) {
5329	fusion->load_balance_info =
5330	vzalloc(array_size(MAX_LOGICAL_DRIVES_EXT,
5331	sizeof(struct LD_LOAD_BALANCE_INFO)));
5332	if (!fusion->load_balance_info)
5333	dev_err(&instance->pdev->dev, "Failed to allocate load_balance_info, "
5334	"continuing without Load Balance support\n");
5335	}
5336
5337	return 0;
5338	}
5339
5340	void
5341	megasas_free_fusion_context(struct megasas_instance *instance)
5342	{
5343	struct fusion_context *fusion = instance->ctrl_context;
5344
5345	if (fusion) {
5346	if (fusion->load_balance_info) {
5347	if (is_vmalloc_addr(x: fusion->load_balance_info))
5348	vfree(addr: fusion->load_balance_info);
5349	else
5350	free_pages(addr: (ulong)fusion->load_balance_info,
5351	order: fusion->load_balance_info_pages);
5352	}
5353
5354	if (fusion->log_to_span) {
5355	if (is_vmalloc_addr(x: fusion->log_to_span))
5356	vfree(addr: fusion->log_to_span);
5357	else
5358	free_pages(addr: (ulong)fusion->log_to_span,
5359	order: fusion->log_to_span_pages);
5360	}
5361
5362	kfree(objp: fusion);
5363	}
5364	}
5365
5366	struct megasas_instance_template megasas_instance_template_fusion = {
5367	.enable_intr = megasas_enable_intr_fusion,
5368	.disable_intr = megasas_disable_intr_fusion,
5369	.clear_intr = megasas_clear_intr_fusion,
5370	.read_fw_status_reg = megasas_read_fw_status_reg_fusion,
5371	.adp_reset = megasas_adp_reset_fusion,
5372	.check_reset = megasas_check_reset_fusion,
5373	.service_isr = megasas_isr_fusion,
5374	.tasklet = megasas_complete_cmd_dpc_fusion,
5375	.init_adapter = megasas_init_adapter_fusion,
5376	.build_and_issue_cmd = megasas_build_and_issue_cmd_fusion,
5377	.issue_dcmd = megasas_issue_dcmd_fusion,
5378	};
5379