1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* Driver for Broadcom MPI3 Storage Controllers
4	*
5	* Copyright (C) 2017-2023 Broadcom Inc.
6	* (mailto: mpi3mr-linuxdrv.pdl@broadcom.com)
7	*
8	*/
9
10	#include "mpi3mr.h"
11	#include <linux/io-64-nonatomic-lo-hi.h>
12
13	static int
14	mpi3mr_issue_reset(struct mpi3mr_ioc *mrioc, u16 reset_type, u32 reset_reason);
15	static int mpi3mr_setup_admin_qpair(struct mpi3mr_ioc *mrioc);
16	static void mpi3mr_process_factsdata(struct mpi3mr_ioc *mrioc,
17	struct mpi3_ioc_facts_data *facts_data);
18	static void mpi3mr_pel_wait_complete(struct mpi3mr_ioc *mrioc,
19	struct mpi3mr_drv_cmd *drv_cmd);
20
21	static int poll_queues;
22	module_param(poll_queues, int, 0444);
23	MODULE_PARM_DESC(poll_queues, "Number of queues for io_uring poll mode. (Range 1 - 126)");
24
25	#if defined(writeq) && defined(CONFIG_64BIT)
26	static inline void mpi3mr_writeq(__u64 b, volatile void __iomem *addr)
27	{
28	writeq(val: b, addr);
29	}
30	#else
31	static inline void mpi3mr_writeq(__u64 b, volatile void __iomem *addr)
32	{
33	__u64 data_out = b;
34
35	writel((u32)(data_out), addr);
36	writel((u32)(data_out >> 32), (addr + 4));
37	}
38	#endif
39
40	static inline bool
41	mpi3mr_check_req_qfull(struct op_req_qinfo *op_req_q)
42	{
43	u16 pi, ci, max_entries;
44	bool is_qfull = false;
45
46	pi = op_req_q->pi;
47	ci = READ_ONCE(op_req_q->ci);
48	max_entries = op_req_q->num_requests;
49
50	if ((ci == (pi + 1)) || ((!ci) && (pi == (max_entries - 1))))
51	is_qfull = true;
52
53	return is_qfull;
54	}
55
56	static void mpi3mr_sync_irqs(struct mpi3mr_ioc *mrioc)
57	{
58	u16 i, max_vectors;
59
60	max_vectors = mrioc->intr_info_count;
61
62	for (i = 0; i < max_vectors; i++)
63	synchronize_irq(irq: pci_irq_vector(dev: mrioc->pdev, nr: i));
64	}
65
66	void mpi3mr_ioc_disable_intr(struct mpi3mr_ioc *mrioc)
67	{
68	mrioc->intr_enabled = 0;
69	mpi3mr_sync_irqs(mrioc);
70	}
71
72	void mpi3mr_ioc_enable_intr(struct mpi3mr_ioc *mrioc)
73	{
74	mrioc->intr_enabled = 1;
75	}
76
77	static void mpi3mr_cleanup_isr(struct mpi3mr_ioc *mrioc)
78	{
79	u16 i;
80
81	mpi3mr_ioc_disable_intr(mrioc);
82
83	if (!mrioc->intr_info)
84	return;
85
86	for (i = 0; i < mrioc->intr_info_count; i++)
87	free_irq(pci_irq_vector(dev: mrioc->pdev, nr: i),
88	(mrioc->intr_info + i));
89
90	kfree(objp: mrioc->intr_info);
91	mrioc->intr_info = NULL;
92	mrioc->intr_info_count = 0;
93	mrioc->is_intr_info_set = false;
94	pci_free_irq_vectors(dev: mrioc->pdev);
95	}
96
97	void mpi3mr_add_sg_single(void *paddr, u8 flags, u32 length,
98	dma_addr_t dma_addr)
99	{
100	struct mpi3_sge_common *sgel = paddr;
101
102	sgel->flags = flags;
103	sgel->length = cpu_to_le32(length);
104	sgel->address = cpu_to_le64(dma_addr);
105	}
106
107	void mpi3mr_build_zero_len_sge(void *paddr)
108	{
109	u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST;
110
111	mpi3mr_add_sg_single(paddr, flags: sgl_flags, length: 0, dma_addr: -1);
112	}
113
114	void *mpi3mr_get_reply_virt_addr(struct mpi3mr_ioc *mrioc,
115	dma_addr_t phys_addr)
116	{
117	if (!phys_addr)
118	return NULL;
119
120	if ((phys_addr < mrioc->reply_buf_dma) ||
121	(phys_addr > mrioc->reply_buf_dma_max_address))
122	return NULL;
123
124	return mrioc->reply_buf + (phys_addr - mrioc->reply_buf_dma);
125	}
126
127	void *mpi3mr_get_sensebuf_virt_addr(struct mpi3mr_ioc *mrioc,
128	dma_addr_t phys_addr)
129	{
130	if (!phys_addr)
131	return NULL;
132
133	return mrioc->sense_buf + (phys_addr - mrioc->sense_buf_dma);
134	}
135
136	static void mpi3mr_repost_reply_buf(struct mpi3mr_ioc *mrioc,
137	u64 reply_dma)
138	{
139	u32 old_idx = 0;
140	unsigned long flags;
141
142	spin_lock_irqsave(&mrioc->reply_free_queue_lock, flags);
143	old_idx = mrioc->reply_free_queue_host_index;
144	mrioc->reply_free_queue_host_index = (
145	(mrioc->reply_free_queue_host_index ==
146	(mrioc->reply_free_qsz - 1)) ? 0 :
147	(mrioc->reply_free_queue_host_index + 1));
148	mrioc->reply_free_q[old_idx] = cpu_to_le64(reply_dma);
149	writel(val: mrioc->reply_free_queue_host_index,
150	addr: &mrioc->sysif_regs->reply_free_host_index);
151	spin_unlock_irqrestore(lock: &mrioc->reply_free_queue_lock, flags);
152	}
153
154	void mpi3mr_repost_sense_buf(struct mpi3mr_ioc *mrioc,
155	u64 sense_buf_dma)
156	{
157	u32 old_idx = 0;
158	unsigned long flags;
159
160	spin_lock_irqsave(&mrioc->sbq_lock, flags);
161	old_idx = mrioc->sbq_host_index;
162	mrioc->sbq_host_index = ((mrioc->sbq_host_index ==
163	(mrioc->sense_buf_q_sz - 1)) ? 0 :
164	(mrioc->sbq_host_index + 1));
165	mrioc->sense_buf_q[old_idx] = cpu_to_le64(sense_buf_dma);
166	writel(val: mrioc->sbq_host_index,
167	addr: &mrioc->sysif_regs->sense_buffer_free_host_index);
168	spin_unlock_irqrestore(lock: &mrioc->sbq_lock, flags);
169	}
170
171	static void mpi3mr_print_event_data(struct mpi3mr_ioc *mrioc,
172	struct mpi3_event_notification_reply *event_reply)
173	{
174	char *desc = NULL;
175	u16 event;
176
177	event = event_reply->event;
178
179	switch (event) {
180	case MPI3_EVENT_LOG_DATA:
181	desc = "Log Data";
182	break;
183	case MPI3_EVENT_CHANGE:
184	desc = "Event Change";
185	break;
186	case MPI3_EVENT_GPIO_INTERRUPT:
187	desc = "GPIO Interrupt";
188	break;
189	case MPI3_EVENT_CABLE_MGMT:
190	desc = "Cable Management";
191	break;
192	case MPI3_EVENT_ENERGY_PACK_CHANGE:
193	desc = "Energy Pack Change";
194	break;
195	case MPI3_EVENT_DEVICE_ADDED:
196	{
197	struct mpi3_device_page0 *event_data =
198	(struct mpi3_device_page0 *)event_reply->event_data;
199	ioc_info(mrioc, "Device Added: dev=0x%04x Form=0x%x\n",
200	event_data->dev_handle, event_data->device_form);
201	return;
202	}
203	case MPI3_EVENT_DEVICE_INFO_CHANGED:
204	{
205	struct mpi3_device_page0 *event_data =
206	(struct mpi3_device_page0 *)event_reply->event_data;
207	ioc_info(mrioc, "Device Info Changed: dev=0x%04x Form=0x%x\n",
208	event_data->dev_handle, event_data->device_form);
209	return;
210	}
211	case MPI3_EVENT_DEVICE_STATUS_CHANGE:
212	{
213	struct mpi3_event_data_device_status_change *event_data =
214	(struct mpi3_event_data_device_status_change *)event_reply->event_data;
215	ioc_info(mrioc, "Device status Change: dev=0x%04x RC=0x%x\n",
216	event_data->dev_handle, event_data->reason_code);
217	return;
218	}
219	case MPI3_EVENT_SAS_DISCOVERY:
220	{
221	struct mpi3_event_data_sas_discovery *event_data =
222	(struct mpi3_event_data_sas_discovery *)event_reply->event_data;
223	ioc_info(mrioc, "SAS Discovery: (%s) status (0x%08x)\n",
224	(event_data->reason_code == MPI3_EVENT_SAS_DISC_RC_STARTED) ?
225	"start" : "stop",
226	le32_to_cpu(event_data->discovery_status));
227	return;
228	}
229	case MPI3_EVENT_SAS_BROADCAST_PRIMITIVE:
230	desc = "SAS Broadcast Primitive";
231	break;
232	case MPI3_EVENT_SAS_NOTIFY_PRIMITIVE:
233	desc = "SAS Notify Primitive";
234	break;
235	case MPI3_EVENT_SAS_INIT_DEVICE_STATUS_CHANGE:
236	desc = "SAS Init Device Status Change";
237	break;
238	case MPI3_EVENT_SAS_INIT_TABLE_OVERFLOW:
239	desc = "SAS Init Table Overflow";
240	break;
241	case MPI3_EVENT_SAS_TOPOLOGY_CHANGE_LIST:
242	desc = "SAS Topology Change List";
243	break;
244	case MPI3_EVENT_ENCL_DEVICE_STATUS_CHANGE:
245	desc = "Enclosure Device Status Change";
246	break;
247	case MPI3_EVENT_ENCL_DEVICE_ADDED:
248	desc = "Enclosure Added";
249	break;
250	case MPI3_EVENT_HARD_RESET_RECEIVED:
251	desc = "Hard Reset Received";
252	break;
253	case MPI3_EVENT_SAS_PHY_COUNTER:
254	desc = "SAS PHY Counter";
255	break;
256	case MPI3_EVENT_SAS_DEVICE_DISCOVERY_ERROR:
257	desc = "SAS Device Discovery Error";
258	break;
259	case MPI3_EVENT_PCIE_TOPOLOGY_CHANGE_LIST:
260	desc = "PCIE Topology Change List";
261	break;
262	case MPI3_EVENT_PCIE_ENUMERATION:
263	{
264	struct mpi3_event_data_pcie_enumeration *event_data =
265	(struct mpi3_event_data_pcie_enumeration *)event_reply->event_data;
266	ioc_info(mrioc, "PCIE Enumeration: (%s)",
267	(event_data->reason_code ==
268	MPI3_EVENT_PCIE_ENUM_RC_STARTED) ? "start" : "stop");
269	if (event_data->enumeration_status)
270	ioc_info(mrioc, "enumeration_status(0x%08x)\n",
271	le32_to_cpu(event_data->enumeration_status));
272	return;
273	}
274	case MPI3_EVENT_PREPARE_FOR_RESET:
275	desc = "Prepare For Reset";
276	break;
277	}
278
279	if (!desc)
280	return;
281
282	ioc_info(mrioc, "%s\n", desc);
283	}
284
285	static void mpi3mr_handle_events(struct mpi3mr_ioc *mrioc,
286	struct mpi3_default_reply *def_reply)
287	{
288	struct mpi3_event_notification_reply *event_reply =
289	(struct mpi3_event_notification_reply *)def_reply;
290
291	mrioc->change_count = le16_to_cpu(event_reply->ioc_change_count);
292	mpi3mr_print_event_data(mrioc, event_reply);
293	mpi3mr_os_handle_events(mrioc, event_reply);
294	}
295
296	static struct mpi3mr_drv_cmd *
297	mpi3mr_get_drv_cmd(struct mpi3mr_ioc *mrioc, u16 host_tag,
298	struct mpi3_default_reply *def_reply)
299	{
300	u16 idx;
301
302	switch (host_tag) {
303	case MPI3MR_HOSTTAG_INITCMDS:
304	return &mrioc->init_cmds;
305	case MPI3MR_HOSTTAG_CFG_CMDS:
306	return &mrioc->cfg_cmds;
307	case MPI3MR_HOSTTAG_BSG_CMDS:
308	return &mrioc->bsg_cmds;
309	case MPI3MR_HOSTTAG_BLK_TMS:
310	return &mrioc->host_tm_cmds;
311	case MPI3MR_HOSTTAG_PEL_ABORT:
312	return &mrioc->pel_abort_cmd;
313	case MPI3MR_HOSTTAG_PEL_WAIT:
314	return &mrioc->pel_cmds;
315	case MPI3MR_HOSTTAG_TRANSPORT_CMDS:
316	return &mrioc->transport_cmds;
317	case MPI3MR_HOSTTAG_INVALID:
318	if (def_reply && def_reply->function ==
319	MPI3_FUNCTION_EVENT_NOTIFICATION)
320	mpi3mr_handle_events(mrioc, def_reply);
321	return NULL;
322	default:
323	break;
324	}
325	if (host_tag >= MPI3MR_HOSTTAG_DEVRMCMD_MIN &&
326	host_tag <= MPI3MR_HOSTTAG_DEVRMCMD_MAX) {
327	idx = host_tag - MPI3MR_HOSTTAG_DEVRMCMD_MIN;
328	return &mrioc->dev_rmhs_cmds[idx];
329	}
330
331	if (host_tag >= MPI3MR_HOSTTAG_EVTACKCMD_MIN &&
332	host_tag <= MPI3MR_HOSTTAG_EVTACKCMD_MAX) {
333	idx = host_tag - MPI3MR_HOSTTAG_EVTACKCMD_MIN;
334	return &mrioc->evtack_cmds[idx];
335	}
336
337	return NULL;
338	}
339
340	static void mpi3mr_process_admin_reply_desc(struct mpi3mr_ioc *mrioc,
341	struct mpi3_default_reply_descriptor *reply_desc, u64 *reply_dma)
342	{
343	u16 reply_desc_type, host_tag = 0;
344	u16 ioc_status = MPI3_IOCSTATUS_SUCCESS;
345	u32 ioc_loginfo = 0;
346	struct mpi3_status_reply_descriptor *status_desc;
347	struct mpi3_address_reply_descriptor *addr_desc;
348	struct mpi3_success_reply_descriptor *success_desc;
349	struct mpi3_default_reply *def_reply = NULL;
350	struct mpi3mr_drv_cmd *cmdptr = NULL;
351	struct mpi3_scsi_io_reply *scsi_reply;
352	u8 *sense_buf = NULL;
353
354	*reply_dma = 0;
355	reply_desc_type = le16_to_cpu(reply_desc->reply_flags) &
356	MPI3_REPLY_DESCRIPT_FLAGS_TYPE_MASK;
357	switch (reply_desc_type) {
358	case MPI3_REPLY_DESCRIPT_FLAGS_TYPE_STATUS:
359	status_desc = (struct mpi3_status_reply_descriptor *)reply_desc;
360	host_tag = le16_to_cpu(status_desc->host_tag);
361	ioc_status = le16_to_cpu(status_desc->ioc_status);
362	if (ioc_status &
363	MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_LOGINFOAVAIL)
364	ioc_loginfo = le32_to_cpu(status_desc->ioc_log_info);
365	ioc_status &= MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_STATUS_MASK;
366	break;
367	case MPI3_REPLY_DESCRIPT_FLAGS_TYPE_ADDRESS_REPLY:
368	addr_desc = (struct mpi3_address_reply_descriptor *)reply_desc;
369	*reply_dma = le64_to_cpu(addr_desc->reply_frame_address);
370	def_reply = mpi3mr_get_reply_virt_addr(mrioc, phys_addr: *reply_dma);
371	if (!def_reply)
372	goto out;
373	host_tag = le16_to_cpu(def_reply->host_tag);
374	ioc_status = le16_to_cpu(def_reply->ioc_status);
375	if (ioc_status &
376	MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_LOGINFOAVAIL)
377	ioc_loginfo = le32_to_cpu(def_reply->ioc_log_info);
378	ioc_status &= MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_STATUS_MASK;
379	if (def_reply->function == MPI3_FUNCTION_SCSI_IO) {
380	scsi_reply = (struct mpi3_scsi_io_reply *)def_reply;
381	sense_buf = mpi3mr_get_sensebuf_virt_addr(mrioc,
382	le64_to_cpu(scsi_reply->sense_data_buffer_address));
383	}
384	break;
385	case MPI3_REPLY_DESCRIPT_FLAGS_TYPE_SUCCESS:
386	success_desc = (struct mpi3_success_reply_descriptor *)reply_desc;
387	host_tag = le16_to_cpu(success_desc->host_tag);
388	break;
389	default:
390	break;
391	}
392
393	cmdptr = mpi3mr_get_drv_cmd(mrioc, host_tag, def_reply);
394	if (cmdptr) {
395	if (cmdptr->state & MPI3MR_CMD_PENDING) {
396	cmdptr->state |= MPI3MR_CMD_COMPLETE;
397	cmdptr->ioc_loginfo = ioc_loginfo;
398	cmdptr->ioc_status = ioc_status;
399	cmdptr->state &= ~MPI3MR_CMD_PENDING;
400	if (def_reply) {
401	cmdptr->state |= MPI3MR_CMD_REPLY_VALID;
402	memcpy((u8 *)cmdptr->reply, (u8 *)def_reply,
403	mrioc->reply_sz);
404	}
405	if (sense_buf && cmdptr->sensebuf) {
406	cmdptr->is_sense = 1;
407	memcpy(cmdptr->sensebuf, sense_buf,
408	MPI3MR_SENSE_BUF_SZ);
409	}
410	if (cmdptr->is_waiting) {
411	complete(&cmdptr->done);
412	cmdptr->is_waiting = 0;
413	} else if (cmdptr->callback)
414	cmdptr->callback(mrioc, cmdptr);
415	}
416	}
417	out:
418	if (sense_buf)
419	mpi3mr_repost_sense_buf(mrioc,
420	le64_to_cpu(scsi_reply->sense_data_buffer_address));
421	}
422
423	int mpi3mr_process_admin_reply_q(struct mpi3mr_ioc *mrioc)
424	{
425	u32 exp_phase = mrioc->admin_reply_ephase;
426	u32 admin_reply_ci = mrioc->admin_reply_ci;
427	u32 num_admin_replies = 0;
428	u64 reply_dma = 0;
429	struct mpi3_default_reply_descriptor *reply_desc;
430
431	if (!atomic_add_unless(v: &mrioc->admin_reply_q_in_use, a: 1, u: 1))
432	return 0;
433
434	reply_desc = (struct mpi3_default_reply_descriptor *)mrioc->admin_reply_base +
435	admin_reply_ci;
436
437	if ((le16_to_cpu(reply_desc->reply_flags) &
438	MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase) {
439	atomic_dec(v: &mrioc->admin_reply_q_in_use);
440	return 0;
441	}
442
443	do {
444	if (mrioc->unrecoverable)
445	break;
446
447	mrioc->admin_req_ci = le16_to_cpu(reply_desc->request_queue_ci);
448	mpi3mr_process_admin_reply_desc(mrioc, reply_desc, reply_dma: &reply_dma);
449	if (reply_dma)
450	mpi3mr_repost_reply_buf(mrioc, reply_dma);
451	num_admin_replies++;
452	if (++admin_reply_ci == mrioc->num_admin_replies) {
453	admin_reply_ci = 0;
454	exp_phase ^= 1;
455	}
456	reply_desc =
457	(struct mpi3_default_reply_descriptor *)mrioc->admin_reply_base +
458	admin_reply_ci;
459	if ((le16_to_cpu(reply_desc->reply_flags) &
460	MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase)
461	break;
462	} while (1);
463
464	writel(val: admin_reply_ci, addr: &mrioc->sysif_regs->admin_reply_queue_ci);
465	mrioc->admin_reply_ci = admin_reply_ci;
466	mrioc->admin_reply_ephase = exp_phase;
467	atomic_dec(v: &mrioc->admin_reply_q_in_use);
468
469	return num_admin_replies;
470	}
471
472	/**
473	* mpi3mr_get_reply_desc - get reply descriptor frame corresponding to
474	* queue's consumer index from operational reply descriptor queue.
475	* @op_reply_q: op_reply_qinfo object
476	* @reply_ci: operational reply descriptor's queue consumer index
477	*
478	* Returns: reply descriptor frame address
479	*/
480	static inline struct mpi3_default_reply_descriptor *
481	mpi3mr_get_reply_desc(struct op_reply_qinfo *op_reply_q, u32 reply_ci)
482	{
483	void *segment_base_addr;
484	struct segments *segments = op_reply_q->q_segments;
485	struct mpi3_default_reply_descriptor *reply_desc = NULL;
486
487	segment_base_addr =
488	segments[reply_ci / op_reply_q->segment_qd].segment;
489	reply_desc = (struct mpi3_default_reply_descriptor *)segment_base_addr +
490	(reply_ci % op_reply_q->segment_qd);
491	return reply_desc;
492	}
493
494	/**
495	* mpi3mr_process_op_reply_q - Operational reply queue handler
496	* @mrioc: Adapter instance reference
497	* @op_reply_q: Operational reply queue info
498	*
499	* Checks the specific operational reply queue and drains the
500	* reply queue entries until the queue is empty and process the
501	* individual reply descriptors.
502	*
503	* Return: 0 if queue is already processed,or number of reply
504	* descriptors processed.
505	*/
506	int mpi3mr_process_op_reply_q(struct mpi3mr_ioc *mrioc,
507	struct op_reply_qinfo *op_reply_q)
508	{
509	struct op_req_qinfo *op_req_q;
510	u32 exp_phase;
511	u32 reply_ci;
512	u32 num_op_reply = 0;
513	u64 reply_dma = 0;
514	struct mpi3_default_reply_descriptor *reply_desc;
515	u16 req_q_idx = 0, reply_qidx;
516
517	reply_qidx = op_reply_q->qid - 1;
518
519	if (!atomic_add_unless(v: &op_reply_q->in_use, a: 1, u: 1))
520	return 0;
521
522	exp_phase = op_reply_q->ephase;
523	reply_ci = op_reply_q->ci;
524
525	reply_desc = mpi3mr_get_reply_desc(op_reply_q, reply_ci);
526	if ((le16_to_cpu(reply_desc->reply_flags) &
527	MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase) {
528	atomic_dec(v: &op_reply_q->in_use);
529	return 0;
530	}
531
532	do {
533	if (mrioc->unrecoverable)
534	break;
535
536	req_q_idx = le16_to_cpu(reply_desc->request_queue_id) - 1;
537	op_req_q = &mrioc->req_qinfo[req_q_idx];
538
539	WRITE_ONCE(op_req_q->ci, le16_to_cpu(reply_desc->request_queue_ci));
540	mpi3mr_process_op_reply_desc(mrioc, reply_desc, reply_dma: &reply_dma,
541	qidx: reply_qidx);
542	atomic_dec(v: &op_reply_q->pend_ios);
543	if (reply_dma)
544	mpi3mr_repost_reply_buf(mrioc, reply_dma);
545	num_op_reply++;
546
547	if (++reply_ci == op_reply_q->num_replies) {
548	reply_ci = 0;
549	exp_phase ^= 1;
550	}
551
552	reply_desc = mpi3mr_get_reply_desc(op_reply_q, reply_ci);
553
554	if ((le16_to_cpu(reply_desc->reply_flags) &
555	MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase)
556	break;
557	#ifndef CONFIG_PREEMPT_RT
558	/*
559	* Exit completion loop to avoid CPU lockup
560	* Ensure remaining completion happens from threaded ISR.
561	*/
562	if (num_op_reply > mrioc->max_host_ios) {
563	op_reply_q->enable_irq_poll = true;
564	break;
565	}
566	#endif
567	} while (1);
568
569	writel(val: reply_ci,
570	addr: &mrioc->sysif_regs->oper_queue_indexes[reply_qidx].consumer_index);
571	op_reply_q->ci = reply_ci;
572	op_reply_q->ephase = exp_phase;
573
574	atomic_dec(v: &op_reply_q->in_use);
575	return num_op_reply;
576	}
577
578	/**
579	* mpi3mr_blk_mq_poll - Operational reply queue handler
580	* @shost: SCSI Host reference
581	* @queue_num: Request queue number (w.r.t OS it is hardware context number)
582	*
583	* Checks the specific operational reply queue and drains the
584	* reply queue entries until the queue is empty and process the
585	* individual reply descriptors.
586	*
587	* Return: 0 if queue is already processed,or number of reply
588	* descriptors processed.
589	*/
590	int mpi3mr_blk_mq_poll(struct Scsi_Host *shost, unsigned int queue_num)
591	{
592	int num_entries = 0;
593	struct mpi3mr_ioc *mrioc;
594
595	mrioc = (struct mpi3mr_ioc *)shost->hostdata;
596
597	if ((mrioc->reset_in_progress || mrioc->prepare_for_reset ||
598	mrioc->unrecoverable))
599	return 0;
600
601	num_entries = mpi3mr_process_op_reply_q(mrioc,
602	op_reply_q: &mrioc->op_reply_qinfo[queue_num]);
603
604	return num_entries;
605	}
606
607	static irqreturn_t mpi3mr_isr_primary(int irq, void *privdata)
608	{
609	struct mpi3mr_intr_info *intr_info = privdata;
610	struct mpi3mr_ioc *mrioc;
611	u16 midx;
612	u32 num_admin_replies = 0, num_op_reply = 0;
613
614	if (!intr_info)
615	return IRQ_NONE;
616
617	mrioc = intr_info->mrioc;
618
619	if (!mrioc->intr_enabled)
620	return IRQ_NONE;
621
622	midx = intr_info->msix_index;
623
624	if (!midx)
625	num_admin_replies = mpi3mr_process_admin_reply_q(mrioc);
626	if (intr_info->op_reply_q)
627	num_op_reply = mpi3mr_process_op_reply_q(mrioc,
628	op_reply_q: intr_info->op_reply_q);
629
630	if (num_admin_replies || num_op_reply)
631	return IRQ_HANDLED;
632	else
633	return IRQ_NONE;
634	}
635
636	#ifndef CONFIG_PREEMPT_RT
637
638	static irqreturn_t mpi3mr_isr(int irq, void *privdata)
639	{
640	struct mpi3mr_intr_info *intr_info = privdata;
641	int ret;
642
643	if (!intr_info)
644	return IRQ_NONE;
645
646	/* Call primary ISR routine */
647	ret = mpi3mr_isr_primary(irq, privdata);
648
649	/*
650	* If more IOs are expected, schedule IRQ polling thread.
651	* Otherwise exit from ISR.
652	*/
653	if (!intr_info->op_reply_q)
654	return ret;
655
656	if (!intr_info->op_reply_q->enable_irq_poll ||
657	!atomic_read(v: &intr_info->op_reply_q->pend_ios))
658	return ret;
659
660	disable_irq_nosync(irq: intr_info->os_irq);
661
662	return IRQ_WAKE_THREAD;
663	}
664
665	/**
666	* mpi3mr_isr_poll - Reply queue polling routine
667	* @irq: IRQ
668	* @privdata: Interrupt info
669	*
670	* poll for pending I/O completions in a loop until pending I/Os
671	* present or controller queue depth I/Os are processed.
672	*
673	* Return: IRQ_NONE or IRQ_HANDLED
674	*/
675	static irqreturn_t mpi3mr_isr_poll(int irq, void *privdata)
676	{
677	struct mpi3mr_intr_info *intr_info = privdata;
678	struct mpi3mr_ioc *mrioc;
679	u16 midx;
680	u32 num_op_reply = 0;
681
682	if (!intr_info || !intr_info->op_reply_q)
683	return IRQ_NONE;
684
685	mrioc = intr_info->mrioc;
686	midx = intr_info->msix_index;
687
688	/* Poll for pending IOs completions */
689	do {
690	if (!mrioc->intr_enabled || mrioc->unrecoverable)
691	break;
692
693	if (!midx)
694	mpi3mr_process_admin_reply_q(mrioc);
695	if (intr_info->op_reply_q)
696	num_op_reply +=
697	mpi3mr_process_op_reply_q(mrioc,
698	op_reply_q: intr_info->op_reply_q);
699
700	usleep_range(MPI3MR_IRQ_POLL_SLEEP, max: 10 * MPI3MR_IRQ_POLL_SLEEP);
701
702	} while (atomic_read(v: &intr_info->op_reply_q->pend_ios) &&
703	(num_op_reply < mrioc->max_host_ios));
704
705	intr_info->op_reply_q->enable_irq_poll = false;
706	enable_irq(irq: intr_info->os_irq);
707
708	return IRQ_HANDLED;
709	}
710
711	#endif
712
713	/**
714	* mpi3mr_request_irq - Request IRQ and register ISR
715	* @mrioc: Adapter instance reference
716	* @index: IRQ vector index
717	*
718	* Request threaded ISR with primary ISR and secondary
719	*
720	* Return: 0 on success and non zero on failures.
721	*/
722	static inline int mpi3mr_request_irq(struct mpi3mr_ioc *mrioc, u16 index)
723	{
724	struct pci_dev *pdev = mrioc->pdev;
725	struct mpi3mr_intr_info *intr_info = mrioc->intr_info + index;
726	int retval = 0;
727
728	intr_info->mrioc = mrioc;
729	intr_info->msix_index = index;
730	intr_info->op_reply_q = NULL;
731
732	snprintf(buf: intr_info->name, MPI3MR_NAME_LENGTH, fmt: "%s%d-msix%d",
733	mrioc->driver_name, mrioc->id, index);
734
735	#ifndef CONFIG_PREEMPT_RT
736	retval = request_threaded_irq(irq: pci_irq_vector(dev: pdev, nr: index), handler: mpi3mr_isr,
737	thread_fn: mpi3mr_isr_poll, IRQF_SHARED, name: intr_info->name, dev: intr_info);
738	#else
739	retval = request_threaded_irq(pci_irq_vector(pdev, index), mpi3mr_isr_primary,
740	NULL, IRQF_SHARED, intr_info->name, intr_info);
741	#endif
742	if (retval) {
743	ioc_err(mrioc, "%s: Unable to allocate interrupt %d!\n",
744	intr_info->name, pci_irq_vector(pdev, index));
745	return retval;
746	}
747
748	intr_info->os_irq = pci_irq_vector(dev: pdev, nr: index);
749	return retval;
750	}
751
752	static void mpi3mr_calc_poll_queues(struct mpi3mr_ioc *mrioc, u16 max_vectors)
753	{
754	if (!mrioc->requested_poll_qcount)
755	return;
756
757	/* Reserved for Admin and Default Queue */
758	if (max_vectors > 2 &&
759	(mrioc->requested_poll_qcount < max_vectors - 2)) {
760	ioc_info(mrioc,
761	"enabled polled queues (%d) msix (%d)\n",
762	mrioc->requested_poll_qcount, max_vectors);
763	} else {
764	ioc_info(mrioc,
765	"disabled polled queues (%d) msix (%d) because of no resources for default queue\n",
766	mrioc->requested_poll_qcount, max_vectors);
767	mrioc->requested_poll_qcount = 0;
768	}
769	}
770
771	/**
772	* mpi3mr_setup_isr - Setup ISR for the controller
773	* @mrioc: Adapter instance reference
774	* @setup_one: Request one IRQ or more
775	*
776	* Allocate IRQ vectors and call mpi3mr_request_irq to setup ISR
777	*
778	* Return: 0 on success and non zero on failures.
779	*/
780	static int mpi3mr_setup_isr(struct mpi3mr_ioc *mrioc, u8 setup_one)
781	{
782	unsigned int irq_flags = PCI_IRQ_MSIX;
783	int max_vectors, min_vec;
784	int retval;
785	int i;
786	struct irq_affinity desc = { .pre_vectors = 1, .post_vectors = 1 };
787
788	if (mrioc->is_intr_info_set)
789	return 0;
790
791	mpi3mr_cleanup_isr(mrioc);
792
793	if (setup_one || reset_devices) {
794	max_vectors = 1;
795	retval = pci_alloc_irq_vectors(dev: mrioc->pdev,
796	min_vecs: 1, max_vecs: max_vectors, flags: irq_flags);
797	if (retval < 0) {
798	ioc_err(mrioc, "cannot allocate irq vectors, ret %d\n",
799	retval);
800	goto out_failed;
801	}
802	} else {
803	max_vectors =
804	min_t(int, mrioc->cpu_count + 1 +
805	mrioc->requested_poll_qcount, mrioc->msix_count);
806
807	mpi3mr_calc_poll_queues(mrioc, max_vectors);
808
809	ioc_info(mrioc,
810	"MSI-X vectors supported: %d, no of cores: %d,",
811	mrioc->msix_count, mrioc->cpu_count);
812	ioc_info(mrioc,
813	"MSI-x vectors requested: %d poll_queues %d\n",
814	max_vectors, mrioc->requested_poll_qcount);
815
816	desc.post_vectors = mrioc->requested_poll_qcount;
817	min_vec = desc.pre_vectors + desc.post_vectors;
818	irq_flags |= PCI_IRQ_AFFINITY | PCI_IRQ_ALL_TYPES;
819
820	retval = pci_alloc_irq_vectors_affinity(dev: mrioc->pdev,
821	min_vecs: min_vec, max_vecs: max_vectors, flags: irq_flags, affd: &desc);
822
823	if (retval < 0) {
824	ioc_err(mrioc, "cannot allocate irq vectors, ret %d\n",
825	retval);
826	goto out_failed;
827	}
828
829
830	/*
831	* If only one MSI-x is allocated, then MSI-x 0 will be shared
832	* between Admin queue and operational queue
833	*/
834	if (retval == min_vec)
835	mrioc->op_reply_q_offset = 0;
836	else if (retval != (max_vectors)) {
837	ioc_info(mrioc,
838	"allocated vectors (%d) are less than configured (%d)\n",
839	retval, max_vectors);
840	}
841
842	max_vectors = retval;
843	mrioc->op_reply_q_offset = (max_vectors > 1) ? 1 : 0;
844
845	mpi3mr_calc_poll_queues(mrioc, max_vectors);
846
847	}
848
849	mrioc->intr_info = kzalloc(size: sizeof(struct mpi3mr_intr_info) * max_vectors,
850	GFP_KERNEL);
851	if (!mrioc->intr_info) {
852	retval = -ENOMEM;
853	pci_free_irq_vectors(dev: mrioc->pdev);
854	goto out_failed;
855	}
856	for (i = 0; i < max_vectors; i++) {
857	retval = mpi3mr_request_irq(mrioc, index: i);
858	if (retval) {
859	mrioc->intr_info_count = i;
860	goto out_failed;
861	}
862	}
863	if (reset_devices || !setup_one)
864	mrioc->is_intr_info_set = true;
865	mrioc->intr_info_count = max_vectors;
866	mpi3mr_ioc_enable_intr(mrioc);
867	return 0;
868
869	out_failed:
870	mpi3mr_cleanup_isr(mrioc);
871
872	return retval;
873	}
874
875	static const struct {
876	enum mpi3mr_iocstate value;
877	char *name;
878	} mrioc_states[] = {
879	{ MRIOC_STATE_READY, "ready" },
880	{ MRIOC_STATE_FAULT, "fault" },
881	{ MRIOC_STATE_RESET, "reset" },
882	{ MRIOC_STATE_BECOMING_READY, "becoming ready" },
883	{ MRIOC_STATE_RESET_REQUESTED, "reset requested" },
884	{ MRIOC_STATE_UNRECOVERABLE, "unrecoverable error" },
885	};
886
887	static const char *mpi3mr_iocstate_name(enum mpi3mr_iocstate mrioc_state)
888	{
889	int i;
890	char *name = NULL;
891
892	for (i = 0; i < ARRAY_SIZE(mrioc_states); i++) {
893	if (mrioc_states[i].value == mrioc_state) {
894	name = mrioc_states[i].name;
895	break;
896	}
897	}
898	return name;
899	}
900
901	/* Reset reason to name mapper structure*/
902	static const struct {
903	enum mpi3mr_reset_reason value;
904	char *name;
905	} mpi3mr_reset_reason_codes[] = {
906	{ MPI3MR_RESET_FROM_BRINGUP, "timeout in bringup" },
907	{ MPI3MR_RESET_FROM_FAULT_WATCH, "fault" },
908	{ MPI3MR_RESET_FROM_APP, "application invocation" },
909	{ MPI3MR_RESET_FROM_EH_HOS, "error handling" },
910	{ MPI3MR_RESET_FROM_TM_TIMEOUT, "TM timeout" },
911	{ MPI3MR_RESET_FROM_APP_TIMEOUT, "application command timeout" },
912	{ MPI3MR_RESET_FROM_MUR_FAILURE, "MUR failure" },
913	{ MPI3MR_RESET_FROM_CTLR_CLEANUP, "timeout in controller cleanup" },
914	{ MPI3MR_RESET_FROM_CIACTIV_FAULT, "component image activation fault" },
915	{ MPI3MR_RESET_FROM_PE_TIMEOUT, "port enable timeout" },
916	{ MPI3MR_RESET_FROM_TSU_TIMEOUT, "time stamp update timeout" },
917	{ MPI3MR_RESET_FROM_DELREQQ_TIMEOUT, "delete request queue timeout" },
918	{ MPI3MR_RESET_FROM_DELREPQ_TIMEOUT, "delete reply queue timeout" },
919	{
920	MPI3MR_RESET_FROM_CREATEREPQ_TIMEOUT,
921	"create request queue timeout"
922	},
923	{
924	MPI3MR_RESET_FROM_CREATEREQQ_TIMEOUT,
925	"create reply queue timeout"
926	},
927	{ MPI3MR_RESET_FROM_IOCFACTS_TIMEOUT, "IOC facts timeout" },
928	{ MPI3MR_RESET_FROM_IOCINIT_TIMEOUT, "IOC init timeout" },
929	{ MPI3MR_RESET_FROM_EVTNOTIFY_TIMEOUT, "event notify timeout" },
930	{ MPI3MR_RESET_FROM_EVTACK_TIMEOUT, "event acknowledgment timeout" },
931	{
932	MPI3MR_RESET_FROM_CIACTVRST_TIMER,
933	"component image activation timeout"
934	},
935	{
936	MPI3MR_RESET_FROM_GETPKGVER_TIMEOUT,
937	"get package version timeout"
938	},
939	{ MPI3MR_RESET_FROM_SYSFS, "sysfs invocation" },
940	{ MPI3MR_RESET_FROM_SYSFS_TIMEOUT, "sysfs TM timeout" },
941	{ MPI3MR_RESET_FROM_FIRMWARE, "firmware asynchronous reset" },
942	{ MPI3MR_RESET_FROM_CFG_REQ_TIMEOUT, "configuration request timeout"},
943	{ MPI3MR_RESET_FROM_SAS_TRANSPORT_TIMEOUT, "timeout of a SAS transport layer request" },
944	};
945
946	/**
947	* mpi3mr_reset_rc_name - get reset reason code name
948	* @reason_code: reset reason code value
949	*
950	* Map reset reason to an NULL terminated ASCII string
951	*
952	* Return: name corresponding to reset reason value or NULL.
953	*/
954	static const char *mpi3mr_reset_rc_name(enum mpi3mr_reset_reason reason_code)
955	{
956	int i;
957	char *name = NULL;
958
959	for (i = 0; i < ARRAY_SIZE(mpi3mr_reset_reason_codes); i++) {
960	if (mpi3mr_reset_reason_codes[i].value == reason_code) {
961	name = mpi3mr_reset_reason_codes[i].name;
962	break;
963	}
964	}
965	return name;
966	}
967
968	/* Reset type to name mapper structure*/
969	static const struct {
970	u16 reset_type;
971	char *name;
972	} mpi3mr_reset_types[] = {
973	{ MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET, "soft" },
974	{ MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT, "diag fault" },
975	};
976
977	/**
978	* mpi3mr_reset_type_name - get reset type name
979	* @reset_type: reset type value
980	*
981	* Map reset type to an NULL terminated ASCII string
982	*
983	* Return: name corresponding to reset type value or NULL.
984	*/
985	static const char *mpi3mr_reset_type_name(u16 reset_type)
986	{
987	int i;
988	char *name = NULL;
989
990	for (i = 0; i < ARRAY_SIZE(mpi3mr_reset_types); i++) {
991	if (mpi3mr_reset_types[i].reset_type == reset_type) {
992	name = mpi3mr_reset_types[i].name;
993	break;
994	}
995	}
996	return name;
997	}
998
999	/**
1000	* mpi3mr_print_fault_info - Display fault information
1001	* @mrioc: Adapter instance reference
1002	*
1003	* Display the controller fault information if there is a
1004	* controller fault.
1005	*
1006	* Return: Nothing.
1007	*/
1008	void mpi3mr_print_fault_info(struct mpi3mr_ioc *mrioc)
1009	{
1010	u32 ioc_status, code, code1, code2, code3;
1011
1012	ioc_status = readl(addr: &mrioc->sysif_regs->ioc_status);
1013
1014	if (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT) {
1015	code = readl(addr: &mrioc->sysif_regs->fault);
1016	code1 = readl(addr: &mrioc->sysif_regs->fault_info[0]);
1017	code2 = readl(addr: &mrioc->sysif_regs->fault_info[1]);
1018	code3 = readl(addr: &mrioc->sysif_regs->fault_info[2]);
1019
1020	ioc_info(mrioc,
1021	"fault code(0x%08X): Additional code: (0x%08X:0x%08X:0x%08X)\n",
1022	code, code1, code2, code3);
1023	}
1024	}
1025
1026	/**
1027	* mpi3mr_get_iocstate - Get IOC State
1028	* @mrioc: Adapter instance reference
1029	*
1030	* Return a proper IOC state enum based on the IOC status and
1031	* IOC configuration and unrcoverable state of the controller.
1032	*
1033	* Return: Current IOC state.
1034	*/
1035	enum mpi3mr_iocstate mpi3mr_get_iocstate(struct mpi3mr_ioc *mrioc)
1036	{
1037	u32 ioc_status, ioc_config;
1038	u8 ready, enabled;
1039
1040	ioc_status = readl(addr: &mrioc->sysif_regs->ioc_status);
1041	ioc_config = readl(addr: &mrioc->sysif_regs->ioc_configuration);
1042
1043	if (mrioc->unrecoverable)
1044	return MRIOC_STATE_UNRECOVERABLE;
1045	if (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT)
1046	return MRIOC_STATE_FAULT;
1047
1048	ready = (ioc_status & MPI3_SYSIF_IOC_STATUS_READY);
1049	enabled = (ioc_config & MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC);
1050
1051	if (ready && enabled)
1052	return MRIOC_STATE_READY;
1053	if ((!ready) && (!enabled))
1054	return MRIOC_STATE_RESET;
1055	if ((!ready) && (enabled))
1056	return MRIOC_STATE_BECOMING_READY;
1057
1058	return MRIOC_STATE_RESET_REQUESTED;
1059	}
1060
1061	/**
1062	* mpi3mr_free_ioctl_dma_memory - free memory for ioctl dma
1063	* @mrioc: Adapter instance reference
1064	*
1065	* Free the DMA memory allocated for IOCTL handling purpose.
1066	*
1067	* Return: None
1068	*/
1069	static void mpi3mr_free_ioctl_dma_memory(struct mpi3mr_ioc *mrioc)
1070	{
1071	struct dma_memory_desc *mem_desc;
1072	u16 i;
1073
1074	if (!mrioc->ioctl_dma_pool)
1075	return;
1076
1077	for (i = 0; i < MPI3MR_NUM_IOCTL_SGE; i++) {
1078	mem_desc = &mrioc->ioctl_sge[i];
1079	if (mem_desc->addr) {
1080	dma_pool_free(pool: mrioc->ioctl_dma_pool,
1081	vaddr: mem_desc->addr,
1082	addr: mem_desc->dma_addr);
1083	mem_desc->addr = NULL;
1084	}
1085	}
1086	dma_pool_destroy(pool: mrioc->ioctl_dma_pool);
1087	mrioc->ioctl_dma_pool = NULL;
1088	mem_desc = &mrioc->ioctl_chain_sge;
1089
1090	if (mem_desc->addr) {
1091	dma_free_coherent(dev: &mrioc->pdev->dev, size: mem_desc->size,
1092	cpu_addr: mem_desc->addr, dma_handle: mem_desc->dma_addr);
1093	mem_desc->addr = NULL;
1094	}
1095	mem_desc = &mrioc->ioctl_resp_sge;
1096	if (mem_desc->addr) {
1097	dma_free_coherent(dev: &mrioc->pdev->dev, size: mem_desc->size,
1098	cpu_addr: mem_desc->addr, dma_handle: mem_desc->dma_addr);
1099	mem_desc->addr = NULL;
1100	}
1101
1102	mrioc->ioctl_sges_allocated = false;
1103	}
1104
1105	/**
1106	* mpi3mr_alloc_ioctl_dma_memory - Alloc memory for ioctl dma
1107	* @mrioc: Adapter instance reference
1108	*
1109	* This function allocates dmaable memory required to handle the
1110	* application issued MPI3 IOCTL requests.
1111	*
1112	* Return: None
1113	*/
1114	static void mpi3mr_alloc_ioctl_dma_memory(struct mpi3mr_ioc *mrioc)
1115
1116	{
1117	struct dma_memory_desc *mem_desc;
1118	u16 i;
1119
1120	mrioc->ioctl_dma_pool = dma_pool_create(name: "ioctl dma pool",
1121	dev: &mrioc->pdev->dev,
1122	MPI3MR_IOCTL_SGE_SIZE,
1123	MPI3MR_PAGE_SIZE_4K, allocation: 0);
1124
1125	if (!mrioc->ioctl_dma_pool) {
1126	ioc_err(mrioc, "ioctl_dma_pool: dma_pool_create failed\n");
1127	goto out_failed;
1128	}
1129
1130	for (i = 0; i < MPI3MR_NUM_IOCTL_SGE; i++) {
1131	mem_desc = &mrioc->ioctl_sge[i];
1132	mem_desc->size = MPI3MR_IOCTL_SGE_SIZE;
1133	mem_desc->addr = dma_pool_zalloc(pool: mrioc->ioctl_dma_pool,
1134	GFP_KERNEL,
1135	handle: &mem_desc->dma_addr);
1136	if (!mem_desc->addr)
1137	goto out_failed;
1138	}
1139
1140	mem_desc = &mrioc->ioctl_chain_sge;
1141	mem_desc->size = MPI3MR_PAGE_SIZE_4K;
1142	mem_desc->addr = dma_alloc_coherent(dev: &mrioc->pdev->dev,
1143	size: mem_desc->size,
1144	dma_handle: &mem_desc->dma_addr,
1145	GFP_KERNEL);
1146	if (!mem_desc->addr)
1147	goto out_failed;
1148
1149	mem_desc = &mrioc->ioctl_resp_sge;
1150	mem_desc->size = MPI3MR_PAGE_SIZE_4K;
1151	mem_desc->addr = dma_alloc_coherent(dev: &mrioc->pdev->dev,
1152	size: mem_desc->size,
1153	dma_handle: &mem_desc->dma_addr,
1154	GFP_KERNEL);
1155	if (!mem_desc->addr)
1156	goto out_failed;
1157
1158	mrioc->ioctl_sges_allocated = true;
1159
1160	return;
1161	out_failed:
1162	ioc_warn(mrioc, "cannot allocate DMA memory for the mpt commands\n"
1163	"from the applications, application interface for MPT command is disabled\n");
1164	mpi3mr_free_ioctl_dma_memory(mrioc);
1165	}
1166
1167	/**
1168	* mpi3mr_clear_reset_history - clear reset history
1169	* @mrioc: Adapter instance reference
1170	*
1171	* Write the reset history bit in IOC status to clear the bit,
1172	* if it is already set.
1173	*
1174	* Return: Nothing.
1175	*/
1176	static inline void mpi3mr_clear_reset_history(struct mpi3mr_ioc *mrioc)
1177	{
1178	u32 ioc_status;
1179
1180	ioc_status = readl(addr: &mrioc->sysif_regs->ioc_status);
1181	if (ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY)
1182	writel(val: ioc_status, addr: &mrioc->sysif_regs->ioc_status);
1183	}
1184
1185	/**
1186	* mpi3mr_issue_and_process_mur - Message unit Reset handler
1187	* @mrioc: Adapter instance reference
1188	* @reset_reason: Reset reason code
1189	*
1190	* Issue Message unit Reset to the controller and wait for it to
1191	* be complete.
1192	*
1193	* Return: 0 on success, -1 on failure.
1194	*/
1195	static int mpi3mr_issue_and_process_mur(struct mpi3mr_ioc *mrioc,
1196	u32 reset_reason)
1197	{
1198	u32 ioc_config, timeout, ioc_status;
1199	int retval = -1;
1200
1201	ioc_info(mrioc, "Issuing Message unit Reset(MUR)\n");
1202	if (mrioc->unrecoverable) {
1203	ioc_info(mrioc, "IOC is unrecoverable MUR not issued\n");
1204	return retval;
1205	}
1206	mpi3mr_clear_reset_history(mrioc);
1207	writel(val: reset_reason, addr: &mrioc->sysif_regs->scratchpad[0]);
1208	ioc_config = readl(addr: &mrioc->sysif_regs->ioc_configuration);
1209	ioc_config &= ~MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC;
1210	writel(val: ioc_config, addr: &mrioc->sysif_regs->ioc_configuration);
1211
1212	timeout = MPI3MR_MUR_TIMEOUT * 10;
1213	do {
1214	ioc_status = readl(addr: &mrioc->sysif_regs->ioc_status);
1215	if ((ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY)) {
1216	mpi3mr_clear_reset_history(mrioc);
1217	break;
1218	}
1219	if (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT) {
1220	mpi3mr_print_fault_info(mrioc);
1221	break;
1222	}
1223	msleep(msecs: 100);
1224	} while (--timeout);
1225
1226	ioc_config = readl(addr: &mrioc->sysif_regs->ioc_configuration);
1227	if (timeout && !((ioc_status & MPI3_SYSIF_IOC_STATUS_READY) ||
1228	(ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT) ||
1229	(ioc_config & MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC)))
1230	retval = 0;
1231
1232	ioc_info(mrioc, "Base IOC Sts/Config after %s MUR is (0x%x)/(0x%x)\n",
1233	(!retval) ? "successful" : "failed", ioc_status, ioc_config);
1234	return retval;
1235	}
1236
1237	/**
1238	* mpi3mr_revalidate_factsdata - validate IOCFacts parameters
1239	* during reset/resume
1240	* @mrioc: Adapter instance reference
1241	*
1242	* Return: zero if the new IOCFacts parameters value is compatible with
1243	* older values else return -EPERM
1244	*/
1245	static int
1246	mpi3mr_revalidate_factsdata(struct mpi3mr_ioc *mrioc)
1247	{
1248	unsigned long *removepend_bitmap;
1249
1250	if (mrioc->facts.reply_sz > mrioc->reply_sz) {
1251	ioc_err(mrioc,
1252	"cannot increase reply size from %d to %d\n",
1253	mrioc->reply_sz, mrioc->facts.reply_sz);
1254	return -EPERM;
1255	}
1256
1257	if (mrioc->facts.max_op_reply_q < mrioc->num_op_reply_q) {
1258	ioc_err(mrioc,
1259	"cannot reduce number of operational reply queues from %d to %d\n",
1260	mrioc->num_op_reply_q,
1261	mrioc->facts.max_op_reply_q);
1262	return -EPERM;
1263	}
1264
1265	if (mrioc->facts.max_op_req_q < mrioc->num_op_req_q) {
1266	ioc_err(mrioc,
1267	"cannot reduce number of operational request queues from %d to %d\n",
1268	mrioc->num_op_req_q, mrioc->facts.max_op_req_q);
1269	return -EPERM;
1270	}
1271
1272	if (mrioc->shost->max_sectors != (mrioc->facts.max_data_length / 512))
1273	ioc_err(mrioc, "Warning: The maximum data transfer length\n"
1274	"\tchanged after reset: previous(%d), new(%d),\n"
1275	"the driver cannot change this at run time\n",
1276	mrioc->shost->max_sectors * 512, mrioc->facts.max_data_length);
1277
1278	if ((mrioc->sas_transport_enabled) && (mrioc->facts.ioc_capabilities &
1279	MPI3_IOCFACTS_CAPABILITY_MULTIPATH_ENABLED))
1280	ioc_err(mrioc,
1281	"critical error: multipath capability is enabled at the\n"
1282	"\tcontroller while sas transport support is enabled at the\n"
1283	"\tdriver, please reboot the system or reload the driver\n");
1284
1285	if (mrioc->facts.max_devhandle > mrioc->dev_handle_bitmap_bits) {
1286	removepend_bitmap = bitmap_zalloc(nbits: mrioc->facts.max_devhandle,
1287	GFP_KERNEL);
1288	if (!removepend_bitmap) {
1289	ioc_err(mrioc,
1290	"failed to increase removepend_bitmap bits from %d to %d\n",
1291	mrioc->dev_handle_bitmap_bits,
1292	mrioc->facts.max_devhandle);
1293	return -EPERM;
1294	}
1295	bitmap_free(bitmap: mrioc->removepend_bitmap);
1296	mrioc->removepend_bitmap = removepend_bitmap;
1297	ioc_info(mrioc,
1298	"increased bits of dev_handle_bitmap from %d to %d\n",
1299	mrioc->dev_handle_bitmap_bits,
1300	mrioc->facts.max_devhandle);
1301	mrioc->dev_handle_bitmap_bits = mrioc->facts.max_devhandle;
1302	}
1303
1304	return 0;
1305	}
1306
1307	/**
1308	* mpi3mr_bring_ioc_ready - Bring controller to ready state
1309	* @mrioc: Adapter instance reference
1310	*
1311	* Set Enable IOC bit in IOC configuration register and wait for
1312	* the controller to become ready.
1313	*
1314	* Return: 0 on success, appropriate error on failure.
1315	*/
1316	static int mpi3mr_bring_ioc_ready(struct mpi3mr_ioc *mrioc)
1317	{
1318	u32 ioc_config, ioc_status, timeout, host_diagnostic;
1319	int retval = 0;
1320	enum mpi3mr_iocstate ioc_state;
1321	u64 base_info;
1322
1323	ioc_status = readl(addr: &mrioc->sysif_regs->ioc_status);
1324	ioc_config = readl(addr: &mrioc->sysif_regs->ioc_configuration);
1325	base_info = lo_hi_readq(addr: &mrioc->sysif_regs->ioc_information);
1326	ioc_info(mrioc, "ioc_status(0x%08x), ioc_config(0x%08x), ioc_info(0x%016llx) at the bringup\n",
1327	ioc_status, ioc_config, base_info);
1328
1329	/*The timeout value is in 2sec unit, changing it to seconds*/
1330	mrioc->ready_timeout =
1331	((base_info & MPI3_SYSIF_IOC_INFO_LOW_TIMEOUT_MASK) >>
1332	MPI3_SYSIF_IOC_INFO_LOW_TIMEOUT_SHIFT) * 2;
1333
1334	ioc_info(mrioc, "ready timeout: %d seconds\n", mrioc->ready_timeout);
1335
1336	ioc_state = mpi3mr_get_iocstate(mrioc);
1337	ioc_info(mrioc, "controller is in %s state during detection\n",
1338	mpi3mr_iocstate_name(ioc_state));
1339
1340	if (ioc_state == MRIOC_STATE_BECOMING_READY ||
1341	ioc_state == MRIOC_STATE_RESET_REQUESTED) {
1342	timeout = mrioc->ready_timeout * 10;
1343	do {
1344	msleep(msecs: 100);
1345	} while (--timeout);
1346
1347	if (!pci_device_is_present(pdev: mrioc->pdev)) {
1348	mrioc->unrecoverable = 1;
1349	ioc_err(mrioc,
1350	"controller is not present while waiting to reset\n");
1351	retval = -1;
1352	goto out_device_not_present;
1353	}
1354
1355	ioc_state = mpi3mr_get_iocstate(mrioc);
1356	ioc_info(mrioc,
1357	"controller is in %s state after waiting to reset\n",
1358	mpi3mr_iocstate_name(ioc_state));
1359	}
1360
1361	if (ioc_state == MRIOC_STATE_READY) {
1362	ioc_info(mrioc, "issuing message unit reset (MUR) to bring to reset state\n");
1363	retval = mpi3mr_issue_and_process_mur(mrioc,
1364	reset_reason: MPI3MR_RESET_FROM_BRINGUP);
1365	ioc_state = mpi3mr_get_iocstate(mrioc);
1366	if (retval)
1367	ioc_err(mrioc,
1368	"message unit reset failed with error %d current state %s\n",
1369	retval, mpi3mr_iocstate_name(ioc_state));
1370	}
1371	if (ioc_state != MRIOC_STATE_RESET) {
1372	if (ioc_state == MRIOC_STATE_FAULT) {
1373	timeout = MPI3_SYSIF_DIAG_SAVE_TIMEOUT * 10;
1374	mpi3mr_print_fault_info(mrioc);
1375	do {
1376	host_diagnostic =
1377	readl(addr: &mrioc->sysif_regs->host_diagnostic);
1378	if (!(host_diagnostic &
1379	MPI3_SYSIF_HOST_DIAG_SAVE_IN_PROGRESS))
1380	break;
1381	if (!pci_device_is_present(pdev: mrioc->pdev)) {
1382	mrioc->unrecoverable = 1;
1383	ioc_err(mrioc, "controller is not present at the bringup\n");
1384	goto out_device_not_present;
1385	}
1386	msleep(msecs: 100);
1387	} while (--timeout);
1388	}
1389	mpi3mr_print_fault_info(mrioc);
1390	ioc_info(mrioc, "issuing soft reset to bring to reset state\n");
1391	retval = mpi3mr_issue_reset(mrioc,
1392	MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET,
1393	reset_reason: MPI3MR_RESET_FROM_BRINGUP);
1394	if (retval) {
1395	ioc_err(mrioc,
1396	"soft reset failed with error %d\n", retval);
1397	goto out_failed;
1398	}
1399	}
1400	ioc_state = mpi3mr_get_iocstate(mrioc);
1401	if (ioc_state != MRIOC_STATE_RESET) {
1402	ioc_err(mrioc,
1403	"cannot bring controller to reset state, current state: %s\n",
1404	mpi3mr_iocstate_name(ioc_state));
1405	goto out_failed;
1406	}
1407	mpi3mr_clear_reset_history(mrioc);
1408	retval = mpi3mr_setup_admin_qpair(mrioc);
1409	if (retval) {
1410	ioc_err(mrioc, "failed to setup admin queues: error %d\n",
1411	retval);
1412	goto out_failed;
1413	}
1414
1415	ioc_info(mrioc, "bringing controller to ready state\n");
1416	ioc_config = readl(addr: &mrioc->sysif_regs->ioc_configuration);
1417	ioc_config |= MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC;
1418	writel(val: ioc_config, addr: &mrioc->sysif_regs->ioc_configuration);
1419
1420	timeout = mrioc->ready_timeout * 10;
1421	do {
1422	ioc_state = mpi3mr_get_iocstate(mrioc);
1423	if (ioc_state == MRIOC_STATE_READY) {
1424	ioc_info(mrioc,
1425	"successfully transitioned to %s state\n",
1426	mpi3mr_iocstate_name(ioc_state));
1427	return 0;
1428	}
1429	if (!pci_device_is_present(pdev: mrioc->pdev)) {
1430	mrioc->unrecoverable = 1;
1431	ioc_err(mrioc,
1432	"controller is not present at the bringup\n");
1433	retval = -1;
1434	goto out_device_not_present;
1435	}
1436	msleep(msecs: 100);
1437	} while (--timeout);
1438
1439	out_failed:
1440	ioc_state = mpi3mr_get_iocstate(mrioc);
1441	ioc_err(mrioc,
1442	"failed to bring to ready state, current state: %s\n",
1443	mpi3mr_iocstate_name(ioc_state));
1444	out_device_not_present:
1445	return retval;
1446	}
1447
1448	/**
1449	* mpi3mr_soft_reset_success - Check softreset is success or not
1450	* @ioc_status: IOC status register value
1451	* @ioc_config: IOC config register value
1452	*
1453	* Check whether the soft reset is successful or not based on
1454	* IOC status and IOC config register values.
1455	*
1456	* Return: True when the soft reset is success, false otherwise.
1457	*/
1458	static inline bool
1459	mpi3mr_soft_reset_success(u32 ioc_status, u32 ioc_config)
1460	{
1461	if (!((ioc_status & MPI3_SYSIF_IOC_STATUS_READY) ||
1462	(ioc_config & MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC)))
1463	return true;
1464	return false;
1465	}
1466
1467	/**
1468	* mpi3mr_diagfault_success - Check diag fault is success or not
1469	* @mrioc: Adapter reference
1470	* @ioc_status: IOC status register value
1471	*
1472	* Check whether the controller hit diag reset fault code.
1473	*
1474	* Return: True when there is diag fault, false otherwise.
1475	*/
1476	static inline bool mpi3mr_diagfault_success(struct mpi3mr_ioc *mrioc,
1477	u32 ioc_status)
1478	{
1479	u32 fault;
1480
1481	if (!(ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT))
1482	return false;
1483	fault = readl(addr: &mrioc->sysif_regs->fault) & MPI3_SYSIF_FAULT_CODE_MASK;
1484	if (fault == MPI3_SYSIF_FAULT_CODE_DIAG_FAULT_RESET) {
1485	mpi3mr_print_fault_info(mrioc);
1486	return true;
1487	}
1488	return false;
1489	}
1490
1491	/**
1492	* mpi3mr_set_diagsave - Set diag save bit for snapdump
1493	* @mrioc: Adapter reference
1494	*
1495	* Set diag save bit in IOC configuration register to enable
1496	* snapdump.
1497	*
1498	* Return: Nothing.
1499	*/
1500	static inline void mpi3mr_set_diagsave(struct mpi3mr_ioc *mrioc)
1501	{
1502	u32 ioc_config;
1503
1504	ioc_config = readl(addr: &mrioc->sysif_regs->ioc_configuration);
1505	ioc_config |= MPI3_SYSIF_IOC_CONFIG_DIAG_SAVE;
1506	writel(val: ioc_config, addr: &mrioc->sysif_regs->ioc_configuration);
1507	}
1508
1509	/**
1510	* mpi3mr_issue_reset - Issue reset to the controller
1511	* @mrioc: Adapter reference
1512	* @reset_type: Reset type
1513	* @reset_reason: Reset reason code
1514	*
1515	* Unlock the host diagnostic registers and write the specific
1516	* reset type to that, wait for reset acknowledgment from the
1517	* controller, if the reset is not successful retry for the
1518	* predefined number of times.
1519	*
1520	* Return: 0 on success, non-zero on failure.
1521	*/
1522	static int mpi3mr_issue_reset(struct mpi3mr_ioc *mrioc, u16 reset_type,
1523	u32 reset_reason)
1524	{
1525	int retval = -1;
1526	u8 unlock_retry_count = 0;
1527	u32 host_diagnostic, ioc_status, ioc_config;
1528	u32 timeout = MPI3MR_RESET_ACK_TIMEOUT * 10;
1529
1530	if ((reset_type != MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET) &&
1531	(reset_type != MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT))
1532	return retval;
1533	if (mrioc->unrecoverable)
1534	return retval;
1535	if (reset_reason == MPI3MR_RESET_FROM_FIRMWARE) {
1536	retval = 0;
1537	return retval;
1538	}
1539
1540	ioc_info(mrioc, "%s reset due to %s(0x%x)\n",
1541	mpi3mr_reset_type_name(reset_type),
1542	mpi3mr_reset_rc_name(reset_reason), reset_reason);
1543
1544	mpi3mr_clear_reset_history(mrioc);
1545	do {
1546	ioc_info(mrioc,
1547	"Write magic sequence to unlock host diag register (retry=%d)\n",
1548	++unlock_retry_count);
1549	if (unlock_retry_count >= MPI3MR_HOSTDIAG_UNLOCK_RETRY_COUNT) {
1550	ioc_err(mrioc,
1551	"%s reset failed due to unlock failure, host_diagnostic(0x%08x)\n",
1552	mpi3mr_reset_type_name(reset_type),
1553	host_diagnostic);
1554	mrioc->unrecoverable = 1;
1555	return retval;
1556	}
1557
1558	writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_FLUSH,
1559	addr: &mrioc->sysif_regs->write_sequence);
1560	writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_1ST,
1561	addr: &mrioc->sysif_regs->write_sequence);
1562	writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_2ND,
1563	addr: &mrioc->sysif_regs->write_sequence);
1564	writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_3RD,
1565	addr: &mrioc->sysif_regs->write_sequence);
1566	writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_4TH,
1567	addr: &mrioc->sysif_regs->write_sequence);
1568	writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_5TH,
1569	addr: &mrioc->sysif_regs->write_sequence);
1570	writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_6TH,
1571	addr: &mrioc->sysif_regs->write_sequence);
1572	usleep_range(min: 1000, max: 1100);
1573	host_diagnostic = readl(addr: &mrioc->sysif_regs->host_diagnostic);
1574	ioc_info(mrioc,
1575	"wrote magic sequence: retry_count(%d), host_diagnostic(0x%08x)\n",
1576	unlock_retry_count, host_diagnostic);
1577	} while (!(host_diagnostic & MPI3_SYSIF_HOST_DIAG_DIAG_WRITE_ENABLE));
1578
1579	writel(val: reset_reason, addr: &mrioc->sysif_regs->scratchpad[0]);
1580	writel(val: host_diagnostic | reset_type,
1581	addr: &mrioc->sysif_regs->host_diagnostic);
1582	switch (reset_type) {
1583	case MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET:
1584	do {
1585	ioc_status = readl(addr: &mrioc->sysif_regs->ioc_status);
1586	ioc_config =
1587	readl(addr: &mrioc->sysif_regs->ioc_configuration);
1588	if ((ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY)
1589	&& mpi3mr_soft_reset_success(ioc_status, ioc_config)
1590	) {
1591	mpi3mr_clear_reset_history(mrioc);
1592	retval = 0;
1593	break;
1594	}
1595	msleep(msecs: 100);
1596	} while (--timeout);
1597	mpi3mr_print_fault_info(mrioc);
1598	break;
1599	case MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT:
1600	do {
1601	ioc_status = readl(addr: &mrioc->sysif_regs->ioc_status);
1602	if (mpi3mr_diagfault_success(mrioc, ioc_status)) {
1603	retval = 0;
1604	break;
1605	}
1606	msleep(msecs: 100);
1607	} while (--timeout);
1608	break;
1609	default:
1610	break;
1611	}
1612
1613	writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_2ND,
1614	addr: &mrioc->sysif_regs->write_sequence);
1615
1616	ioc_config = readl(addr: &mrioc->sysif_regs->ioc_configuration);
1617	ioc_status = readl(addr: &mrioc->sysif_regs->ioc_status);
1618	ioc_info(mrioc,
1619	"ioc_status/ioc_onfig after %s reset is (0x%x)/(0x%x)\n",
1620	(!retval)?"successful":"failed", ioc_status,
1621	ioc_config);
1622	if (retval)
1623	mrioc->unrecoverable = 1;
1624	return retval;
1625	}
1626
1627	/**
1628	* mpi3mr_admin_request_post - Post request to admin queue
1629	* @mrioc: Adapter reference
1630	* @admin_req: MPI3 request
1631	* @admin_req_sz: Request size
1632	* @ignore_reset: Ignore reset in process
1633	*
1634	* Post the MPI3 request into admin request queue and
1635	* inform the controller, if the queue is full return
1636	* appropriate error.
1637	*
1638	* Return: 0 on success, non-zero on failure.
1639	*/
1640	int mpi3mr_admin_request_post(struct mpi3mr_ioc *mrioc, void *admin_req,
1641	u16 admin_req_sz, u8 ignore_reset)
1642	{
1643	u16 areq_pi = 0, areq_ci = 0, max_entries = 0;
1644	int retval = 0;
1645	unsigned long flags;
1646	u8 *areq_entry;
1647
1648	if (mrioc->unrecoverable) {
1649	ioc_err(mrioc, "%s : Unrecoverable controller\n", __func__);
1650	return -EFAULT;
1651	}
1652
1653	spin_lock_irqsave(&mrioc->admin_req_lock, flags);
1654	areq_pi = mrioc->admin_req_pi;
1655	areq_ci = mrioc->admin_req_ci;
1656	max_entries = mrioc->num_admin_req;
1657	if ((areq_ci == (areq_pi + 1)) || ((!areq_ci) &&
1658	(areq_pi == (max_entries - 1)))) {
1659	ioc_err(mrioc, "AdminReqQ full condition detected\n");
1660	retval = -EAGAIN;
1661	goto out;
1662	}
1663	if (!ignore_reset && mrioc->reset_in_progress) {
1664	ioc_err(mrioc, "AdminReqQ submit reset in progress\n");
1665	retval = -EAGAIN;
1666	goto out;
1667	}
1668	areq_entry = (u8 *)mrioc->admin_req_base +
1669	(areq_pi * MPI3MR_ADMIN_REQ_FRAME_SZ);
1670	memset(areq_entry, 0, MPI3MR_ADMIN_REQ_FRAME_SZ);
1671	memcpy(areq_entry, (u8 *)admin_req, admin_req_sz);
1672
1673	if (++areq_pi == max_entries)
1674	areq_pi = 0;
1675	mrioc->admin_req_pi = areq_pi;
1676
1677	writel(val: mrioc->admin_req_pi, addr: &mrioc->sysif_regs->admin_request_queue_pi);
1678
1679	out:
1680	spin_unlock_irqrestore(lock: &mrioc->admin_req_lock, flags);
1681
1682	return retval;
1683	}
1684
1685	/**
1686	* mpi3mr_free_op_req_q_segments - free request memory segments
1687	* @mrioc: Adapter instance reference
1688	* @q_idx: operational request queue index
1689	*
1690	* Free memory segments allocated for operational request queue
1691	*
1692	* Return: Nothing.
1693	*/
1694	static void mpi3mr_free_op_req_q_segments(struct mpi3mr_ioc *mrioc, u16 q_idx)
1695	{
1696	u16 j;
1697	int size;
1698	struct segments *segments;
1699
1700	segments = mrioc->req_qinfo[q_idx].q_segments;
1701	if (!segments)
1702	return;
1703
1704	if (mrioc->enable_segqueue) {
1705	size = MPI3MR_OP_REQ_Q_SEG_SIZE;
1706	if (mrioc->req_qinfo[q_idx].q_segment_list) {
1707	dma_free_coherent(dev: &mrioc->pdev->dev,
1708	MPI3MR_MAX_SEG_LIST_SIZE,
1709	cpu_addr: mrioc->req_qinfo[q_idx].q_segment_list,
1710	dma_handle: mrioc->req_qinfo[q_idx].q_segment_list_dma);
1711	mrioc->req_qinfo[q_idx].q_segment_list = NULL;
1712	}
1713	} else
1714	size = mrioc->req_qinfo[q_idx].segment_qd *
1715	mrioc->facts.op_req_sz;
1716
1717	for (j = 0; j < mrioc->req_qinfo[q_idx].num_segments; j++) {
1718	if (!segments[j].segment)
1719	continue;
1720	dma_free_coherent(dev: &mrioc->pdev->dev,
1721	size, cpu_addr: segments[j].segment, dma_handle: segments[j].segment_dma);
1722	segments[j].segment = NULL;
1723	}
1724	kfree(objp: mrioc->req_qinfo[q_idx].q_segments);
1725	mrioc->req_qinfo[q_idx].q_segments = NULL;
1726	mrioc->req_qinfo[q_idx].qid = 0;
1727	}
1728
1729	/**
1730	* mpi3mr_free_op_reply_q_segments - free reply memory segments
1731	* @mrioc: Adapter instance reference
1732	* @q_idx: operational reply queue index
1733	*
1734	* Free memory segments allocated for operational reply queue
1735	*
1736	* Return: Nothing.
1737	*/
1738	static void mpi3mr_free_op_reply_q_segments(struct mpi3mr_ioc *mrioc, u16 q_idx)
1739	{
1740	u16 j;
1741	int size;
1742	struct segments *segments;
1743
1744	segments = mrioc->op_reply_qinfo[q_idx].q_segments;
1745	if (!segments)
1746	return;
1747
1748	if (mrioc->enable_segqueue) {
1749	size = MPI3MR_OP_REP_Q_SEG_SIZE;
1750	if (mrioc->op_reply_qinfo[q_idx].q_segment_list) {
1751	dma_free_coherent(dev: &mrioc->pdev->dev,
1752	MPI3MR_MAX_SEG_LIST_SIZE,
1753	cpu_addr: mrioc->op_reply_qinfo[q_idx].q_segment_list,
1754	dma_handle: mrioc->op_reply_qinfo[q_idx].q_segment_list_dma);
1755	mrioc->op_reply_qinfo[q_idx].q_segment_list = NULL;
1756	}
1757	} else
1758	size = mrioc->op_reply_qinfo[q_idx].segment_qd *
1759	mrioc->op_reply_desc_sz;
1760
1761	for (j = 0; j < mrioc->op_reply_qinfo[q_idx].num_segments; j++) {
1762	if (!segments[j].segment)
1763	continue;
1764	dma_free_coherent(dev: &mrioc->pdev->dev,
1765	size, cpu_addr: segments[j].segment, dma_handle: segments[j].segment_dma);
1766	segments[j].segment = NULL;
1767	}
1768
1769	kfree(objp: mrioc->op_reply_qinfo[q_idx].q_segments);
1770	mrioc->op_reply_qinfo[q_idx].q_segments = NULL;
1771	mrioc->op_reply_qinfo[q_idx].qid = 0;
1772	}
1773
1774	/**
1775	* mpi3mr_delete_op_reply_q - delete operational reply queue
1776	* @mrioc: Adapter instance reference
1777	* @qidx: operational reply queue index
1778	*
1779	* Delete operatinal reply queue by issuing MPI request
1780	* through admin queue.
1781	*
1782	* Return: 0 on success, non-zero on failure.
1783	*/
1784	static int mpi3mr_delete_op_reply_q(struct mpi3mr_ioc *mrioc, u16 qidx)
1785	{
1786	struct mpi3_delete_reply_queue_request delq_req;
1787	struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;
1788	int retval = 0;
1789	u16 reply_qid = 0, midx;
1790
1791	reply_qid = op_reply_q->qid;
1792
1793	midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(qidx, mrioc->op_reply_q_offset);
1794
1795	if (!reply_qid) {
1796	retval = -1;
1797	ioc_err(mrioc, "Issue DelRepQ: called with invalid ReqQID\n");
1798	goto out;
1799	}
1800
1801	(op_reply_q->qtype == MPI3MR_DEFAULT_QUEUE) ? mrioc->default_qcount-- :
1802	mrioc->active_poll_qcount--;
1803
1804	memset(&delq_req, 0, sizeof(delq_req));
1805	mutex_lock(&mrioc->init_cmds.mutex);
1806	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
1807	retval = -1;
1808	ioc_err(mrioc, "Issue DelRepQ: Init command is in use\n");
1809	mutex_unlock(lock: &mrioc->init_cmds.mutex);
1810	goto out;
1811	}
1812	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
1813	mrioc->init_cmds.is_waiting = 1;
1814	mrioc->init_cmds.callback = NULL;
1815	delq_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
1816	delq_req.function = MPI3_FUNCTION_DELETE_REPLY_QUEUE;
1817	delq_req.queue_id = cpu_to_le16(reply_qid);
1818
1819	init_completion(x: &mrioc->init_cmds.done);
1820	retval = mpi3mr_admin_request_post(mrioc, admin_req: &delq_req, admin_req_sz: sizeof(delq_req),
1821	ignore_reset: 1);
1822	if (retval) {
1823	ioc_err(mrioc, "Issue DelRepQ: Admin Post failed\n");
1824	goto out_unlock;
1825	}
1826	wait_for_completion_timeout(x: &mrioc->init_cmds.done,
1827	timeout: (MPI3MR_INTADMCMD_TIMEOUT * HZ));
1828	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
1829	ioc_err(mrioc, "delete reply queue timed out\n");
1830	mpi3mr_check_rh_fault_ioc(mrioc,
1831	reason_code: MPI3MR_RESET_FROM_DELREPQ_TIMEOUT);
1832	retval = -1;
1833	goto out_unlock;
1834	}
1835	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
1836	!= MPI3_IOCSTATUS_SUCCESS) {
1837	ioc_err(mrioc,
1838	"Issue DelRepQ: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
1839	(mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
1840	mrioc->init_cmds.ioc_loginfo);
1841	retval = -1;
1842	goto out_unlock;
1843	}
1844	mrioc->intr_info[midx].op_reply_q = NULL;
1845
1846	mpi3mr_free_op_reply_q_segments(mrioc, q_idx: qidx);
1847	out_unlock:
1848	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
1849	mutex_unlock(lock: &mrioc->init_cmds.mutex);
1850	out:
1851
1852	return retval;
1853	}
1854
1855	/**
1856	* mpi3mr_alloc_op_reply_q_segments -Alloc segmented reply pool
1857	* @mrioc: Adapter instance reference
1858	* @qidx: request queue index
1859	*
1860	* Allocate segmented memory pools for operational reply
1861	* queue.
1862	*
1863	* Return: 0 on success, non-zero on failure.
1864	*/
1865	static int mpi3mr_alloc_op_reply_q_segments(struct mpi3mr_ioc *mrioc, u16 qidx)
1866	{
1867	struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;
1868	int i, size;
1869	u64 *q_segment_list_entry = NULL;
1870	struct segments *segments;
1871
1872	if (mrioc->enable_segqueue) {
1873	op_reply_q->segment_qd =
1874	MPI3MR_OP_REP_Q_SEG_SIZE / mrioc->op_reply_desc_sz;
1875
1876	size = MPI3MR_OP_REP_Q_SEG_SIZE;
1877
1878	op_reply_q->q_segment_list = dma_alloc_coherent(dev: &mrioc->pdev->dev,
1879	MPI3MR_MAX_SEG_LIST_SIZE, dma_handle: &op_reply_q->q_segment_list_dma,
1880	GFP_KERNEL);
1881	if (!op_reply_q->q_segment_list)
1882	return -ENOMEM;
1883	q_segment_list_entry = (u64 *)op_reply_q->q_segment_list;
1884	} else {
1885	op_reply_q->segment_qd = op_reply_q->num_replies;
1886	size = op_reply_q->num_replies * mrioc->op_reply_desc_sz;
1887	}
1888
1889	op_reply_q->num_segments = DIV_ROUND_UP(op_reply_q->num_replies,
1890	op_reply_q->segment_qd);
1891
1892	op_reply_q->q_segments = kcalloc(n: op_reply_q->num_segments,
1893	size: sizeof(struct segments), GFP_KERNEL);
1894	if (!op_reply_q->q_segments)
1895	return -ENOMEM;
1896
1897	segments = op_reply_q->q_segments;
1898	for (i = 0; i < op_reply_q->num_segments; i++) {
1899	segments[i].segment =
1900	dma_alloc_coherent(dev: &mrioc->pdev->dev,
1901	size, dma_handle: &segments[i].segment_dma, GFP_KERNEL);
1902	if (!segments[i].segment)
1903	return -ENOMEM;
1904	if (mrioc->enable_segqueue)
1905	q_segment_list_entry[i] =
1906	(unsigned long)segments[i].segment_dma;
1907	}
1908
1909	return 0;
1910	}
1911
1912	/**
1913	* mpi3mr_alloc_op_req_q_segments - Alloc segmented req pool.
1914	* @mrioc: Adapter instance reference
1915	* @qidx: request queue index
1916	*
1917	* Allocate segmented memory pools for operational request
1918	* queue.
1919	*
1920	* Return: 0 on success, non-zero on failure.
1921	*/
1922	static int mpi3mr_alloc_op_req_q_segments(struct mpi3mr_ioc *mrioc, u16 qidx)
1923	{
1924	struct op_req_qinfo *op_req_q = mrioc->req_qinfo + qidx;
1925	int i, size;
1926	u64 *q_segment_list_entry = NULL;
1927	struct segments *segments;
1928
1929	if (mrioc->enable_segqueue) {
1930	op_req_q->segment_qd =
1931	MPI3MR_OP_REQ_Q_SEG_SIZE / mrioc->facts.op_req_sz;
1932
1933	size = MPI3MR_OP_REQ_Q_SEG_SIZE;
1934
1935	op_req_q->q_segment_list = dma_alloc_coherent(dev: &mrioc->pdev->dev,
1936	MPI3MR_MAX_SEG_LIST_SIZE, dma_handle: &op_req_q->q_segment_list_dma,
1937	GFP_KERNEL);
1938	if (!op_req_q->q_segment_list)
1939	return -ENOMEM;
1940	q_segment_list_entry = (u64 *)op_req_q->q_segment_list;
1941
1942	} else {
1943	op_req_q->segment_qd = op_req_q->num_requests;
1944	size = op_req_q->num_requests * mrioc->facts.op_req_sz;
1945	}
1946
1947	op_req_q->num_segments = DIV_ROUND_UP(op_req_q->num_requests,
1948	op_req_q->segment_qd);
1949
1950	op_req_q->q_segments = kcalloc(n: op_req_q->num_segments,
1951	size: sizeof(struct segments), GFP_KERNEL);
1952	if (!op_req_q->q_segments)
1953	return -ENOMEM;
1954
1955	segments = op_req_q->q_segments;
1956	for (i = 0; i < op_req_q->num_segments; i++) {
1957	segments[i].segment =
1958	dma_alloc_coherent(dev: &mrioc->pdev->dev,
1959	size, dma_handle: &segments[i].segment_dma, GFP_KERNEL);
1960	if (!segments[i].segment)
1961	return -ENOMEM;
1962	if (mrioc->enable_segqueue)
1963	q_segment_list_entry[i] =
1964	(unsigned long)segments[i].segment_dma;
1965	}
1966
1967	return 0;
1968	}
1969
1970	/**
1971	* mpi3mr_create_op_reply_q - create operational reply queue
1972	* @mrioc: Adapter instance reference
1973	* @qidx: operational reply queue index
1974	*
1975	* Create operatinal reply queue by issuing MPI request
1976	* through admin queue.
1977	*
1978	* Return: 0 on success, non-zero on failure.
1979	*/
1980	static int mpi3mr_create_op_reply_q(struct mpi3mr_ioc *mrioc, u16 qidx)
1981	{
1982	struct mpi3_create_reply_queue_request create_req;
1983	struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;
1984	int retval = 0;
1985	u16 reply_qid = 0, midx;
1986
1987	reply_qid = op_reply_q->qid;
1988
1989	midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(qidx, mrioc->op_reply_q_offset);
1990
1991	if (reply_qid) {
1992	retval = -1;
1993	ioc_err(mrioc, "CreateRepQ: called for duplicate qid %d\n",
1994	reply_qid);
1995
1996	return retval;
1997	}
1998
1999	reply_qid = qidx + 1;
2000	op_reply_q->num_replies = MPI3MR_OP_REP_Q_QD;
2001	if ((mrioc->pdev->device == MPI3_MFGPAGE_DEVID_SAS4116) &&
2002	!mrioc->pdev->revision)
2003	op_reply_q->num_replies = MPI3MR_OP_REP_Q_QD4K;
2004	op_reply_q->ci = 0;
2005	op_reply_q->ephase = 1;
2006	atomic_set(v: &op_reply_q->pend_ios, i: 0);
2007	atomic_set(v: &op_reply_q->in_use, i: 0);
2008	op_reply_q->enable_irq_poll = false;
2009
2010	if (!op_reply_q->q_segments) {
2011	retval = mpi3mr_alloc_op_reply_q_segments(mrioc, qidx);
2012	if (retval) {
2013	mpi3mr_free_op_reply_q_segments(mrioc, q_idx: qidx);
2014	goto out;
2015	}
2016	}
2017
2018	memset(&create_req, 0, sizeof(create_req));
2019	mutex_lock(&mrioc->init_cmds.mutex);
2020	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2021	retval = -1;
2022	ioc_err(mrioc, "CreateRepQ: Init command is in use\n");
2023	goto out_unlock;
2024	}
2025	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2026	mrioc->init_cmds.is_waiting = 1;
2027	mrioc->init_cmds.callback = NULL;
2028	create_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2029	create_req.function = MPI3_FUNCTION_CREATE_REPLY_QUEUE;
2030	create_req.queue_id = cpu_to_le16(reply_qid);
2031
2032	if (midx < (mrioc->intr_info_count - mrioc->requested_poll_qcount))
2033	op_reply_q->qtype = MPI3MR_DEFAULT_QUEUE;
2034	else
2035	op_reply_q->qtype = MPI3MR_POLL_QUEUE;
2036
2037	if (op_reply_q->qtype == MPI3MR_DEFAULT_QUEUE) {
2038	create_req.flags =
2039	MPI3_CREATE_REPLY_QUEUE_FLAGS_INT_ENABLE_ENABLE;
2040	create_req.msix_index =
2041	cpu_to_le16(mrioc->intr_info[midx].msix_index);
2042	} else {
2043	create_req.msix_index = cpu_to_le16(mrioc->intr_info_count - 1);
2044	ioc_info(mrioc, "create reply queue(polled): for qid(%d), midx(%d)\n",
2045	reply_qid, midx);
2046	if (!mrioc->active_poll_qcount)
2047	disable_irq_nosync(irq: pci_irq_vector(dev: mrioc->pdev,
2048	nr: mrioc->intr_info_count - 1));
2049	}
2050
2051	if (mrioc->enable_segqueue) {
2052	create_req.flags |=
2053	MPI3_CREATE_REQUEST_QUEUE_FLAGS_SEGMENTED_SEGMENTED;
2054	create_req.base_address = cpu_to_le64(
2055	op_reply_q->q_segment_list_dma);
2056	} else
2057	create_req.base_address = cpu_to_le64(
2058	op_reply_q->q_segments[0].segment_dma);
2059
2060	create_req.size = cpu_to_le16(op_reply_q->num_replies);
2061
2062	init_completion(x: &mrioc->init_cmds.done);
2063	retval = mpi3mr_admin_request_post(mrioc, admin_req: &create_req,
2064	admin_req_sz: sizeof(create_req), ignore_reset: 1);
2065	if (retval) {
2066	ioc_err(mrioc, "CreateRepQ: Admin Post failed\n");
2067	goto out_unlock;
2068	}
2069	wait_for_completion_timeout(x: &mrioc->init_cmds.done,
2070	timeout: (MPI3MR_INTADMCMD_TIMEOUT * HZ));
2071	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2072	ioc_err(mrioc, "create reply queue timed out\n");
2073	mpi3mr_check_rh_fault_ioc(mrioc,
2074	reason_code: MPI3MR_RESET_FROM_CREATEREPQ_TIMEOUT);
2075	retval = -1;
2076	goto out_unlock;
2077	}
2078	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2079	!= MPI3_IOCSTATUS_SUCCESS) {
2080	ioc_err(mrioc,
2081	"CreateRepQ: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
2082	(mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2083	mrioc->init_cmds.ioc_loginfo);
2084	retval = -1;
2085	goto out_unlock;
2086	}
2087	op_reply_q->qid = reply_qid;
2088	if (midx < mrioc->intr_info_count)
2089	mrioc->intr_info[midx].op_reply_q = op_reply_q;
2090
2091	(op_reply_q->qtype == MPI3MR_DEFAULT_QUEUE) ? mrioc->default_qcount++ :
2092	mrioc->active_poll_qcount++;
2093
2094	out_unlock:
2095	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2096	mutex_unlock(lock: &mrioc->init_cmds.mutex);
2097	out:
2098
2099	return retval;
2100	}
2101
2102	/**
2103	* mpi3mr_create_op_req_q - create operational request queue
2104	* @mrioc: Adapter instance reference
2105	* @idx: operational request queue index
2106	* @reply_qid: Reply queue ID
2107	*
2108	* Create operatinal request queue by issuing MPI request
2109	* through admin queue.
2110	*
2111	* Return: 0 on success, non-zero on failure.
2112	*/
2113	static int mpi3mr_create_op_req_q(struct mpi3mr_ioc *mrioc, u16 idx,
2114	u16 reply_qid)
2115	{
2116	struct mpi3_create_request_queue_request create_req;
2117	struct op_req_qinfo *op_req_q = mrioc->req_qinfo + idx;
2118	int retval = 0;
2119	u16 req_qid = 0;
2120
2121	req_qid = op_req_q->qid;
2122
2123	if (req_qid) {
2124	retval = -1;
2125	ioc_err(mrioc, "CreateReqQ: called for duplicate qid %d\n",
2126	req_qid);
2127
2128	return retval;
2129	}
2130	req_qid = idx + 1;
2131
2132	op_req_q->num_requests = MPI3MR_OP_REQ_Q_QD;
2133	op_req_q->ci = 0;
2134	op_req_q->pi = 0;
2135	op_req_q->reply_qid = reply_qid;
2136	spin_lock_init(&op_req_q->q_lock);
2137
2138	if (!op_req_q->q_segments) {
2139	retval = mpi3mr_alloc_op_req_q_segments(mrioc, qidx: idx);
2140	if (retval) {
2141	mpi3mr_free_op_req_q_segments(mrioc, q_idx: idx);
2142	goto out;
2143	}
2144	}
2145
2146	memset(&create_req, 0, sizeof(create_req));
2147	mutex_lock(&mrioc->init_cmds.mutex);
2148	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2149	retval = -1;
2150	ioc_err(mrioc, "CreateReqQ: Init command is in use\n");
2151	goto out_unlock;
2152	}
2153	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2154	mrioc->init_cmds.is_waiting = 1;
2155	mrioc->init_cmds.callback = NULL;
2156	create_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2157	create_req.function = MPI3_FUNCTION_CREATE_REQUEST_QUEUE;
2158	create_req.queue_id = cpu_to_le16(req_qid);
2159	if (mrioc->enable_segqueue) {
2160	create_req.flags =
2161	MPI3_CREATE_REQUEST_QUEUE_FLAGS_SEGMENTED_SEGMENTED;
2162	create_req.base_address = cpu_to_le64(
2163	op_req_q->q_segment_list_dma);
2164	} else
2165	create_req.base_address = cpu_to_le64(
2166	op_req_q->q_segments[0].segment_dma);
2167	create_req.reply_queue_id = cpu_to_le16(reply_qid);
2168	create_req.size = cpu_to_le16(op_req_q->num_requests);
2169
2170	init_completion(x: &mrioc->init_cmds.done);
2171	retval = mpi3mr_admin_request_post(mrioc, admin_req: &create_req,
2172	admin_req_sz: sizeof(create_req), ignore_reset: 1);
2173	if (retval) {
2174	ioc_err(mrioc, "CreateReqQ: Admin Post failed\n");
2175	goto out_unlock;
2176	}
2177	wait_for_completion_timeout(x: &mrioc->init_cmds.done,
2178	timeout: (MPI3MR_INTADMCMD_TIMEOUT * HZ));
2179	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2180	ioc_err(mrioc, "create request queue timed out\n");
2181	mpi3mr_check_rh_fault_ioc(mrioc,
2182	reason_code: MPI3MR_RESET_FROM_CREATEREQQ_TIMEOUT);
2183	retval = -1;
2184	goto out_unlock;
2185	}
2186	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2187	!= MPI3_IOCSTATUS_SUCCESS) {
2188	ioc_err(mrioc,
2189	"CreateReqQ: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
2190	(mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2191	mrioc->init_cmds.ioc_loginfo);
2192	retval = -1;
2193	goto out_unlock;
2194	}
2195	op_req_q->qid = req_qid;
2196
2197	out_unlock:
2198	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2199	mutex_unlock(lock: &mrioc->init_cmds.mutex);
2200	out:
2201
2202	return retval;
2203	}
2204
2205	/**
2206	* mpi3mr_create_op_queues - create operational queue pairs
2207	* @mrioc: Adapter instance reference
2208	*
2209	* Allocate memory for operational queue meta data and call
2210	* create request and reply queue functions.
2211	*
2212	* Return: 0 on success, non-zero on failures.
2213	*/
2214	static int mpi3mr_create_op_queues(struct mpi3mr_ioc *mrioc)
2215	{
2216	int retval = 0;
2217	u16 num_queues = 0, i = 0, msix_count_op_q = 1;
2218
2219	num_queues = min_t(int, mrioc->facts.max_op_reply_q,
2220	mrioc->facts.max_op_req_q);
2221
2222	msix_count_op_q =
2223	mrioc->intr_info_count - mrioc->op_reply_q_offset;
2224	if (!mrioc->num_queues)
2225	mrioc->num_queues = min_t(int, num_queues, msix_count_op_q);
2226	/*
2227	* During reset set the num_queues to the number of queues
2228	* that was set before the reset.
2229	*/
2230	num_queues = mrioc->num_op_reply_q ?
2231	mrioc->num_op_reply_q : mrioc->num_queues;
2232	ioc_info(mrioc, "trying to create %d operational queue pairs\n",
2233	num_queues);
2234
2235	if (!mrioc->req_qinfo) {
2236	mrioc->req_qinfo = kcalloc(n: num_queues,
2237	size: sizeof(struct op_req_qinfo), GFP_KERNEL);
2238	if (!mrioc->req_qinfo) {
2239	retval = -1;
2240	goto out_failed;
2241	}
2242
2243	mrioc->op_reply_qinfo = kzalloc(size: sizeof(struct op_reply_qinfo) *
2244	num_queues, GFP_KERNEL);
2245	if (!mrioc->op_reply_qinfo) {
2246	retval = -1;
2247	goto out_failed;
2248	}
2249	}
2250
2251	if (mrioc->enable_segqueue)
2252	ioc_info(mrioc,
2253	"allocating operational queues through segmented queues\n");
2254
2255	for (i = 0; i < num_queues; i++) {
2256	if (mpi3mr_create_op_reply_q(mrioc, qidx: i)) {
2257	ioc_err(mrioc, "Cannot create OP RepQ %d\n", i);
2258	break;
2259	}
2260	if (mpi3mr_create_op_req_q(mrioc, idx: i,
2261	reply_qid: mrioc->op_reply_qinfo[i].qid)) {
2262	ioc_err(mrioc, "Cannot create OP ReqQ %d\n", i);
2263	mpi3mr_delete_op_reply_q(mrioc, qidx: i);
2264	break;
2265	}
2266	}
2267
2268	if (i == 0) {
2269	/* Not even one queue is created successfully*/
2270	retval = -1;
2271	goto out_failed;
2272	}
2273	mrioc->num_op_reply_q = mrioc->num_op_req_q = i;
2274	ioc_info(mrioc,
2275	"successfully created %d operational queue pairs(default/polled) queue = (%d/%d)\n",
2276	mrioc->num_op_reply_q, mrioc->default_qcount,
2277	mrioc->active_poll_qcount);
2278
2279	return retval;
2280	out_failed:
2281	kfree(objp: mrioc->req_qinfo);
2282	mrioc->req_qinfo = NULL;
2283
2284	kfree(objp: mrioc->op_reply_qinfo);
2285	mrioc->op_reply_qinfo = NULL;
2286
2287	return retval;
2288	}
2289
2290	/**
2291	* mpi3mr_op_request_post - Post request to operational queue
2292	* @mrioc: Adapter reference
2293	* @op_req_q: Operational request queue info
2294	* @req: MPI3 request
2295	*
2296	* Post the MPI3 request into operational request queue and
2297	* inform the controller, if the queue is full return
2298	* appropriate error.
2299	*
2300	* Return: 0 on success, non-zero on failure.
2301	*/
2302	int mpi3mr_op_request_post(struct mpi3mr_ioc *mrioc,
2303	struct op_req_qinfo *op_req_q, u8 *req)
2304	{
2305	u16 pi = 0, max_entries, reply_qidx = 0, midx;
2306	int retval = 0;
2307	unsigned long flags;
2308	u8 *req_entry;
2309	void *segment_base_addr;
2310	u16 req_sz = mrioc->facts.op_req_sz;
2311	struct segments *segments = op_req_q->q_segments;
2312
2313	reply_qidx = op_req_q->reply_qid - 1;
2314
2315	if (mrioc->unrecoverable)
2316	return -EFAULT;
2317
2318	spin_lock_irqsave(&op_req_q->q_lock, flags);
2319	pi = op_req_q->pi;
2320	max_entries = op_req_q->num_requests;
2321
2322	if (mpi3mr_check_req_qfull(op_req_q)) {
2323	midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(
2324	reply_qidx, mrioc->op_reply_q_offset);
2325	mpi3mr_process_op_reply_q(mrioc, op_reply_q: mrioc->intr_info[midx].op_reply_q);
2326
2327	if (mpi3mr_check_req_qfull(op_req_q)) {
2328	retval = -EAGAIN;
2329	goto out;
2330	}
2331	}
2332
2333	if (mrioc->reset_in_progress) {
2334	ioc_err(mrioc, "OpReqQ submit reset in progress\n");
2335	retval = -EAGAIN;
2336	goto out;
2337	}
2338
2339	segment_base_addr = segments[pi / op_req_q->segment_qd].segment;
2340	req_entry = (u8 *)segment_base_addr +
2341	((pi % op_req_q->segment_qd) * req_sz);
2342
2343	memset(req_entry, 0, req_sz);
2344	memcpy(req_entry, req, MPI3MR_ADMIN_REQ_FRAME_SZ);
2345
2346	if (++pi == max_entries)
2347	pi = 0;
2348	op_req_q->pi = pi;
2349
2350	#ifndef CONFIG_PREEMPT_RT
2351	if (atomic_inc_return(v: &mrioc->op_reply_qinfo[reply_qidx].pend_ios)
2352	> MPI3MR_IRQ_POLL_TRIGGER_IOCOUNT)
2353	mrioc->op_reply_qinfo[reply_qidx].enable_irq_poll = true;
2354	#else
2355	atomic_inc_return(&mrioc->op_reply_qinfo[reply_qidx].pend_ios);
2356	#endif
2357
2358	writel(val: op_req_q->pi,
2359	addr: &mrioc->sysif_regs->oper_queue_indexes[reply_qidx].producer_index);
2360
2361	out:
2362	spin_unlock_irqrestore(lock: &op_req_q->q_lock, flags);
2363	return retval;
2364	}
2365
2366	/**
2367	* mpi3mr_check_rh_fault_ioc - check reset history and fault
2368	* controller
2369	* @mrioc: Adapter instance reference
2370	* @reason_code: reason code for the fault.
2371	*
2372	* This routine will save snapdump and fault the controller with
2373	* the given reason code if it is not already in the fault or
2374	* not asynchronosuly reset. This will be used to handle
2375	* initilaization time faults/resets/timeout as in those cases
2376	* immediate soft reset invocation is not required.
2377	*
2378	* Return: None.
2379	*/
2380	void mpi3mr_check_rh_fault_ioc(struct mpi3mr_ioc *mrioc, u32 reason_code)
2381	{
2382	u32 ioc_status, host_diagnostic, timeout;
2383
2384	if (mrioc->unrecoverable) {
2385	ioc_err(mrioc, "controller is unrecoverable\n");
2386	return;
2387	}
2388
2389	if (!pci_device_is_present(pdev: mrioc->pdev)) {
2390	mrioc->unrecoverable = 1;
2391	ioc_err(mrioc, "controller is not present\n");
2392	return;
2393	}
2394
2395	ioc_status = readl(addr: &mrioc->sysif_regs->ioc_status);
2396	if ((ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY) ||
2397	(ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT)) {
2398	mpi3mr_print_fault_info(mrioc);
2399	return;
2400	}
2401	mpi3mr_set_diagsave(mrioc);
2402	mpi3mr_issue_reset(mrioc, MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
2403	reset_reason: reason_code);
2404	timeout = MPI3_SYSIF_DIAG_SAVE_TIMEOUT * 10;
2405	do {
2406	host_diagnostic = readl(addr: &mrioc->sysif_regs->host_diagnostic);
2407	if (!(host_diagnostic & MPI3_SYSIF_HOST_DIAG_SAVE_IN_PROGRESS))
2408	break;
2409	msleep(msecs: 100);
2410	} while (--timeout);
2411	}
2412
2413	/**
2414	* mpi3mr_sync_timestamp - Issue time stamp sync request
2415	* @mrioc: Adapter reference
2416	*
2417	* Issue IO unit control MPI request to synchornize firmware
2418	* timestamp with host time.
2419	*
2420	* Return: 0 on success, non-zero on failure.
2421	*/
2422	static int mpi3mr_sync_timestamp(struct mpi3mr_ioc *mrioc)
2423	{
2424	ktime_t current_time;
2425	struct mpi3_iounit_control_request iou_ctrl;
2426	int retval = 0;
2427
2428	memset(&iou_ctrl, 0, sizeof(iou_ctrl));
2429	mutex_lock(&mrioc->init_cmds.mutex);
2430	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2431	retval = -1;
2432	ioc_err(mrioc, "Issue IOUCTL time_stamp: command is in use\n");
2433	mutex_unlock(lock: &mrioc->init_cmds.mutex);
2434	goto out;
2435	}
2436	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2437	mrioc->init_cmds.is_waiting = 1;
2438	mrioc->init_cmds.callback = NULL;
2439	iou_ctrl.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2440	iou_ctrl.function = MPI3_FUNCTION_IO_UNIT_CONTROL;
2441	iou_ctrl.operation = MPI3_CTRL_OP_UPDATE_TIMESTAMP;
2442	current_time = ktime_get_real();
2443	iou_ctrl.param64[0] = cpu_to_le64(ktime_to_ms(current_time));
2444
2445	init_completion(x: &mrioc->init_cmds.done);
2446	retval = mpi3mr_admin_request_post(mrioc, admin_req: &iou_ctrl,
2447	admin_req_sz: sizeof(iou_ctrl), ignore_reset: 0);
2448	if (retval) {
2449	ioc_err(mrioc, "Issue IOUCTL time_stamp: Admin Post failed\n");
2450	goto out_unlock;
2451	}
2452
2453	wait_for_completion_timeout(x: &mrioc->init_cmds.done,
2454	timeout: (MPI3MR_INTADMCMD_TIMEOUT * HZ));
2455	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2456	ioc_err(mrioc, "Issue IOUCTL time_stamp: command timed out\n");
2457	mrioc->init_cmds.is_waiting = 0;
2458	if (!(mrioc->init_cmds.state & MPI3MR_CMD_RESET))
2459	mpi3mr_check_rh_fault_ioc(mrioc,
2460	reason_code: MPI3MR_RESET_FROM_TSU_TIMEOUT);
2461	retval = -1;
2462	goto out_unlock;
2463	}
2464	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2465	!= MPI3_IOCSTATUS_SUCCESS) {
2466	ioc_err(mrioc,
2467	"Issue IOUCTL time_stamp: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
2468	(mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2469	mrioc->init_cmds.ioc_loginfo);
2470	retval = -1;
2471	goto out_unlock;
2472	}
2473
2474	out_unlock:
2475	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2476	mutex_unlock(lock: &mrioc->init_cmds.mutex);
2477
2478	out:
2479	return retval;
2480	}
2481
2482	/**
2483	* mpi3mr_print_pkg_ver - display controller fw package version
2484	* @mrioc: Adapter reference
2485	*
2486	* Retrieve firmware package version from the component image
2487	* header of the controller flash and display it.
2488	*
2489	* Return: 0 on success and non-zero on failure.
2490	*/
2491	static int mpi3mr_print_pkg_ver(struct mpi3mr_ioc *mrioc)
2492	{
2493	struct mpi3_ci_upload_request ci_upload;
2494	int retval = -1;
2495	void *data = NULL;
2496	dma_addr_t data_dma;
2497	struct mpi3_ci_manifest_mpi *manifest;
2498	u32 data_len = sizeof(struct mpi3_ci_manifest_mpi);
2499	u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST;
2500
2501	data = dma_alloc_coherent(dev: &mrioc->pdev->dev, size: data_len, dma_handle: &data_dma,
2502	GFP_KERNEL);
2503	if (!data)
2504	return -ENOMEM;
2505
2506	memset(&ci_upload, 0, sizeof(ci_upload));
2507	mutex_lock(&mrioc->init_cmds.mutex);
2508	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2509	ioc_err(mrioc, "sending get package version failed due to command in use\n");
2510	mutex_unlock(lock: &mrioc->init_cmds.mutex);
2511	goto out;
2512	}
2513	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2514	mrioc->init_cmds.is_waiting = 1;
2515	mrioc->init_cmds.callback = NULL;
2516	ci_upload.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2517	ci_upload.function = MPI3_FUNCTION_CI_UPLOAD;
2518	ci_upload.msg_flags = MPI3_CI_UPLOAD_MSGFLAGS_LOCATION_PRIMARY;
2519	ci_upload.signature1 = cpu_to_le32(MPI3_IMAGE_HEADER_SIGNATURE1_MANIFEST);
2520	ci_upload.image_offset = cpu_to_le32(MPI3_IMAGE_HEADER_SIZE);
2521	ci_upload.segment_size = cpu_to_le32(data_len);
2522
2523	mpi3mr_add_sg_single(paddr: &ci_upload.sgl, flags: sgl_flags, length: data_len,
2524	dma_addr: data_dma);
2525	init_completion(x: &mrioc->init_cmds.done);
2526	retval = mpi3mr_admin_request_post(mrioc, admin_req: &ci_upload,
2527	admin_req_sz: sizeof(ci_upload), ignore_reset: 1);
2528	if (retval) {
2529	ioc_err(mrioc, "posting get package version failed\n");
2530	goto out_unlock;
2531	}
2532	wait_for_completion_timeout(x: &mrioc->init_cmds.done,
2533	timeout: (MPI3MR_INTADMCMD_TIMEOUT * HZ));
2534	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2535	ioc_err(mrioc, "get package version timed out\n");
2536	mpi3mr_check_rh_fault_ioc(mrioc,
2537	reason_code: MPI3MR_RESET_FROM_GETPKGVER_TIMEOUT);
2538	retval = -1;
2539	goto out_unlock;
2540	}
2541	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2542	== MPI3_IOCSTATUS_SUCCESS) {
2543	manifest = (struct mpi3_ci_manifest_mpi *) data;
2544	if (manifest->manifest_type == MPI3_CI_MANIFEST_TYPE_MPI) {
2545	ioc_info(mrioc,
2546	"firmware package version(%d.%d.%d.%d.%05d-%05d)\n",
2547	manifest->package_version.gen_major,
2548	manifest->package_version.gen_minor,
2549	manifest->package_version.phase_major,
2550	manifest->package_version.phase_minor,
2551	manifest->package_version.customer_id,
2552	manifest->package_version.build_num);
2553	}
2554	}
2555	retval = 0;
2556	out_unlock:
2557	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2558	mutex_unlock(lock: &mrioc->init_cmds.mutex);
2559
2560	out:
2561	if (data)
2562	dma_free_coherent(dev: &mrioc->pdev->dev, size: data_len, cpu_addr: data,
2563	dma_handle: data_dma);
2564	return retval;
2565	}
2566
2567	/**
2568	* mpi3mr_watchdog_work - watchdog thread to monitor faults
2569	* @work: work struct
2570	*
2571	* Watch dog work periodically executed (1 second interval) to
2572	* monitor firmware fault and to issue periodic timer sync to
2573	* the firmware.
2574	*
2575	* Return: Nothing.
2576	*/
2577	static void mpi3mr_watchdog_work(struct work_struct *work)
2578	{
2579	struct mpi3mr_ioc *mrioc =
2580	container_of(work, struct mpi3mr_ioc, watchdog_work.work);
2581	unsigned long flags;
2582	enum mpi3mr_iocstate ioc_state;
2583	u32 fault, host_diagnostic, ioc_status;
2584	u32 reset_reason = MPI3MR_RESET_FROM_FAULT_WATCH;
2585
2586	if (mrioc->reset_in_progress)
2587	return;
2588
2589	if (!mrioc->unrecoverable && !pci_device_is_present(pdev: mrioc->pdev)) {
2590	ioc_err(mrioc, "watchdog could not detect the controller\n");
2591	mrioc->unrecoverable = 1;
2592	}
2593
2594	if (mrioc->unrecoverable) {
2595	ioc_err(mrioc,
2596	"flush pending commands for unrecoverable controller\n");
2597	mpi3mr_flush_cmds_for_unrecovered_controller(mrioc);
2598	return;
2599	}
2600
2601	if (mrioc->ts_update_counter++ >= MPI3MR_TSUPDATE_INTERVAL) {
2602	mrioc->ts_update_counter = 0;
2603	mpi3mr_sync_timestamp(mrioc);
2604	}
2605
2606	if ((mrioc->prepare_for_reset) &&
2607	((mrioc->prepare_for_reset_timeout_counter++) >=
2608	MPI3MR_PREPARE_FOR_RESET_TIMEOUT)) {
2609	mpi3mr_soft_reset_handler(mrioc,
2610	reset_reason: MPI3MR_RESET_FROM_CIACTVRST_TIMER, snapdump: 1);
2611	return;
2612	}
2613
2614	ioc_status = readl(addr: &mrioc->sysif_regs->ioc_status);
2615	if (ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY) {
2616	mpi3mr_soft_reset_handler(mrioc, reset_reason: MPI3MR_RESET_FROM_FIRMWARE, snapdump: 0);
2617	return;
2618	}
2619
2620	/*Check for fault state every one second and issue Soft reset*/
2621	ioc_state = mpi3mr_get_iocstate(mrioc);
2622	if (ioc_state != MRIOC_STATE_FAULT)
2623	goto schedule_work;
2624
2625	fault = readl(addr: &mrioc->sysif_regs->fault) & MPI3_SYSIF_FAULT_CODE_MASK;
2626	host_diagnostic = readl(addr: &mrioc->sysif_regs->host_diagnostic);
2627	if (host_diagnostic & MPI3_SYSIF_HOST_DIAG_SAVE_IN_PROGRESS) {
2628	if (!mrioc->diagsave_timeout) {
2629	mpi3mr_print_fault_info(mrioc);
2630	ioc_warn(mrioc, "diag save in progress\n");
2631	}
2632	if ((mrioc->diagsave_timeout++) <= MPI3_SYSIF_DIAG_SAVE_TIMEOUT)
2633	goto schedule_work;
2634	}
2635
2636	mpi3mr_print_fault_info(mrioc);
2637	mrioc->diagsave_timeout = 0;
2638
2639	switch (fault) {
2640	case MPI3_SYSIF_FAULT_CODE_COMPLETE_RESET_NEEDED:
2641	case MPI3_SYSIF_FAULT_CODE_POWER_CYCLE_REQUIRED:
2642	ioc_warn(mrioc,
2643	"controller requires system power cycle, marking controller as unrecoverable\n");
2644	mrioc->unrecoverable = 1;
2645	goto schedule_work;
2646	case MPI3_SYSIF_FAULT_CODE_SOFT_RESET_IN_PROGRESS:
2647	goto schedule_work;
2648	case MPI3_SYSIF_FAULT_CODE_CI_ACTIVATION_RESET:
2649	reset_reason = MPI3MR_RESET_FROM_CIACTIV_FAULT;
2650	break;
2651	default:
2652	break;
2653	}
2654	mpi3mr_soft_reset_handler(mrioc, reset_reason, snapdump: 0);
2655	return;
2656
2657	schedule_work:
2658	spin_lock_irqsave(&mrioc->watchdog_lock, flags);
2659	if (mrioc->watchdog_work_q)
2660	queue_delayed_work(wq: mrioc->watchdog_work_q,
2661	dwork: &mrioc->watchdog_work,
2662	delay: msecs_to_jiffies(MPI3MR_WATCHDOG_INTERVAL));
2663	spin_unlock_irqrestore(lock: &mrioc->watchdog_lock, flags);
2664	return;
2665	}
2666
2667	/**
2668	* mpi3mr_start_watchdog - Start watchdog
2669	* @mrioc: Adapter instance reference
2670	*
2671	* Create and start the watchdog thread to monitor controller
2672	* faults.
2673	*
2674	* Return: Nothing.
2675	*/
2676	void mpi3mr_start_watchdog(struct mpi3mr_ioc *mrioc)
2677	{
2678	if (mrioc->watchdog_work_q)
2679	return;
2680
2681	INIT_DELAYED_WORK(&mrioc->watchdog_work, mpi3mr_watchdog_work);
2682	snprintf(buf: mrioc->watchdog_work_q_name,
2683	size: sizeof(mrioc->watchdog_work_q_name), fmt: "watchdog_%s%d", mrioc->name,
2684	mrioc->id);
2685	mrioc->watchdog_work_q =
2686	create_singlethread_workqueue(mrioc->watchdog_work_q_name);
2687	if (!mrioc->watchdog_work_q) {
2688	ioc_err(mrioc, "%s: failed (line=%d)\n", __func__, __LINE__);
2689	return;
2690	}
2691
2692	if (mrioc->watchdog_work_q)
2693	queue_delayed_work(wq: mrioc->watchdog_work_q,
2694	dwork: &mrioc->watchdog_work,
2695	delay: msecs_to_jiffies(MPI3MR_WATCHDOG_INTERVAL));
2696	}
2697
2698	/**
2699	* mpi3mr_stop_watchdog - Stop watchdog
2700	* @mrioc: Adapter instance reference
2701	*
2702	* Stop the watchdog thread created to monitor controller
2703	* faults.
2704	*
2705	* Return: Nothing.
2706	*/
2707	void mpi3mr_stop_watchdog(struct mpi3mr_ioc *mrioc)
2708	{
2709	unsigned long flags;
2710	struct workqueue_struct *wq;
2711
2712	spin_lock_irqsave(&mrioc->watchdog_lock, flags);
2713	wq = mrioc->watchdog_work_q;
2714	mrioc->watchdog_work_q = NULL;
2715	spin_unlock_irqrestore(lock: &mrioc->watchdog_lock, flags);
2716	if (wq) {
2717	if (!cancel_delayed_work_sync(dwork: &mrioc->watchdog_work))
2718	flush_workqueue(wq);
2719	destroy_workqueue(wq);
2720	}
2721	}
2722
2723	/**
2724	* mpi3mr_setup_admin_qpair - Setup admin queue pair
2725	* @mrioc: Adapter instance reference
2726	*
2727	* Allocate memory for admin queue pair if required and register
2728	* the admin queue with the controller.
2729	*
2730	* Return: 0 on success, non-zero on failures.
2731	*/
2732	static int mpi3mr_setup_admin_qpair(struct mpi3mr_ioc *mrioc)
2733	{
2734	int retval = 0;
2735	u32 num_admin_entries = 0;
2736
2737	mrioc->admin_req_q_sz = MPI3MR_ADMIN_REQ_Q_SIZE;
2738	mrioc->num_admin_req = mrioc->admin_req_q_sz /
2739	MPI3MR_ADMIN_REQ_FRAME_SZ;
2740	mrioc->admin_req_ci = mrioc->admin_req_pi = 0;
2741
2742	mrioc->admin_reply_q_sz = MPI3MR_ADMIN_REPLY_Q_SIZE;
2743	mrioc->num_admin_replies = mrioc->admin_reply_q_sz /
2744	MPI3MR_ADMIN_REPLY_FRAME_SZ;
2745	mrioc->admin_reply_ci = 0;
2746	mrioc->admin_reply_ephase = 1;
2747	atomic_set(v: &mrioc->admin_reply_q_in_use, i: 0);
2748
2749	if (!mrioc->admin_req_base) {
2750	mrioc->admin_req_base = dma_alloc_coherent(dev: &mrioc->pdev->dev,
2751	size: mrioc->admin_req_q_sz, dma_handle: &mrioc->admin_req_dma, GFP_KERNEL);
2752
2753	if (!mrioc->admin_req_base) {
2754	retval = -1;
2755	goto out_failed;
2756	}
2757
2758	mrioc->admin_reply_base = dma_alloc_coherent(dev: &mrioc->pdev->dev,
2759	size: mrioc->admin_reply_q_sz, dma_handle: &mrioc->admin_reply_dma,
2760	GFP_KERNEL);
2761
2762	if (!mrioc->admin_reply_base) {
2763	retval = -1;
2764	goto out_failed;
2765	}
2766	}
2767
2768	num_admin_entries = (mrioc->num_admin_replies << 16) |
2769	(mrioc->num_admin_req);
2770	writel(val: num_admin_entries, addr: &mrioc->sysif_regs->admin_queue_num_entries);
2771	mpi3mr_writeq(b: mrioc->admin_req_dma,
2772	addr: &mrioc->sysif_regs->admin_request_queue_address);
2773	mpi3mr_writeq(b: mrioc->admin_reply_dma,
2774	addr: &mrioc->sysif_regs->admin_reply_queue_address);
2775	writel(val: mrioc->admin_req_pi, addr: &mrioc->sysif_regs->admin_request_queue_pi);
2776	writel(val: mrioc->admin_reply_ci, addr: &mrioc->sysif_regs->admin_reply_queue_ci);
2777	return retval;
2778
2779	out_failed:
2780
2781	if (mrioc->admin_reply_base) {
2782	dma_free_coherent(dev: &mrioc->pdev->dev, size: mrioc->admin_reply_q_sz,
2783	cpu_addr: mrioc->admin_reply_base, dma_handle: mrioc->admin_reply_dma);
2784	mrioc->admin_reply_base = NULL;
2785	}
2786	if (mrioc->admin_req_base) {
2787	dma_free_coherent(dev: &mrioc->pdev->dev, size: mrioc->admin_req_q_sz,
2788	cpu_addr: mrioc->admin_req_base, dma_handle: mrioc->admin_req_dma);
2789	mrioc->admin_req_base = NULL;
2790	}
2791	return retval;
2792	}
2793
2794	/**
2795	* mpi3mr_issue_iocfacts - Send IOC Facts
2796	* @mrioc: Adapter instance reference
2797	* @facts_data: Cached IOC facts data
2798	*
2799	* Issue IOC Facts MPI request through admin queue and wait for
2800	* the completion of it or time out.
2801	*
2802	* Return: 0 on success, non-zero on failures.
2803	*/
2804	static int mpi3mr_issue_iocfacts(struct mpi3mr_ioc *mrioc,
2805	struct mpi3_ioc_facts_data *facts_data)
2806	{
2807	struct mpi3_ioc_facts_request iocfacts_req;
2808	void *data = NULL;
2809	dma_addr_t data_dma;
2810	u32 data_len = sizeof(*facts_data);
2811	int retval = 0;
2812	u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST;
2813
2814	data = dma_alloc_coherent(dev: &mrioc->pdev->dev, size: data_len, dma_handle: &data_dma,
2815	GFP_KERNEL);
2816
2817	if (!data) {
2818	retval = -1;
2819	goto out;
2820	}
2821
2822	memset(&iocfacts_req, 0, sizeof(iocfacts_req));
2823	mutex_lock(&mrioc->init_cmds.mutex);
2824	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2825	retval = -1;
2826	ioc_err(mrioc, "Issue IOCFacts: Init command is in use\n");
2827	mutex_unlock(lock: &mrioc->init_cmds.mutex);
2828	goto out;
2829	}
2830	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2831	mrioc->init_cmds.is_waiting = 1;
2832	mrioc->init_cmds.callback = NULL;
2833	iocfacts_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2834	iocfacts_req.function = MPI3_FUNCTION_IOC_FACTS;
2835
2836	mpi3mr_add_sg_single(paddr: &iocfacts_req.sgl, flags: sgl_flags, length: data_len,
2837	dma_addr: data_dma);
2838
2839	init_completion(x: &mrioc->init_cmds.done);
2840	retval = mpi3mr_admin_request_post(mrioc, admin_req: &iocfacts_req,
2841	admin_req_sz: sizeof(iocfacts_req), ignore_reset: 1);
2842	if (retval) {
2843	ioc_err(mrioc, "Issue IOCFacts: Admin Post failed\n");
2844	goto out_unlock;
2845	}
2846	wait_for_completion_timeout(x: &mrioc->init_cmds.done,
2847	timeout: (MPI3MR_INTADMCMD_TIMEOUT * HZ));
2848	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2849	ioc_err(mrioc, "ioc_facts timed out\n");
2850	mpi3mr_check_rh_fault_ioc(mrioc,
2851	reason_code: MPI3MR_RESET_FROM_IOCFACTS_TIMEOUT);
2852	retval = -1;
2853	goto out_unlock;
2854	}
2855	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2856	!= MPI3_IOCSTATUS_SUCCESS) {
2857	ioc_err(mrioc,
2858	"Issue IOCFacts: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
2859	(mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2860	mrioc->init_cmds.ioc_loginfo);
2861	retval = -1;
2862	goto out_unlock;
2863	}
2864	memcpy(facts_data, (u8 *)data, data_len);
2865	mpi3mr_process_factsdata(mrioc, facts_data);
2866	out_unlock:
2867	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2868	mutex_unlock(lock: &mrioc->init_cmds.mutex);
2869
2870	out:
2871	if (data)
2872	dma_free_coherent(dev: &mrioc->pdev->dev, size: data_len, cpu_addr: data, dma_handle: data_dma);
2873
2874	return retval;
2875	}
2876
2877	/**
2878	* mpi3mr_check_reset_dma_mask - Process IOC facts data
2879	* @mrioc: Adapter instance reference
2880	*
2881	* Check whether the new DMA mask requested through IOCFacts by
2882	* firmware needs to be set, if so set it .
2883	*
2884	* Return: 0 on success, non-zero on failure.
2885	*/
2886	static inline int mpi3mr_check_reset_dma_mask(struct mpi3mr_ioc *mrioc)
2887	{
2888	struct pci_dev *pdev = mrioc->pdev;
2889	int r;
2890	u64 facts_dma_mask = DMA_BIT_MASK(mrioc->facts.dma_mask);
2891
2892	if (!mrioc->facts.dma_mask || (mrioc->dma_mask <= facts_dma_mask))
2893	return 0;
2894
2895	ioc_info(mrioc, "Changing DMA mask from 0x%016llx to 0x%016llx\n",
2896	mrioc->dma_mask, facts_dma_mask);
2897
2898	r = dma_set_mask_and_coherent(dev: &pdev->dev, mask: facts_dma_mask);
2899	if (r) {
2900	ioc_err(mrioc, "Setting DMA mask to 0x%016llx failed: %d\n",
2901	facts_dma_mask, r);
2902	return r;
2903	}
2904	mrioc->dma_mask = facts_dma_mask;
2905	return r;
2906	}
2907
2908	/**
2909	* mpi3mr_process_factsdata - Process IOC facts data
2910	* @mrioc: Adapter instance reference
2911	* @facts_data: Cached IOC facts data
2912	*
2913	* Convert IOC facts data into cpu endianness and cache it in
2914	* the driver .
2915	*
2916	* Return: Nothing.
2917	*/
2918	static void mpi3mr_process_factsdata(struct mpi3mr_ioc *mrioc,
2919	struct mpi3_ioc_facts_data *facts_data)
2920	{
2921	u32 ioc_config, req_sz, facts_flags;
2922
2923	if ((le16_to_cpu(facts_data->ioc_facts_data_length)) !=
2924	(sizeof(*facts_data) / 4)) {
2925	ioc_warn(mrioc,
2926	"IOCFactsdata length mismatch driver_sz(%zu) firmware_sz(%d)\n",
2927	sizeof(*facts_data),
2928	le16_to_cpu(facts_data->ioc_facts_data_length) * 4);
2929	}
2930
2931	ioc_config = readl(addr: &mrioc->sysif_regs->ioc_configuration);
2932	req_sz = 1 << ((ioc_config & MPI3_SYSIF_IOC_CONFIG_OPER_REQ_ENT_SZ) >>
2933	MPI3_SYSIF_IOC_CONFIG_OPER_REQ_ENT_SZ_SHIFT);
2934	if (le16_to_cpu(facts_data->ioc_request_frame_size) != (req_sz / 4)) {
2935	ioc_err(mrioc,
2936	"IOCFacts data reqFrameSize mismatch hw_size(%d) firmware_sz(%d)\n",
2937	req_sz / 4, le16_to_cpu(facts_data->ioc_request_frame_size));
2938	}
2939
2940	memset(&mrioc->facts, 0, sizeof(mrioc->facts));
2941
2942	facts_flags = le32_to_cpu(facts_data->flags);
2943	mrioc->facts.op_req_sz = req_sz;
2944	mrioc->op_reply_desc_sz = 1 << ((ioc_config &
2945	MPI3_SYSIF_IOC_CONFIG_OPER_RPY_ENT_SZ) >>
2946	MPI3_SYSIF_IOC_CONFIG_OPER_RPY_ENT_SZ_SHIFT);
2947
2948	mrioc->facts.ioc_num = facts_data->ioc_number;
2949	mrioc->facts.who_init = facts_data->who_init;
2950	mrioc->facts.max_msix_vectors = le16_to_cpu(facts_data->max_msix_vectors);
2951	mrioc->facts.personality = (facts_flags &
2952	MPI3_IOCFACTS_FLAGS_PERSONALITY_MASK);
2953	mrioc->facts.dma_mask = (facts_flags &
2954	MPI3_IOCFACTS_FLAGS_DMA_ADDRESS_WIDTH_MASK) >>
2955	MPI3_IOCFACTS_FLAGS_DMA_ADDRESS_WIDTH_SHIFT;
2956	mrioc->facts.protocol_flags = facts_data->protocol_flags;
2957	mrioc->facts.mpi_version = le32_to_cpu(facts_data->mpi_version.word);
2958	mrioc->facts.max_reqs = le16_to_cpu(facts_data->max_outstanding_requests);
2959	mrioc->facts.product_id = le16_to_cpu(facts_data->product_id);
2960	mrioc->facts.reply_sz = le16_to_cpu(facts_data->reply_frame_size) * 4;
2961	mrioc->facts.exceptions = le16_to_cpu(facts_data->ioc_exceptions);
2962	mrioc->facts.max_perids = le16_to_cpu(facts_data->max_persistent_id);
2963	mrioc->facts.max_vds = le16_to_cpu(facts_data->max_vds);
2964	mrioc->facts.max_hpds = le16_to_cpu(facts_data->max_host_pds);
2965	mrioc->facts.max_advhpds = le16_to_cpu(facts_data->max_adv_host_pds);
2966	mrioc->facts.max_raid_pds = le16_to_cpu(facts_data->max_raid_pds);
2967	mrioc->facts.max_nvme = le16_to_cpu(facts_data->max_nvme);
2968	mrioc->facts.max_pcie_switches =
2969	le16_to_cpu(facts_data->max_pcie_switches);
2970	mrioc->facts.max_sasexpanders =
2971	le16_to_cpu(facts_data->max_sas_expanders);
2972	mrioc->facts.max_data_length = le16_to_cpu(facts_data->max_data_length);
2973	mrioc->facts.max_sasinitiators =
2974	le16_to_cpu(facts_data->max_sas_initiators);
2975	mrioc->facts.max_enclosures = le16_to_cpu(facts_data->max_enclosures);
2976	mrioc->facts.min_devhandle = le16_to_cpu(facts_data->min_dev_handle);
2977	mrioc->facts.max_devhandle = le16_to_cpu(facts_data->max_dev_handle);
2978	mrioc->facts.max_op_req_q =
2979	le16_to_cpu(facts_data->max_operational_request_queues);
2980	mrioc->facts.max_op_reply_q =
2981	le16_to_cpu(facts_data->max_operational_reply_queues);
2982	mrioc->facts.ioc_capabilities =
2983	le32_to_cpu(facts_data->ioc_capabilities);
2984	mrioc->facts.fw_ver.build_num =
2985	le16_to_cpu(facts_data->fw_version.build_num);
2986	mrioc->facts.fw_ver.cust_id =
2987	le16_to_cpu(facts_data->fw_version.customer_id);
2988	mrioc->facts.fw_ver.ph_minor = facts_data->fw_version.phase_minor;
2989	mrioc->facts.fw_ver.ph_major = facts_data->fw_version.phase_major;
2990	mrioc->facts.fw_ver.gen_minor = facts_data->fw_version.gen_minor;
2991	mrioc->facts.fw_ver.gen_major = facts_data->fw_version.gen_major;
2992	mrioc->msix_count = min_t(int, mrioc->msix_count,
2993	mrioc->facts.max_msix_vectors);
2994	mrioc->facts.sge_mod_mask = facts_data->sge_modifier_mask;
2995	mrioc->facts.sge_mod_value = facts_data->sge_modifier_value;
2996	mrioc->facts.sge_mod_shift = facts_data->sge_modifier_shift;
2997	mrioc->facts.shutdown_timeout =
2998	le16_to_cpu(facts_data->shutdown_timeout);
2999
3000	mrioc->facts.max_dev_per_tg =
3001	facts_data->max_devices_per_throttle_group;
3002	mrioc->facts.io_throttle_data_length =
3003	le16_to_cpu(facts_data->io_throttle_data_length);
3004	mrioc->facts.max_io_throttle_group =
3005	le16_to_cpu(facts_data->max_io_throttle_group);
3006	mrioc->facts.io_throttle_low = le16_to_cpu(facts_data->io_throttle_low);
3007	mrioc->facts.io_throttle_high =
3008	le16_to_cpu(facts_data->io_throttle_high);
3009
3010	if (mrioc->facts.max_data_length ==
3011	MPI3_IOCFACTS_MAX_DATA_LENGTH_NOT_REPORTED)
3012	mrioc->facts.max_data_length = MPI3MR_DEFAULT_MAX_IO_SIZE;
3013	else
3014	mrioc->facts.max_data_length *= MPI3MR_PAGE_SIZE_4K;
3015	/* Store in 512b block count */
3016	if (mrioc->facts.io_throttle_data_length)
3017	mrioc->io_throttle_data_length =
3018	(mrioc->facts.io_throttle_data_length * 2 * 4);
3019	else
3020	/* set the length to 1MB + 1K to disable throttle */
3021	mrioc->io_throttle_data_length = (mrioc->facts.max_data_length / 512) + 2;
3022
3023	mrioc->io_throttle_high = (mrioc->facts.io_throttle_high * 2 * 1024);
3024	mrioc->io_throttle_low = (mrioc->facts.io_throttle_low * 2 * 1024);
3025
3026	ioc_info(mrioc, "ioc_num(%d), maxopQ(%d), maxopRepQ(%d), maxdh(%d),",
3027	mrioc->facts.ioc_num, mrioc->facts.max_op_req_q,
3028	mrioc->facts.max_op_reply_q, mrioc->facts.max_devhandle);
3029	ioc_info(mrioc,
3030	"maxreqs(%d), mindh(%d) maxvectors(%d) maxperids(%d)\n",
3031	mrioc->facts.max_reqs, mrioc->facts.min_devhandle,
3032	mrioc->facts.max_msix_vectors, mrioc->facts.max_perids);
3033	ioc_info(mrioc, "SGEModMask 0x%x SGEModVal 0x%x SGEModShift 0x%x ",
3034	mrioc->facts.sge_mod_mask, mrioc->facts.sge_mod_value,
3035	mrioc->facts.sge_mod_shift);
3036	ioc_info(mrioc, "DMA mask %d InitialPE status 0x%x max_data_len (%d)\n",
3037	mrioc->facts.dma_mask, (facts_flags &
3038	MPI3_IOCFACTS_FLAGS_INITIAL_PORT_ENABLE_MASK), mrioc->facts.max_data_length);
3039	ioc_info(mrioc,
3040	"max_dev_per_throttle_group(%d), max_throttle_groups(%d)\n",
3041	mrioc->facts.max_dev_per_tg, mrioc->facts.max_io_throttle_group);
3042	ioc_info(mrioc,
3043	"io_throttle_data_len(%dKiB), io_throttle_high(%dMiB), io_throttle_low(%dMiB)\n",
3044	mrioc->facts.io_throttle_data_length * 4,
3045	mrioc->facts.io_throttle_high, mrioc->facts.io_throttle_low);
3046	}
3047
3048	/**
3049	* mpi3mr_alloc_reply_sense_bufs - Send IOC Init
3050	* @mrioc: Adapter instance reference
3051	*
3052	* Allocate and initialize the reply free buffers, sense
3053	* buffers, reply free queue and sense buffer queue.
3054	*
3055	* Return: 0 on success, non-zero on failures.
3056	*/
3057	static int mpi3mr_alloc_reply_sense_bufs(struct mpi3mr_ioc *mrioc)
3058	{
3059	int retval = 0;
3060	u32 sz, i;
3061
3062	if (mrioc->init_cmds.reply)
3063	return retval;
3064
3065	mrioc->init_cmds.reply = kzalloc(size: mrioc->reply_sz, GFP_KERNEL);
3066	if (!mrioc->init_cmds.reply)
3067	goto out_failed;
3068
3069	mrioc->bsg_cmds.reply = kzalloc(size: mrioc->reply_sz, GFP_KERNEL);
3070	if (!mrioc->bsg_cmds.reply)
3071	goto out_failed;
3072
3073	mrioc->transport_cmds.reply = kzalloc(size: mrioc->reply_sz, GFP_KERNEL);
3074	if (!mrioc->transport_cmds.reply)
3075	goto out_failed;
3076
3077	for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++) {
3078	mrioc->dev_rmhs_cmds[i].reply = kzalloc(size: mrioc->reply_sz,
3079	GFP_KERNEL);
3080	if (!mrioc->dev_rmhs_cmds[i].reply)
3081	goto out_failed;
3082	}
3083
3084	for (i = 0; i < MPI3MR_NUM_EVTACKCMD; i++) {
3085	mrioc->evtack_cmds[i].reply = kzalloc(size: mrioc->reply_sz,
3086	GFP_KERNEL);
3087	if (!mrioc->evtack_cmds[i].reply)
3088	goto out_failed;
3089	}
3090
3091	mrioc->host_tm_cmds.reply = kzalloc(size: mrioc->reply_sz, GFP_KERNEL);
3092	if (!mrioc->host_tm_cmds.reply)
3093	goto out_failed;
3094
3095	mrioc->pel_cmds.reply = kzalloc(size: mrioc->reply_sz, GFP_KERNEL);
3096	if (!mrioc->pel_cmds.reply)
3097	goto out_failed;
3098
3099	mrioc->pel_abort_cmd.reply = kzalloc(size: mrioc->reply_sz, GFP_KERNEL);
3100	if (!mrioc->pel_abort_cmd.reply)
3101	goto out_failed;
3102
3103	mrioc->dev_handle_bitmap_bits = mrioc->facts.max_devhandle;
3104	mrioc->removepend_bitmap = bitmap_zalloc(nbits: mrioc->dev_handle_bitmap_bits,
3105	GFP_KERNEL);
3106	if (!mrioc->removepend_bitmap)
3107	goto out_failed;
3108
3109	mrioc->devrem_bitmap = bitmap_zalloc(MPI3MR_NUM_DEVRMCMD, GFP_KERNEL);
3110	if (!mrioc->devrem_bitmap)
3111	goto out_failed;
3112
3113	mrioc->evtack_cmds_bitmap = bitmap_zalloc(MPI3MR_NUM_EVTACKCMD,
3114	GFP_KERNEL);
3115	if (!mrioc->evtack_cmds_bitmap)
3116	goto out_failed;
3117
3118	mrioc->num_reply_bufs = mrioc->facts.max_reqs + MPI3MR_NUM_EVT_REPLIES;
3119	mrioc->reply_free_qsz = mrioc->num_reply_bufs + 1;
3120	mrioc->num_sense_bufs = mrioc->facts.max_reqs / MPI3MR_SENSEBUF_FACTOR;
3121	mrioc->sense_buf_q_sz = mrioc->num_sense_bufs + 1;
3122
3123	/* reply buffer pool, 16 byte align */
3124	sz = mrioc->num_reply_bufs * mrioc->reply_sz;
3125	mrioc->reply_buf_pool = dma_pool_create(name: "reply_buf pool",
3126	dev: &mrioc->pdev->dev, size: sz, align: 16, allocation: 0);
3127	if (!mrioc->reply_buf_pool) {
3128	ioc_err(mrioc, "reply buf pool: dma_pool_create failed\n");
3129	goto out_failed;
3130	}
3131
3132	mrioc->reply_buf = dma_pool_zalloc(pool: mrioc->reply_buf_pool, GFP_KERNEL,
3133	handle: &mrioc->reply_buf_dma);
3134	if (!mrioc->reply_buf)
3135	goto out_failed;
3136
3137	mrioc->reply_buf_dma_max_address = mrioc->reply_buf_dma + sz;
3138
3139	/* reply free queue, 8 byte align */
3140	sz = mrioc->reply_free_qsz * 8;
3141	mrioc->reply_free_q_pool = dma_pool_create(name: "reply_free_q pool",
3142	dev: &mrioc->pdev->dev, size: sz, align: 8, allocation: 0);
3143	if (!mrioc->reply_free_q_pool) {
3144	ioc_err(mrioc, "reply_free_q pool: dma_pool_create failed\n");
3145	goto out_failed;
3146	}
3147	mrioc->reply_free_q = dma_pool_zalloc(pool: mrioc->reply_free_q_pool,
3148	GFP_KERNEL, handle: &mrioc->reply_free_q_dma);
3149	if (!mrioc->reply_free_q)
3150	goto out_failed;
3151
3152	/* sense buffer pool, 4 byte align */
3153	sz = mrioc->num_sense_bufs * MPI3MR_SENSE_BUF_SZ;
3154	mrioc->sense_buf_pool = dma_pool_create(name: "sense_buf pool",
3155	dev: &mrioc->pdev->dev, size: sz, align: 4, allocation: 0);
3156	if (!mrioc->sense_buf_pool) {
3157	ioc_err(mrioc, "sense_buf pool: dma_pool_create failed\n");
3158	goto out_failed;
3159	}
3160	mrioc->sense_buf = dma_pool_zalloc(pool: mrioc->sense_buf_pool, GFP_KERNEL,
3161	handle: &mrioc->sense_buf_dma);
3162	if (!mrioc->sense_buf)
3163	goto out_failed;
3164
3165	/* sense buffer queue, 8 byte align */
3166	sz = mrioc->sense_buf_q_sz * 8;
3167	mrioc->sense_buf_q_pool = dma_pool_create(name: "sense_buf_q pool",
3168	dev: &mrioc->pdev->dev, size: sz, align: 8, allocation: 0);
3169	if (!mrioc->sense_buf_q_pool) {
3170	ioc_err(mrioc, "sense_buf_q pool: dma_pool_create failed\n");
3171	goto out_failed;
3172	}
3173	mrioc->sense_buf_q = dma_pool_zalloc(pool: mrioc->sense_buf_q_pool,
3174	GFP_KERNEL, handle: &mrioc->sense_buf_q_dma);
3175	if (!mrioc->sense_buf_q)
3176	goto out_failed;
3177
3178	return retval;
3179
3180	out_failed:
3181	retval = -1;
3182	return retval;
3183	}
3184
3185	/**
3186	* mpimr_initialize_reply_sbuf_queues - initialize reply sense
3187	* buffers
3188	* @mrioc: Adapter instance reference
3189	*
3190	* Helper function to initialize reply and sense buffers along
3191	* with some debug prints.
3192	*
3193	* Return: None.
3194	*/
3195	static void mpimr_initialize_reply_sbuf_queues(struct mpi3mr_ioc *mrioc)
3196	{
3197	u32 sz, i;
3198	dma_addr_t phy_addr;
3199
3200	sz = mrioc->num_reply_bufs * mrioc->reply_sz;
3201	ioc_info(mrioc,
3202	"reply buf pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB), reply_dma(0x%llx)\n",
3203	mrioc->reply_buf, mrioc->num_reply_bufs, mrioc->reply_sz,
3204	(sz / 1024), (unsigned long long)mrioc->reply_buf_dma);
3205	sz = mrioc->reply_free_qsz * 8;
3206	ioc_info(mrioc,
3207	"reply_free_q pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB), reply_dma(0x%llx)\n",
3208	mrioc->reply_free_q, mrioc->reply_free_qsz, 8, (sz / 1024),
3209	(unsigned long long)mrioc->reply_free_q_dma);
3210	sz = mrioc->num_sense_bufs * MPI3MR_SENSE_BUF_SZ;
3211	ioc_info(mrioc,
3212	"sense_buf pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB), sense_dma(0x%llx)\n",
3213	mrioc->sense_buf, mrioc->num_sense_bufs, MPI3MR_SENSE_BUF_SZ,
3214	(sz / 1024), (unsigned long long)mrioc->sense_buf_dma);
3215	sz = mrioc->sense_buf_q_sz * 8;
3216	ioc_info(mrioc,
3217	"sense_buf_q pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB), sense_dma(0x%llx)\n",
3218	mrioc->sense_buf_q, mrioc->sense_buf_q_sz, 8, (sz / 1024),
3219	(unsigned long long)mrioc->sense_buf_q_dma);
3220
3221	/* initialize Reply buffer Queue */
3222	for (i = 0, phy_addr = mrioc->reply_buf_dma;
3223	i < mrioc->num_reply_bufs; i++, phy_addr += mrioc->reply_sz)
3224	mrioc->reply_free_q[i] = cpu_to_le64(phy_addr);
3225	mrioc->reply_free_q[i] = cpu_to_le64(0);
3226
3227	/* initialize Sense Buffer Queue */
3228	for (i = 0, phy_addr = mrioc->sense_buf_dma;
3229	i < mrioc->num_sense_bufs; i++, phy_addr += MPI3MR_SENSE_BUF_SZ)
3230	mrioc->sense_buf_q[i] = cpu_to_le64(phy_addr);
3231	mrioc->sense_buf_q[i] = cpu_to_le64(0);
3232	}
3233
3234	/**
3235	* mpi3mr_issue_iocinit - Send IOC Init
3236	* @mrioc: Adapter instance reference
3237	*
3238	* Issue IOC Init MPI request through admin queue and wait for
3239	* the completion of it or time out.
3240	*
3241	* Return: 0 on success, non-zero on failures.
3242	*/
3243	static int mpi3mr_issue_iocinit(struct mpi3mr_ioc *mrioc)
3244	{
3245	struct mpi3_ioc_init_request iocinit_req;
3246	struct mpi3_driver_info_layout *drv_info;
3247	dma_addr_t data_dma;
3248	u32 data_len = sizeof(*drv_info);
3249	int retval = 0;
3250	ktime_t current_time;
3251
3252	drv_info = dma_alloc_coherent(dev: &mrioc->pdev->dev, size: data_len, dma_handle: &data_dma,
3253	GFP_KERNEL);
3254	if (!drv_info) {
3255	retval = -1;
3256	goto out;
3257	}
3258	mpimr_initialize_reply_sbuf_queues(mrioc);
3259
3260	drv_info->information_length = cpu_to_le32(data_len);
3261	strscpy(drv_info->driver_signature, "Broadcom", sizeof(drv_info->driver_signature));
3262	strscpy(drv_info->os_name, utsname()->sysname, sizeof(drv_info->os_name));
3263	strscpy(drv_info->os_version, utsname()->release, sizeof(drv_info->os_version));
3264	strscpy(drv_info->driver_name, MPI3MR_DRIVER_NAME, sizeof(drv_info->driver_name));
3265	strscpy(drv_info->driver_version, MPI3MR_DRIVER_VERSION, sizeof(drv_info->driver_version));
3266	strscpy(drv_info->driver_release_date, MPI3MR_DRIVER_RELDATE,
3267	sizeof(drv_info->driver_release_date));
3268	drv_info->driver_capabilities = 0;
3269	memcpy((u8 *)&mrioc->driver_info, (u8 *)drv_info,
3270	sizeof(mrioc->driver_info));
3271
3272	memset(&iocinit_req, 0, sizeof(iocinit_req));
3273	mutex_lock(&mrioc->init_cmds.mutex);
3274	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
3275	retval = -1;
3276	ioc_err(mrioc, "Issue IOCInit: Init command is in use\n");
3277	mutex_unlock(lock: &mrioc->init_cmds.mutex);
3278	goto out;
3279	}
3280	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
3281	mrioc->init_cmds.is_waiting = 1;
3282	mrioc->init_cmds.callback = NULL;
3283	iocinit_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
3284	iocinit_req.function = MPI3_FUNCTION_IOC_INIT;
3285	iocinit_req.mpi_version.mpi3_version.dev = MPI3_VERSION_DEV;
3286	iocinit_req.mpi_version.mpi3_version.unit = MPI3_VERSION_UNIT;
3287	iocinit_req.mpi_version.mpi3_version.major = MPI3_VERSION_MAJOR;
3288	iocinit_req.mpi_version.mpi3_version.minor = MPI3_VERSION_MINOR;
3289	iocinit_req.who_init = MPI3_WHOINIT_HOST_DRIVER;
3290	iocinit_req.reply_free_queue_depth = cpu_to_le16(mrioc->reply_free_qsz);
3291	iocinit_req.reply_free_queue_address =
3292	cpu_to_le64(mrioc->reply_free_q_dma);
3293	iocinit_req.sense_buffer_length = cpu_to_le16(MPI3MR_SENSE_BUF_SZ);
3294	iocinit_req.sense_buffer_free_queue_depth =
3295	cpu_to_le16(mrioc->sense_buf_q_sz);
3296	iocinit_req.sense_buffer_free_queue_address =
3297	cpu_to_le64(mrioc->sense_buf_q_dma);
3298	iocinit_req.driver_information_address = cpu_to_le64(data_dma);
3299
3300	current_time = ktime_get_real();
3301	iocinit_req.time_stamp = cpu_to_le64(ktime_to_ms(current_time));
3302
3303	iocinit_req.msg_flags |=
3304	MPI3_IOCINIT_MSGFLAGS_SCSIIOSTATUSREPLY_SUPPORTED;
3305
3306	init_completion(x: &mrioc->init_cmds.done);
3307	retval = mpi3mr_admin_request_post(mrioc, admin_req: &iocinit_req,
3308	admin_req_sz: sizeof(iocinit_req), ignore_reset: 1);
3309	if (retval) {
3310	ioc_err(mrioc, "Issue IOCInit: Admin Post failed\n");
3311	goto out_unlock;
3312	}
3313	wait_for_completion_timeout(x: &mrioc->init_cmds.done,
3314	timeout: (MPI3MR_INTADMCMD_TIMEOUT * HZ));
3315	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
3316	mpi3mr_check_rh_fault_ioc(mrioc,
3317	reason_code: MPI3MR_RESET_FROM_IOCINIT_TIMEOUT);
3318	ioc_err(mrioc, "ioc_init timed out\n");
3319	retval = -1;
3320	goto out_unlock;
3321	}
3322	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
3323	!= MPI3_IOCSTATUS_SUCCESS) {
3324	ioc_err(mrioc,
3325	"Issue IOCInit: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
3326	(mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
3327	mrioc->init_cmds.ioc_loginfo);
3328	retval = -1;
3329	goto out_unlock;
3330	}
3331
3332	mrioc->reply_free_queue_host_index = mrioc->num_reply_bufs;
3333	writel(val: mrioc->reply_free_queue_host_index,
3334	addr: &mrioc->sysif_regs->reply_free_host_index);
3335
3336	mrioc->sbq_host_index = mrioc->num_sense_bufs;
3337	writel(val: mrioc->sbq_host_index,
3338	addr: &mrioc->sysif_regs->sense_buffer_free_host_index);
3339	out_unlock:
3340	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
3341	mutex_unlock(lock: &mrioc->init_cmds.mutex);
3342
3343	out:
3344	if (drv_info)
3345	dma_free_coherent(dev: &mrioc->pdev->dev, size: data_len, cpu_addr: drv_info,
3346	dma_handle: data_dma);
3347
3348	return retval;
3349	}
3350
3351	/**
3352	* mpi3mr_unmask_events - Unmask events in event mask bitmap
3353	* @mrioc: Adapter instance reference
3354	* @event: MPI event ID
3355	*
3356	* Un mask the specific event by resetting the event_mask
3357	* bitmap.
3358	*
3359	* Return: 0 on success, non-zero on failures.
3360	*/
3361	static void mpi3mr_unmask_events(struct mpi3mr_ioc *mrioc, u16 event)
3362	{
3363	u32 desired_event;
3364	u8 word;
3365
3366	if (event >= 128)
3367	return;
3368
3369	desired_event = (1 << (event % 32));
3370	word = event / 32;
3371
3372	mrioc->event_masks[word] &= ~desired_event;
3373	}
3374
3375	/**
3376	* mpi3mr_issue_event_notification - Send event notification
3377	* @mrioc: Adapter instance reference
3378	*
3379	* Issue event notification MPI request through admin queue and
3380	* wait for the completion of it or time out.
3381	*
3382	* Return: 0 on success, non-zero on failures.
3383	*/
3384	static int mpi3mr_issue_event_notification(struct mpi3mr_ioc *mrioc)
3385	{
3386	struct mpi3_event_notification_request evtnotify_req;
3387	int retval = 0;
3388	u8 i;
3389
3390	memset(&evtnotify_req, 0, sizeof(evtnotify_req));
3391	mutex_lock(&mrioc->init_cmds.mutex);
3392	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
3393	retval = -1;
3394	ioc_err(mrioc, "Issue EvtNotify: Init command is in use\n");
3395	mutex_unlock(lock: &mrioc->init_cmds.mutex);
3396	goto out;
3397	}
3398	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
3399	mrioc->init_cmds.is_waiting = 1;
3400	mrioc->init_cmds.callback = NULL;
3401	evtnotify_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
3402	evtnotify_req.function = MPI3_FUNCTION_EVENT_NOTIFICATION;
3403	for (i = 0; i < MPI3_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
3404	evtnotify_req.event_masks[i] =
3405	cpu_to_le32(mrioc->event_masks[i]);
3406	init_completion(x: &mrioc->init_cmds.done);
3407	retval = mpi3mr_admin_request_post(mrioc, admin_req: &evtnotify_req,
3408	admin_req_sz: sizeof(evtnotify_req), ignore_reset: 1);
3409	if (retval) {
3410	ioc_err(mrioc, "Issue EvtNotify: Admin Post failed\n");
3411	goto out_unlock;
3412	}
3413	wait_for_completion_timeout(x: &mrioc->init_cmds.done,
3414	timeout: (MPI3MR_INTADMCMD_TIMEOUT * HZ));
3415	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
3416	ioc_err(mrioc, "event notification timed out\n");
3417	mpi3mr_check_rh_fault_ioc(mrioc,
3418	reason_code: MPI3MR_RESET_FROM_EVTNOTIFY_TIMEOUT);
3419	retval = -1;
3420	goto out_unlock;
3421	}
3422	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
3423	!= MPI3_IOCSTATUS_SUCCESS) {
3424	ioc_err(mrioc,
3425	"Issue EvtNotify: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
3426	(mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
3427	mrioc->init_cmds.ioc_loginfo);
3428	retval = -1;
3429	goto out_unlock;
3430	}
3431
3432	out_unlock:
3433	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
3434	mutex_unlock(lock: &mrioc->init_cmds.mutex);
3435	out:
3436	return retval;
3437	}
3438
3439	/**
3440	* mpi3mr_process_event_ack - Process event acknowledgment
3441	* @mrioc: Adapter instance reference
3442	* @event: MPI3 event ID
3443	* @event_ctx: event context
3444	*
3445	* Send event acknowledgment through admin queue and wait for
3446	* it to complete.
3447	*
3448	* Return: 0 on success, non-zero on failures.
3449	*/
3450	int mpi3mr_process_event_ack(struct mpi3mr_ioc *mrioc, u8 event,
3451	u32 event_ctx)
3452	{
3453	struct mpi3_event_ack_request evtack_req;
3454	int retval = 0;
3455
3456	memset(&evtack_req, 0, sizeof(evtack_req));
3457	mutex_lock(&mrioc->init_cmds.mutex);
3458	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
3459	retval = -1;
3460	ioc_err(mrioc, "Send EvtAck: Init command is in use\n");
3461	mutex_unlock(lock: &mrioc->init_cmds.mutex);
3462	goto out;
3463	}
3464	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
3465	mrioc->init_cmds.is_waiting = 1;
3466	mrioc->init_cmds.callback = NULL;
3467	evtack_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
3468	evtack_req.function = MPI3_FUNCTION_EVENT_ACK;
3469	evtack_req.event = event;
3470	evtack_req.event_context = cpu_to_le32(event_ctx);
3471
3472	init_completion(x: &mrioc->init_cmds.done);
3473	retval = mpi3mr_admin_request_post(mrioc, admin_req: &evtack_req,
3474	admin_req_sz: sizeof(evtack_req), ignore_reset: 1);
3475	if (retval) {
3476	ioc_err(mrioc, "Send EvtAck: Admin Post failed\n");
3477	goto out_unlock;
3478	}
3479	wait_for_completion_timeout(x: &mrioc->init_cmds.done,
3480	timeout: (MPI3MR_INTADMCMD_TIMEOUT * HZ));
3481	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
3482	ioc_err(mrioc, "Issue EvtNotify: command timed out\n");
3483	if (!(mrioc->init_cmds.state & MPI3MR_CMD_RESET))
3484	mpi3mr_check_rh_fault_ioc(mrioc,
3485	reason_code: MPI3MR_RESET_FROM_EVTACK_TIMEOUT);
3486	retval = -1;
3487	goto out_unlock;
3488	}
3489	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
3490	!= MPI3_IOCSTATUS_SUCCESS) {
3491	ioc_err(mrioc,
3492	"Send EvtAck: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
3493	(mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
3494	mrioc->init_cmds.ioc_loginfo);
3495	retval = -1;
3496	goto out_unlock;
3497	}
3498
3499	out_unlock:
3500	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
3501	mutex_unlock(lock: &mrioc->init_cmds.mutex);
3502	out:
3503	return retval;
3504	}
3505
3506	/**
3507	* mpi3mr_alloc_chain_bufs - Allocate chain buffers
3508	* @mrioc: Adapter instance reference
3509	*
3510	* Allocate chain buffers and set a bitmap to indicate free
3511	* chain buffers. Chain buffers are used to pass the SGE
3512	* information along with MPI3 SCSI IO requests for host I/O.
3513	*
3514	* Return: 0 on success, non-zero on failure
3515	*/
3516	static int mpi3mr_alloc_chain_bufs(struct mpi3mr_ioc *mrioc)
3517	{
3518	int retval = 0;
3519	u32 sz, i;
3520	u16 num_chains;
3521
3522	if (mrioc->chain_sgl_list)
3523	return retval;
3524
3525	num_chains = mrioc->max_host_ios / MPI3MR_CHAINBUF_FACTOR;
3526
3527	if (prot_mask & (SHOST_DIX_TYPE0_PROTECTION
3528	| SHOST_DIX_TYPE1_PROTECTION
3529	| SHOST_DIX_TYPE2_PROTECTION
3530	| SHOST_DIX_TYPE3_PROTECTION))
3531	num_chains += (num_chains / MPI3MR_CHAINBUFDIX_FACTOR);
3532
3533	mrioc->chain_buf_count = num_chains;
3534	sz = sizeof(struct chain_element) * num_chains;
3535	mrioc->chain_sgl_list = kzalloc(size: sz, GFP_KERNEL);
3536	if (!mrioc->chain_sgl_list)
3537	goto out_failed;
3538
3539	if (mrioc->max_sgl_entries > (mrioc->facts.max_data_length /
3540	MPI3MR_PAGE_SIZE_4K))
3541	mrioc->max_sgl_entries = mrioc->facts.max_data_length /
3542	MPI3MR_PAGE_SIZE_4K;
3543	sz = mrioc->max_sgl_entries * sizeof(struct mpi3_sge_common);
3544	ioc_info(mrioc, "number of sgl entries=%d chain buffer size=%dKB\n",
3545	mrioc->max_sgl_entries, sz/1024);
3546
3547	mrioc->chain_buf_pool = dma_pool_create(name: "chain_buf pool",
3548	dev: &mrioc->pdev->dev, size: sz, align: 16, allocation: 0);
3549	if (!mrioc->chain_buf_pool) {
3550	ioc_err(mrioc, "chain buf pool: dma_pool_create failed\n");
3551	goto out_failed;
3552	}
3553
3554	for (i = 0; i < num_chains; i++) {
3555	mrioc->chain_sgl_list[i].addr =
3556	dma_pool_zalloc(pool: mrioc->chain_buf_pool, GFP_KERNEL,
3557	handle: &mrioc->chain_sgl_list[i].dma_addr);
3558
3559	if (!mrioc->chain_sgl_list[i].addr)
3560	goto out_failed;
3561	}
3562	mrioc->chain_bitmap = bitmap_zalloc(nbits: num_chains, GFP_KERNEL);
3563	if (!mrioc->chain_bitmap)
3564	goto out_failed;
3565	return retval;
3566	out_failed:
3567	retval = -1;
3568	return retval;
3569	}
3570
3571	/**
3572	* mpi3mr_port_enable_complete - Mark port enable complete
3573	* @mrioc: Adapter instance reference
3574	* @drv_cmd: Internal command tracker
3575	*
3576	* Call back for asynchronous port enable request sets the
3577	* driver command to indicate port enable request is complete.
3578	*
3579	* Return: Nothing
3580	*/
3581	static void mpi3mr_port_enable_complete(struct mpi3mr_ioc *mrioc,
3582	struct mpi3mr_drv_cmd *drv_cmd)
3583	{
3584	drv_cmd->callback = NULL;
3585	mrioc->scan_started = 0;
3586	if (drv_cmd->state & MPI3MR_CMD_RESET)
3587	mrioc->scan_failed = MPI3_IOCSTATUS_INTERNAL_ERROR;
3588	else
3589	mrioc->scan_failed = drv_cmd->ioc_status;
3590	drv_cmd->state = MPI3MR_CMD_NOTUSED;
3591	}
3592
3593	/**
3594	* mpi3mr_issue_port_enable - Issue Port Enable
3595	* @mrioc: Adapter instance reference
3596	* @async: Flag to wait for completion or not
3597	*
3598	* Issue Port Enable MPI request through admin queue and if the
3599	* async flag is not set wait for the completion of the port
3600	* enable or time out.
3601	*
3602	* Return: 0 on success, non-zero on failures.
3603	*/
3604	int mpi3mr_issue_port_enable(struct mpi3mr_ioc *mrioc, u8 async)
3605	{
3606	struct mpi3_port_enable_request pe_req;
3607	int retval = 0;
3608	u32 pe_timeout = MPI3MR_PORTENABLE_TIMEOUT;
3609
3610	memset(&pe_req, 0, sizeof(pe_req));
3611	mutex_lock(&mrioc->init_cmds.mutex);
3612	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
3613	retval = -1;
3614	ioc_err(mrioc, "Issue PortEnable: Init command is in use\n");
3615	mutex_unlock(lock: &mrioc->init_cmds.mutex);
3616	goto out;
3617	}
3618	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
3619	if (async) {
3620	mrioc->init_cmds.is_waiting = 0;
3621	mrioc->init_cmds.callback = mpi3mr_port_enable_complete;
3622	} else {
3623	mrioc->init_cmds.is_waiting = 1;
3624	mrioc->init_cmds.callback = NULL;
3625	init_completion(x: &mrioc->init_cmds.done);
3626	}
3627	pe_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
3628	pe_req.function = MPI3_FUNCTION_PORT_ENABLE;
3629
3630	retval = mpi3mr_admin_request_post(mrioc, admin_req: &pe_req, admin_req_sz: sizeof(pe_req), ignore_reset: 1);
3631	if (retval) {
3632	ioc_err(mrioc, "Issue PortEnable: Admin Post failed\n");
3633	goto out_unlock;
3634	}
3635	if (async) {
3636	mutex_unlock(lock: &mrioc->init_cmds.mutex);
3637	goto out;
3638	}
3639
3640	wait_for_completion_timeout(x: &mrioc->init_cmds.done, timeout: (pe_timeout * HZ));
3641	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
3642	ioc_err(mrioc, "port enable timed out\n");
3643	retval = -1;
3644	mpi3mr_check_rh_fault_ioc(mrioc, reason_code: MPI3MR_RESET_FROM_PE_TIMEOUT);
3645	goto out_unlock;
3646	}
3647	mpi3mr_port_enable_complete(mrioc, drv_cmd: &mrioc->init_cmds);
3648
3649	out_unlock:
3650	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
3651	mutex_unlock(lock: &mrioc->init_cmds.mutex);
3652	out:
3653	return retval;
3654	}
3655
3656	/* Protocol type to name mapper structure */
3657	static const struct {
3658	u8 protocol;
3659	char *name;
3660	} mpi3mr_protocols[] = {
3661	{ MPI3_IOCFACTS_PROTOCOL_SCSI_INITIATOR, "Initiator" },
3662	{ MPI3_IOCFACTS_PROTOCOL_SCSI_TARGET, "Target" },
3663	{ MPI3_IOCFACTS_PROTOCOL_NVME, "NVMe attachment" },
3664	};
3665
3666	/* Capability to name mapper structure*/
3667	static const struct {
3668	u32 capability;
3669	char *name;
3670	} mpi3mr_capabilities[] = {
3671	{ MPI3_IOCFACTS_CAPABILITY_RAID_CAPABLE, "RAID" },
3672	{ MPI3_IOCFACTS_CAPABILITY_MULTIPATH_ENABLED, "MultiPath" },
3673	};
3674
3675	/**
3676	* mpi3mr_print_ioc_info - Display controller information
3677	* @mrioc: Adapter instance reference
3678	*
3679	* Display controller personalit, capability, supported
3680	* protocols etc.
3681	*
3682	* Return: Nothing
3683	*/
3684	static void
3685	mpi3mr_print_ioc_info(struct mpi3mr_ioc *mrioc)
3686	{
3687	int i = 0, bytes_written = 0;
3688	char personality[16];
3689	char protocol[50] = {0};
3690	char capabilities[100] = {0};
3691	struct mpi3mr_compimg_ver *fwver = &mrioc->facts.fw_ver;
3692
3693	switch (mrioc->facts.personality) {
3694	case MPI3_IOCFACTS_FLAGS_PERSONALITY_EHBA:
3695	strncpy(p: personality, q: "Enhanced HBA", size: sizeof(personality));
3696	break;
3697	case MPI3_IOCFACTS_FLAGS_PERSONALITY_RAID_DDR:
3698	strncpy(p: personality, q: "RAID", size: sizeof(personality));
3699	break;
3700	default:
3701	strncpy(p: personality, q: "Unknown", size: sizeof(personality));
3702	break;
3703	}
3704
3705	ioc_info(mrioc, "Running in %s Personality", personality);
3706
3707	ioc_info(mrioc, "FW version(%d.%d.%d.%d.%d.%d)\n",
3708	fwver->gen_major, fwver->gen_minor, fwver->ph_major,
3709	fwver->ph_minor, fwver->cust_id, fwver->build_num);
3710
3711	for (i = 0; i < ARRAY_SIZE(mpi3mr_protocols); i++) {
3712	if (mrioc->facts.protocol_flags &
3713	mpi3mr_protocols[i].protocol) {
3714	bytes_written += scnprintf(buf: protocol + bytes_written,
3715	size: sizeof(protocol) - bytes_written, fmt: "%s%s",
3716	bytes_written ? "," : "",
3717	mpi3mr_protocols[i].name);
3718	}
3719	}
3720
3721	bytes_written = 0;
3722	for (i = 0; i < ARRAY_SIZE(mpi3mr_capabilities); i++) {
3723	if (mrioc->facts.protocol_flags &
3724	mpi3mr_capabilities[i].capability) {
3725	bytes_written += scnprintf(buf: capabilities + bytes_written,
3726	size: sizeof(capabilities) - bytes_written, fmt: "%s%s",
3727	bytes_written ? "," : "",
3728	mpi3mr_capabilities[i].name);
3729	}
3730	}
3731
3732	ioc_info(mrioc, "Protocol=(%s), Capabilities=(%s)\n",
3733	protocol, capabilities);
3734	}
3735
3736	/**
3737	* mpi3mr_cleanup_resources - Free PCI resources
3738	* @mrioc: Adapter instance reference
3739	*
3740	* Unmap PCI device memory and disable PCI device.
3741	*
3742	* Return: 0 on success and non-zero on failure.
3743	*/
3744	void mpi3mr_cleanup_resources(struct mpi3mr_ioc *mrioc)
3745	{
3746	struct pci_dev *pdev = mrioc->pdev;
3747
3748	mpi3mr_cleanup_isr(mrioc);
3749
3750	if (mrioc->sysif_regs) {
3751	iounmap(addr: (void __iomem *)mrioc->sysif_regs);
3752	mrioc->sysif_regs = NULL;
3753	}
3754
3755	if (pci_is_enabled(pdev)) {
3756	if (mrioc->bars)
3757	pci_release_selected_regions(pdev, mrioc->bars);
3758	pci_disable_device(dev: pdev);
3759	}
3760	}
3761
3762	/**
3763	* mpi3mr_setup_resources - Enable PCI resources
3764	* @mrioc: Adapter instance reference
3765	*
3766	* Enable PCI device memory, MSI-x registers and set DMA mask.
3767	*
3768	* Return: 0 on success and non-zero on failure.
3769	*/
3770	int mpi3mr_setup_resources(struct mpi3mr_ioc *mrioc)
3771	{
3772	struct pci_dev *pdev = mrioc->pdev;
3773	u32 memap_sz = 0;
3774	int i, retval = 0, capb = 0;
3775	u16 message_control;
3776	u64 dma_mask = mrioc->dma_mask ? mrioc->dma_mask :
3777	((sizeof(dma_addr_t) > 4) ? DMA_BIT_MASK(64) : DMA_BIT_MASK(32));
3778
3779	if (pci_enable_device_mem(dev: pdev)) {
3780	ioc_err(mrioc, "pci_enable_device_mem: failed\n");
3781	retval = -ENODEV;
3782	goto out_failed;
3783	}
3784
3785	capb = pci_find_capability(dev: pdev, PCI_CAP_ID_MSIX);
3786	if (!capb) {
3787	ioc_err(mrioc, "Unable to find MSI-X Capabilities\n");
3788	retval = -ENODEV;
3789	goto out_failed;
3790	}
3791	mrioc->bars = pci_select_bars(dev: pdev, IORESOURCE_MEM);
3792
3793	if (pci_request_selected_regions(pdev, mrioc->bars,
3794	mrioc->driver_name)) {
3795	ioc_err(mrioc, "pci_request_selected_regions: failed\n");
3796	retval = -ENODEV;
3797	goto out_failed;
3798	}
3799
3800	for (i = 0; (i < DEVICE_COUNT_RESOURCE); i++) {
3801	if (pci_resource_flags(pdev, i) & IORESOURCE_MEM) {
3802	mrioc->sysif_regs_phys = pci_resource_start(pdev, i);
3803	memap_sz = pci_resource_len(pdev, i);
3804	mrioc->sysif_regs =
3805	ioremap(offset: mrioc->sysif_regs_phys, size: memap_sz);
3806	break;
3807	}
3808	}
3809
3810	pci_set_master(dev: pdev);
3811
3812	retval = dma_set_mask_and_coherent(dev: &pdev->dev, mask: dma_mask);
3813	if (retval) {
3814	if (dma_mask != DMA_BIT_MASK(32)) {
3815	ioc_warn(mrioc, "Setting 64 bit DMA mask failed\n");
3816	dma_mask = DMA_BIT_MASK(32);
3817	retval = dma_set_mask_and_coherent(dev: &pdev->dev,
3818	mask: dma_mask);
3819	}
3820	if (retval) {
3821	mrioc->dma_mask = 0;
3822	ioc_err(mrioc, "Setting 32 bit DMA mask also failed\n");
3823	goto out_failed;
3824	}
3825	}
3826	mrioc->dma_mask = dma_mask;
3827
3828	if (!mrioc->sysif_regs) {
3829	ioc_err(mrioc,
3830	"Unable to map adapter memory or resource not found\n");
3831	retval = -EINVAL;
3832	goto out_failed;
3833	}
3834
3835	pci_read_config_word(dev: pdev, where: capb + 2, val: &message_control);
3836	mrioc->msix_count = (message_control & 0x3FF) + 1;
3837
3838	pci_save_state(dev: pdev);
3839
3840	pci_set_drvdata(pdev, data: mrioc->shost);
3841
3842	mpi3mr_ioc_disable_intr(mrioc);
3843
3844	ioc_info(mrioc, "iomem(0x%016llx), mapped(0x%p), size(%d)\n",
3845	(unsigned long long)mrioc->sysif_regs_phys,
3846	mrioc->sysif_regs, memap_sz);
3847	ioc_info(mrioc, "Number of MSI-X vectors found in capabilities: (%d)\n",
3848	mrioc->msix_count);
3849
3850	if (!reset_devices && poll_queues > 0)
3851	mrioc->requested_poll_qcount = min_t(int, poll_queues,
3852	mrioc->msix_count - 2);
3853	return retval;
3854
3855	out_failed:
3856	mpi3mr_cleanup_resources(mrioc);
3857	return retval;
3858	}
3859
3860	/**
3861	* mpi3mr_enable_events - Enable required events
3862	* @mrioc: Adapter instance reference
3863	*
3864	* This routine unmasks the events required by the driver by
3865	* sennding appropriate event mask bitmapt through an event
3866	* notification request.
3867	*
3868	* Return: 0 on success and non-zero on failure.
3869	*/
3870	static int mpi3mr_enable_events(struct mpi3mr_ioc *mrioc)
3871	{
3872	int retval = 0;
3873	u32 i;
3874
3875	for (i = 0; i < MPI3_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
3876	mrioc->event_masks[i] = -1;
3877
3878	mpi3mr_unmask_events(mrioc, MPI3_EVENT_DEVICE_ADDED);
3879	mpi3mr_unmask_events(mrioc, MPI3_EVENT_DEVICE_INFO_CHANGED);
3880	mpi3mr_unmask_events(mrioc, MPI3_EVENT_DEVICE_STATUS_CHANGE);
3881	mpi3mr_unmask_events(mrioc, MPI3_EVENT_ENCL_DEVICE_STATUS_CHANGE);
3882	mpi3mr_unmask_events(mrioc, MPI3_EVENT_ENCL_DEVICE_ADDED);
3883	mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_TOPOLOGY_CHANGE_LIST);
3884	mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_DISCOVERY);
3885	mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_DEVICE_DISCOVERY_ERROR);
3886	mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_BROADCAST_PRIMITIVE);
3887	mpi3mr_unmask_events(mrioc, MPI3_EVENT_PCIE_TOPOLOGY_CHANGE_LIST);
3888	mpi3mr_unmask_events(mrioc, MPI3_EVENT_PCIE_ENUMERATION);
3889	mpi3mr_unmask_events(mrioc, MPI3_EVENT_PREPARE_FOR_RESET);
3890	mpi3mr_unmask_events(mrioc, MPI3_EVENT_CABLE_MGMT);
3891	mpi3mr_unmask_events(mrioc, MPI3_EVENT_ENERGY_PACK_CHANGE);
3892
3893	retval = mpi3mr_issue_event_notification(mrioc);
3894	if (retval)
3895	ioc_err(mrioc, "failed to issue event notification %d\n",
3896	retval);
3897	return retval;
3898	}
3899
3900	/**
3901	* mpi3mr_init_ioc - Initialize the controller
3902	* @mrioc: Adapter instance reference
3903	*
3904	* This the controller initialization routine, executed either
3905	* after soft reset or from pci probe callback.
3906	* Setup the required resources, memory map the controller
3907	* registers, create admin and operational reply queue pairs,
3908	* allocate required memory for reply pool, sense buffer pool,
3909	* issue IOC init request to the firmware, unmask the events and
3910	* issue port enable to discover SAS/SATA/NVMe devies and RAID
3911	* volumes.
3912	*
3913	* Return: 0 on success and non-zero on failure.
3914	*/
3915	int mpi3mr_init_ioc(struct mpi3mr_ioc *mrioc)
3916	{
3917	int retval = 0;
3918	u8 retry = 0;
3919	struct mpi3_ioc_facts_data facts_data;
3920	u32 sz;
3921
3922	retry_init:
3923	retval = mpi3mr_bring_ioc_ready(mrioc);
3924	if (retval) {
3925	ioc_err(mrioc, "Failed to bring ioc ready: error %d\n",
3926	retval);
3927	goto out_failed_noretry;
3928	}
3929
3930	retval = mpi3mr_setup_isr(mrioc, setup_one: 1);
3931	if (retval) {
3932	ioc_err(mrioc, "Failed to setup ISR error %d\n",
3933	retval);
3934	goto out_failed_noretry;
3935	}
3936
3937	retval = mpi3mr_issue_iocfacts(mrioc, facts_data: &facts_data);
3938	if (retval) {
3939	ioc_err(mrioc, "Failed to Issue IOC Facts %d\n",
3940	retval);
3941	goto out_failed;
3942	}
3943
3944	mrioc->max_host_ios = mrioc->facts.max_reqs - MPI3MR_INTERNAL_CMDS_RESVD;
3945	mrioc->shost->max_sectors = mrioc->facts.max_data_length / 512;
3946	mrioc->num_io_throttle_group = mrioc->facts.max_io_throttle_group;
3947	atomic_set(v: &mrioc->pend_large_data_sz, i: 0);
3948
3949	if (reset_devices)
3950	mrioc->max_host_ios = min_t(int, mrioc->max_host_ios,
3951	MPI3MR_HOST_IOS_KDUMP);
3952
3953	if (!(mrioc->facts.ioc_capabilities &
3954	MPI3_IOCFACTS_CAPABILITY_MULTIPATH_ENABLED)) {
3955	mrioc->sas_transport_enabled = 1;
3956	mrioc->scsi_device_channel = 1;
3957	mrioc->shost->max_channel = 1;
3958	mrioc->shost->transportt = mpi3mr_transport_template;
3959	}
3960
3961	mrioc->reply_sz = mrioc->facts.reply_sz;
3962
3963	retval = mpi3mr_check_reset_dma_mask(mrioc);
3964	if (retval) {
3965	ioc_err(mrioc, "Resetting dma mask failed %d\n",
3966	retval);
3967	goto out_failed_noretry;
3968	}
3969
3970	mpi3mr_print_ioc_info(mrioc);
3971
3972	if (!mrioc->cfg_page) {
3973	dprint_init(mrioc, "allocating config page buffers\n");
3974	mrioc->cfg_page_sz = MPI3MR_DEFAULT_CFG_PAGE_SZ;
3975	mrioc->cfg_page = dma_alloc_coherent(dev: &mrioc->pdev->dev,
3976	size: mrioc->cfg_page_sz, dma_handle: &mrioc->cfg_page_dma, GFP_KERNEL);
3977	if (!mrioc->cfg_page) {
3978	retval = -1;
3979	goto out_failed_noretry;
3980	}
3981	}
3982
3983	dprint_init(mrioc, "allocating ioctl dma buffers\n");
3984	mpi3mr_alloc_ioctl_dma_memory(mrioc);
3985
3986	if (!mrioc->init_cmds.reply) {
3987	retval = mpi3mr_alloc_reply_sense_bufs(mrioc);
3988	if (retval) {
3989	ioc_err(mrioc,
3990	"%s :Failed to allocated reply sense buffers %d\n",
3991	__func__, retval);
3992	goto out_failed_noretry;
3993	}
3994	}
3995
3996	if (!mrioc->chain_sgl_list) {
3997	retval = mpi3mr_alloc_chain_bufs(mrioc);
3998	if (retval) {
3999	ioc_err(mrioc, "Failed to allocated chain buffers %d\n",
4000	retval);
4001	goto out_failed_noretry;
4002	}
4003	}
4004
4005	retval = mpi3mr_issue_iocinit(mrioc);
4006	if (retval) {
4007	ioc_err(mrioc, "Failed to Issue IOC Init %d\n",
4008	retval);
4009	goto out_failed;
4010	}
4011
4012	retval = mpi3mr_print_pkg_ver(mrioc);
4013	if (retval) {
4014	ioc_err(mrioc, "failed to get package version\n");
4015	goto out_failed;
4016	}
4017
4018	retval = mpi3mr_setup_isr(mrioc, setup_one: 0);
4019	if (retval) {
4020	ioc_err(mrioc, "Failed to re-setup ISR, error %d\n",
4021	retval);
4022	goto out_failed_noretry;
4023	}
4024
4025	retval = mpi3mr_create_op_queues(mrioc);
4026	if (retval) {
4027	ioc_err(mrioc, "Failed to create OpQueues error %d\n",
4028	retval);
4029	goto out_failed;
4030	}
4031
4032	if (!mrioc->pel_seqnum_virt) {
4033	dprint_init(mrioc, "allocating memory for pel_seqnum_virt\n");
4034	mrioc->pel_seqnum_sz = sizeof(struct mpi3_pel_seq);
4035	mrioc->pel_seqnum_virt = dma_alloc_coherent(dev: &mrioc->pdev->dev,
4036	size: mrioc->pel_seqnum_sz, dma_handle: &mrioc->pel_seqnum_dma,
4037	GFP_KERNEL);
4038	if (!mrioc->pel_seqnum_virt) {
4039	retval = -ENOMEM;
4040	goto out_failed_noretry;
4041	}
4042	}
4043
4044	if (!mrioc->throttle_groups && mrioc->num_io_throttle_group) {
4045	dprint_init(mrioc, "allocating memory for throttle groups\n");
4046	sz = sizeof(struct mpi3mr_throttle_group_info);
4047	mrioc->throttle_groups = kcalloc(n: mrioc->num_io_throttle_group, size: sz, GFP_KERNEL);
4048	if (!mrioc->throttle_groups) {
4049	retval = -1;
4050	goto out_failed_noretry;
4051	}
4052	}
4053
4054	retval = mpi3mr_enable_events(mrioc);
4055	if (retval) {
4056	ioc_err(mrioc, "failed to enable events %d\n",
4057	retval);
4058	goto out_failed;
4059	}
4060
4061	ioc_info(mrioc, "controller initialization completed successfully\n");
4062	return retval;
4063	out_failed:
4064	if (retry < 2) {
4065	retry++;
4066	ioc_warn(mrioc, "retrying controller initialization, retry_count:%d\n",
4067	retry);
4068	mpi3mr_memset_buffers(mrioc);
4069	goto retry_init;
4070	}
4071	retval = -1;
4072	out_failed_noretry:
4073	ioc_err(mrioc, "controller initialization failed\n");
4074	mpi3mr_issue_reset(mrioc, MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
4075	reset_reason: MPI3MR_RESET_FROM_CTLR_CLEANUP);
4076	mrioc->unrecoverable = 1;
4077	return retval;
4078	}
4079
4080	/**
4081	* mpi3mr_reinit_ioc - Re-Initialize the controller
4082	* @mrioc: Adapter instance reference
4083	* @is_resume: Called from resume or reset path
4084	*
4085	* This the controller re-initialization routine, executed from
4086	* the soft reset handler or resume callback. Creates
4087	* operational reply queue pairs, allocate required memory for
4088	* reply pool, sense buffer pool, issue IOC init request to the
4089	* firmware, unmask the events and issue port enable to discover
4090	* SAS/SATA/NVMe devices and RAID volumes.
4091	*
4092	* Return: 0 on success and non-zero on failure.
4093	*/
4094	int mpi3mr_reinit_ioc(struct mpi3mr_ioc *mrioc, u8 is_resume)
4095	{
4096	int retval = 0;
4097	u8 retry = 0;
4098	struct mpi3_ioc_facts_data facts_data;
4099	u32 pe_timeout, ioc_status;
4100
4101	retry_init:
4102	pe_timeout =
4103	(MPI3MR_PORTENABLE_TIMEOUT / MPI3MR_PORTENABLE_POLL_INTERVAL);
4104
4105	dprint_reset(mrioc, "bringing up the controller to ready state\n");
4106	retval = mpi3mr_bring_ioc_ready(mrioc);
4107	if (retval) {
4108	ioc_err(mrioc, "failed to bring to ready state\n");
4109	goto out_failed_noretry;
4110	}
4111
4112	if (is_resume) {
4113	dprint_reset(mrioc, "setting up single ISR\n");
4114	retval = mpi3mr_setup_isr(mrioc, setup_one: 1);
4115	if (retval) {
4116	ioc_err(mrioc, "failed to setup ISR\n");
4117	goto out_failed_noretry;
4118	}
4119	} else
4120	mpi3mr_ioc_enable_intr(mrioc);
4121
4122	dprint_reset(mrioc, "getting ioc_facts\n");
4123	retval = mpi3mr_issue_iocfacts(mrioc, facts_data: &facts_data);
4124	if (retval) {
4125	ioc_err(mrioc, "failed to get ioc_facts\n");
4126	goto out_failed;
4127	}
4128
4129	dprint_reset(mrioc, "validating ioc_facts\n");
4130	retval = mpi3mr_revalidate_factsdata(mrioc);
4131	if (retval) {
4132	ioc_err(mrioc, "failed to revalidate ioc_facts data\n");
4133	goto out_failed_noretry;
4134	}
4135
4136	mpi3mr_print_ioc_info(mrioc);
4137
4138	dprint_reset(mrioc, "sending ioc_init\n");
4139	retval = mpi3mr_issue_iocinit(mrioc);
4140	if (retval) {
4141	ioc_err(mrioc, "failed to send ioc_init\n");
4142	goto out_failed;
4143	}
4144
4145	dprint_reset(mrioc, "getting package version\n");
4146	retval = mpi3mr_print_pkg_ver(mrioc);
4147	if (retval) {
4148	ioc_err(mrioc, "failed to get package version\n");
4149	goto out_failed;
4150	}
4151
4152	if (is_resume) {
4153	dprint_reset(mrioc, "setting up multiple ISR\n");
4154	retval = mpi3mr_setup_isr(mrioc, setup_one: 0);
4155	if (retval) {
4156	ioc_err(mrioc, "failed to re-setup ISR\n");
4157	goto out_failed_noretry;
4158	}
4159	}
4160
4161	dprint_reset(mrioc, "creating operational queue pairs\n");
4162	retval = mpi3mr_create_op_queues(mrioc);
4163	if (retval) {
4164	ioc_err(mrioc, "failed to create operational queue pairs\n");
4165	goto out_failed;
4166	}
4167
4168	if (!mrioc->pel_seqnum_virt) {
4169	dprint_reset(mrioc, "allocating memory for pel_seqnum_virt\n");
4170	mrioc->pel_seqnum_sz = sizeof(struct mpi3_pel_seq);
4171	mrioc->pel_seqnum_virt = dma_alloc_coherent(dev: &mrioc->pdev->dev,
4172	size: mrioc->pel_seqnum_sz, dma_handle: &mrioc->pel_seqnum_dma,
4173	GFP_KERNEL);
4174	if (!mrioc->pel_seqnum_virt) {
4175	retval = -ENOMEM;
4176	goto out_failed_noretry;
4177	}
4178	}
4179
4180	if (mrioc->shost->nr_hw_queues > mrioc->num_op_reply_q) {
4181	ioc_err(mrioc,
4182	"cannot create minimum number of operational queues expected:%d created:%d\n",
4183	mrioc->shost->nr_hw_queues, mrioc->num_op_reply_q);
4184	retval = -1;
4185	goto out_failed_noretry;
4186	}
4187
4188	dprint_reset(mrioc, "enabling events\n");
4189	retval = mpi3mr_enable_events(mrioc);
4190	if (retval) {
4191	ioc_err(mrioc, "failed to enable events\n");
4192	goto out_failed;
4193	}
4194
4195	mrioc->device_refresh_on = 1;
4196	mpi3mr_add_event_wait_for_device_refresh(mrioc);
4197
4198	ioc_info(mrioc, "sending port enable\n");
4199	retval = mpi3mr_issue_port_enable(mrioc, async: 1);
4200	if (retval) {
4201	ioc_err(mrioc, "failed to issue port enable\n");
4202	goto out_failed;
4203	}
4204	do {
4205	ssleep(MPI3MR_PORTENABLE_POLL_INTERVAL);
4206	if (mrioc->init_cmds.state == MPI3MR_CMD_NOTUSED)
4207	break;
4208	if (!pci_device_is_present(pdev: mrioc->pdev))
4209	mrioc->unrecoverable = 1;
4210	if (mrioc->unrecoverable) {
4211	retval = -1;
4212	goto out_failed_noretry;
4213	}
4214	ioc_status = readl(addr: &mrioc->sysif_regs->ioc_status);
4215	if ((ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY) ||
4216	(ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT)) {
4217	mpi3mr_print_fault_info(mrioc);
4218	mrioc->init_cmds.is_waiting = 0;
4219	mrioc->init_cmds.callback = NULL;
4220	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
4221	goto out_failed;
4222	}
4223	} while (--pe_timeout);
4224
4225	if (!pe_timeout) {
4226	ioc_err(mrioc, "port enable timed out\n");
4227	mpi3mr_check_rh_fault_ioc(mrioc,
4228	reason_code: MPI3MR_RESET_FROM_PE_TIMEOUT);
4229	mrioc->init_cmds.is_waiting = 0;
4230	mrioc->init_cmds.callback = NULL;
4231	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
4232	goto out_failed;
4233	} else if (mrioc->scan_failed) {
4234	ioc_err(mrioc,
4235	"port enable failed with status=0x%04x\n",
4236	mrioc->scan_failed);
4237	} else
4238	ioc_info(mrioc, "port enable completed successfully\n");
4239
4240	ioc_info(mrioc, "controller %s completed successfully\n",
4241	(is_resume)?"resume":"re-initialization");
4242	return retval;
4243	out_failed:
4244	if (retry < 2) {
4245	retry++;
4246	ioc_warn(mrioc, "retrying controller %s, retry_count:%d\n",
4247	(is_resume)?"resume":"re-initialization", retry);
4248	mpi3mr_memset_buffers(mrioc);
4249	goto retry_init;
4250	}
4251	retval = -1;
4252	out_failed_noretry:
4253	ioc_err(mrioc, "controller %s is failed\n",
4254	(is_resume)?"resume":"re-initialization");
4255	mpi3mr_issue_reset(mrioc, MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
4256	reset_reason: MPI3MR_RESET_FROM_CTLR_CLEANUP);
4257	mrioc->unrecoverable = 1;
4258	return retval;
4259	}
4260
4261	/**
4262	* mpi3mr_memset_op_reply_q_buffers - memset the operational reply queue's
4263	* segments
4264	* @mrioc: Adapter instance reference
4265	* @qidx: Operational reply queue index
4266	*
4267	* Return: Nothing.
4268	*/
4269	static void mpi3mr_memset_op_reply_q_buffers(struct mpi3mr_ioc *mrioc, u16 qidx)
4270	{
4271	struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;
4272	struct segments *segments;
4273	int i, size;
4274
4275	if (!op_reply_q->q_segments)
4276	return;
4277
4278	size = op_reply_q->segment_qd * mrioc->op_reply_desc_sz;
4279	segments = op_reply_q->q_segments;
4280	for (i = 0; i < op_reply_q->num_segments; i++)
4281	memset(segments[i].segment, 0, size);
4282	}
4283
4284	/**
4285	* mpi3mr_memset_op_req_q_buffers - memset the operational request queue's
4286	* segments
4287	* @mrioc: Adapter instance reference
4288	* @qidx: Operational request queue index
4289	*
4290	* Return: Nothing.
4291	*/
4292	static void mpi3mr_memset_op_req_q_buffers(struct mpi3mr_ioc *mrioc, u16 qidx)
4293	{
4294	struct op_req_qinfo *op_req_q = mrioc->req_qinfo + qidx;
4295	struct segments *segments;
4296	int i, size;
4297
4298	if (!op_req_q->q_segments)
4299	return;
4300
4301	size = op_req_q->segment_qd * mrioc->facts.op_req_sz;
4302	segments = op_req_q->q_segments;
4303	for (i = 0; i < op_req_q->num_segments; i++)
4304	memset(segments[i].segment, 0, size);
4305	}
4306
4307	/**
4308	* mpi3mr_memset_buffers - memset memory for a controller
4309	* @mrioc: Adapter instance reference
4310	*
4311	* clear all the memory allocated for a controller, typically
4312	* called post reset to reuse the memory allocated during the
4313	* controller init.
4314	*
4315	* Return: Nothing.
4316	*/
4317	void mpi3mr_memset_buffers(struct mpi3mr_ioc *mrioc)
4318	{
4319	u16 i;
4320	struct mpi3mr_throttle_group_info *tg;
4321
4322	mrioc->change_count = 0;
4323	mrioc->active_poll_qcount = 0;
4324	mrioc->default_qcount = 0;
4325	if (mrioc->admin_req_base)
4326	memset(mrioc->admin_req_base, 0, mrioc->admin_req_q_sz);
4327	if (mrioc->admin_reply_base)
4328	memset(mrioc->admin_reply_base, 0, mrioc->admin_reply_q_sz);
4329	atomic_set(v: &mrioc->admin_reply_q_in_use, i: 0);
4330
4331	if (mrioc->init_cmds.reply) {
4332	memset(mrioc->init_cmds.reply, 0, sizeof(*mrioc->init_cmds.reply));
4333	memset(mrioc->bsg_cmds.reply, 0,
4334	sizeof(*mrioc->bsg_cmds.reply));
4335	memset(mrioc->host_tm_cmds.reply, 0,
4336	sizeof(*mrioc->host_tm_cmds.reply));
4337	memset(mrioc->pel_cmds.reply, 0,
4338	sizeof(*mrioc->pel_cmds.reply));
4339	memset(mrioc->pel_abort_cmd.reply, 0,
4340	sizeof(*mrioc->pel_abort_cmd.reply));
4341	memset(mrioc->transport_cmds.reply, 0,
4342	sizeof(*mrioc->transport_cmds.reply));
4343	for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++)
4344	memset(mrioc->dev_rmhs_cmds[i].reply, 0,
4345	sizeof(*mrioc->dev_rmhs_cmds[i].reply));
4346	for (i = 0; i < MPI3MR_NUM_EVTACKCMD; i++)
4347	memset(mrioc->evtack_cmds[i].reply, 0,
4348	sizeof(*mrioc->evtack_cmds[i].reply));
4349	bitmap_clear(map: mrioc->removepend_bitmap, start: 0,
4350	nbits: mrioc->dev_handle_bitmap_bits);
4351	bitmap_clear(map: mrioc->devrem_bitmap, start: 0, MPI3MR_NUM_DEVRMCMD);
4352	bitmap_clear(map: mrioc->evtack_cmds_bitmap, start: 0,
4353	MPI3MR_NUM_EVTACKCMD);
4354	}
4355
4356	for (i = 0; i < mrioc->num_queues; i++) {
4357	mrioc->op_reply_qinfo[i].qid = 0;
4358	mrioc->op_reply_qinfo[i].ci = 0;
4359	mrioc->op_reply_qinfo[i].num_replies = 0;
4360	mrioc->op_reply_qinfo[i].ephase = 0;
4361	atomic_set(v: &mrioc->op_reply_qinfo[i].pend_ios, i: 0);
4362	atomic_set(v: &mrioc->op_reply_qinfo[i].in_use, i: 0);
4363	mpi3mr_memset_op_reply_q_buffers(mrioc, qidx: i);
4364
4365	mrioc->req_qinfo[i].ci = 0;
4366	mrioc->req_qinfo[i].pi = 0;
4367	mrioc->req_qinfo[i].num_requests = 0;
4368	mrioc->req_qinfo[i].qid = 0;
4369	mrioc->req_qinfo[i].reply_qid = 0;
4370	spin_lock_init(&mrioc->req_qinfo[i].q_lock);
4371	mpi3mr_memset_op_req_q_buffers(mrioc, qidx: i);
4372	}
4373
4374	atomic_set(v: &mrioc->pend_large_data_sz, i: 0);
4375	if (mrioc->throttle_groups) {
4376	tg = mrioc->throttle_groups;
4377	for (i = 0; i < mrioc->num_io_throttle_group; i++, tg++) {
4378	tg->id = 0;
4379	tg->fw_qd = 0;
4380	tg->modified_qd = 0;
4381	tg->io_divert = 0;
4382	tg->need_qd_reduction = 0;
4383	tg->high = 0;
4384	tg->low = 0;
4385	tg->qd_reduction = 0;
4386	atomic_set(v: &tg->pend_large_data_sz, i: 0);
4387	}
4388	}
4389	}
4390
4391	/**
4392	* mpi3mr_free_mem - Free memory allocated for a controller
4393	* @mrioc: Adapter instance reference
4394	*
4395	* Free all the memory allocated for a controller.
4396	*
4397	* Return: Nothing.
4398	*/
4399	void mpi3mr_free_mem(struct mpi3mr_ioc *mrioc)
4400	{
4401	u16 i;
4402	struct mpi3mr_intr_info *intr_info;
4403
4404	mpi3mr_free_enclosure_list(mrioc);
4405	mpi3mr_free_ioctl_dma_memory(mrioc);
4406
4407	if (mrioc->sense_buf_pool) {
4408	if (mrioc->sense_buf)
4409	dma_pool_free(pool: mrioc->sense_buf_pool, vaddr: mrioc->sense_buf,
4410	addr: mrioc->sense_buf_dma);
4411	dma_pool_destroy(pool: mrioc->sense_buf_pool);
4412	mrioc->sense_buf = NULL;
4413	mrioc->sense_buf_pool = NULL;
4414	}
4415	if (mrioc->sense_buf_q_pool) {
4416	if (mrioc->sense_buf_q)
4417	dma_pool_free(pool: mrioc->sense_buf_q_pool,
4418	vaddr: mrioc->sense_buf_q, addr: mrioc->sense_buf_q_dma);
4419	dma_pool_destroy(pool: mrioc->sense_buf_q_pool);
4420	mrioc->sense_buf_q = NULL;
4421	mrioc->sense_buf_q_pool = NULL;
4422	}
4423
4424	if (mrioc->reply_buf_pool) {
4425	if (mrioc->reply_buf)
4426	dma_pool_free(pool: mrioc->reply_buf_pool, vaddr: mrioc->reply_buf,
4427	addr: mrioc->reply_buf_dma);
4428	dma_pool_destroy(pool: mrioc->reply_buf_pool);
4429	mrioc->reply_buf = NULL;
4430	mrioc->reply_buf_pool = NULL;
4431	}
4432	if (mrioc->reply_free_q_pool) {
4433	if (mrioc->reply_free_q)
4434	dma_pool_free(pool: mrioc->reply_free_q_pool,
4435	vaddr: mrioc->reply_free_q, addr: mrioc->reply_free_q_dma);
4436	dma_pool_destroy(pool: mrioc->reply_free_q_pool);
4437	mrioc->reply_free_q = NULL;
4438	mrioc->reply_free_q_pool = NULL;
4439	}
4440
4441	for (i = 0; i < mrioc->num_op_req_q; i++)
4442	mpi3mr_free_op_req_q_segments(mrioc, q_idx: i);
4443
4444	for (i = 0; i < mrioc->num_op_reply_q; i++)
4445	mpi3mr_free_op_reply_q_segments(mrioc, q_idx: i);
4446
4447	for (i = 0; i < mrioc->intr_info_count; i++) {
4448	intr_info = mrioc->intr_info + i;
4449	intr_info->op_reply_q = NULL;
4450	}
4451
4452	kfree(objp: mrioc->req_qinfo);
4453	mrioc->req_qinfo = NULL;
4454	mrioc->num_op_req_q = 0;
4455
4456	kfree(objp: mrioc->op_reply_qinfo);
4457	mrioc->op_reply_qinfo = NULL;
4458	mrioc->num_op_reply_q = 0;
4459
4460	kfree(objp: mrioc->init_cmds.reply);
4461	mrioc->init_cmds.reply = NULL;
4462
4463	kfree(objp: mrioc->bsg_cmds.reply);
4464	mrioc->bsg_cmds.reply = NULL;
4465
4466	kfree(objp: mrioc->host_tm_cmds.reply);
4467	mrioc->host_tm_cmds.reply = NULL;
4468
4469	kfree(objp: mrioc->pel_cmds.reply);
4470	mrioc->pel_cmds.reply = NULL;
4471
4472	kfree(objp: mrioc->pel_abort_cmd.reply);
4473	mrioc->pel_abort_cmd.reply = NULL;
4474
4475	for (i = 0; i < MPI3MR_NUM_EVTACKCMD; i++) {
4476	kfree(objp: mrioc->evtack_cmds[i].reply);
4477	mrioc->evtack_cmds[i].reply = NULL;
4478	}
4479
4480	bitmap_free(bitmap: mrioc->removepend_bitmap);
4481	mrioc->removepend_bitmap = NULL;
4482
4483	bitmap_free(bitmap: mrioc->devrem_bitmap);
4484	mrioc->devrem_bitmap = NULL;
4485
4486	bitmap_free(bitmap: mrioc->evtack_cmds_bitmap);
4487	mrioc->evtack_cmds_bitmap = NULL;
4488
4489	bitmap_free(bitmap: mrioc->chain_bitmap);
4490	mrioc->chain_bitmap = NULL;
4491
4492	kfree(objp: mrioc->transport_cmds.reply);
4493	mrioc->transport_cmds.reply = NULL;
4494
4495	for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++) {
4496	kfree(objp: mrioc->dev_rmhs_cmds[i].reply);
4497	mrioc->dev_rmhs_cmds[i].reply = NULL;
4498	}
4499
4500	if (mrioc->chain_buf_pool) {
4501	for (i = 0; i < mrioc->chain_buf_count; i++) {
4502	if (mrioc->chain_sgl_list[i].addr) {
4503	dma_pool_free(pool: mrioc->chain_buf_pool,
4504	vaddr: mrioc->chain_sgl_list[i].addr,
4505	addr: mrioc->chain_sgl_list[i].dma_addr);
4506	mrioc->chain_sgl_list[i].addr = NULL;
4507	}
4508	}
4509	dma_pool_destroy(pool: mrioc->chain_buf_pool);
4510	mrioc->chain_buf_pool = NULL;
4511	}
4512
4513	kfree(objp: mrioc->chain_sgl_list);
4514	mrioc->chain_sgl_list = NULL;
4515
4516	if (mrioc->admin_reply_base) {
4517	dma_free_coherent(dev: &mrioc->pdev->dev, size: mrioc->admin_reply_q_sz,
4518	cpu_addr: mrioc->admin_reply_base, dma_handle: mrioc->admin_reply_dma);
4519	mrioc->admin_reply_base = NULL;
4520	}
4521	if (mrioc->admin_req_base) {
4522	dma_free_coherent(dev: &mrioc->pdev->dev, size: mrioc->admin_req_q_sz,
4523	cpu_addr: mrioc->admin_req_base, dma_handle: mrioc->admin_req_dma);
4524	mrioc->admin_req_base = NULL;
4525	}
4526	if (mrioc->cfg_page) {
4527	dma_free_coherent(dev: &mrioc->pdev->dev, size: mrioc->cfg_page_sz,
4528	cpu_addr: mrioc->cfg_page, dma_handle: mrioc->cfg_page_dma);
4529	mrioc->cfg_page = NULL;
4530	}
4531	if (mrioc->pel_seqnum_virt) {
4532	dma_free_coherent(dev: &mrioc->pdev->dev, size: mrioc->pel_seqnum_sz,
4533	cpu_addr: mrioc->pel_seqnum_virt, dma_handle: mrioc->pel_seqnum_dma);
4534	mrioc->pel_seqnum_virt = NULL;
4535	}
4536
4537	kfree(objp: mrioc->throttle_groups);
4538	mrioc->throttle_groups = NULL;
4539
4540	kfree(objp: mrioc->logdata_buf);
4541	mrioc->logdata_buf = NULL;
4542
4543	}
4544
4545	/**
4546	* mpi3mr_issue_ioc_shutdown - shutdown controller
4547	* @mrioc: Adapter instance reference
4548	*
4549	* Send shutodwn notification to the controller and wait for the
4550	* shutdown_timeout for it to be completed.
4551	*
4552	* Return: Nothing.
4553	*/
4554	static void mpi3mr_issue_ioc_shutdown(struct mpi3mr_ioc *mrioc)
4555	{
4556	u32 ioc_config, ioc_status;
4557	u8 retval = 1;
4558	u32 timeout = MPI3MR_DEFAULT_SHUTDOWN_TIME * 10;
4559
4560	ioc_info(mrioc, "Issuing shutdown Notification\n");
4561	if (mrioc->unrecoverable) {
4562	ioc_warn(mrioc,
4563	"IOC is unrecoverable shutdown is not issued\n");
4564	return;
4565	}
4566	ioc_status = readl(addr: &mrioc->sysif_regs->ioc_status);
4567	if ((ioc_status & MPI3_SYSIF_IOC_STATUS_SHUTDOWN_MASK)
4568	== MPI3_SYSIF_IOC_STATUS_SHUTDOWN_IN_PROGRESS) {
4569	ioc_info(mrioc, "shutdown already in progress\n");
4570	return;
4571	}
4572
4573	ioc_config = readl(addr: &mrioc->sysif_regs->ioc_configuration);
4574	ioc_config |= MPI3_SYSIF_IOC_CONFIG_SHUTDOWN_NORMAL;
4575	ioc_config |= MPI3_SYSIF_IOC_CONFIG_DEVICE_SHUTDOWN_SEND_REQ;
4576
4577	writel(val: ioc_config, addr: &mrioc->sysif_regs->ioc_configuration);
4578
4579	if (mrioc->facts.shutdown_timeout)
4580	timeout = mrioc->facts.shutdown_timeout * 10;
4581
4582	do {
4583	ioc_status = readl(addr: &mrioc->sysif_regs->ioc_status);
4584	if ((ioc_status & MPI3_SYSIF_IOC_STATUS_SHUTDOWN_MASK)
4585	== MPI3_SYSIF_IOC_STATUS_SHUTDOWN_COMPLETE) {
4586	retval = 0;
4587	break;
4588	}
4589	msleep(msecs: 100);
4590	} while (--timeout);
4591
4592	ioc_status = readl(addr: &mrioc->sysif_regs->ioc_status);
4593	ioc_config = readl(addr: &mrioc->sysif_regs->ioc_configuration);
4594
4595	if (retval) {
4596	if ((ioc_status & MPI3_SYSIF_IOC_STATUS_SHUTDOWN_MASK)
4597	== MPI3_SYSIF_IOC_STATUS_SHUTDOWN_IN_PROGRESS)
4598	ioc_warn(mrioc,
4599	"shutdown still in progress after timeout\n");
4600	}
4601
4602	ioc_info(mrioc,
4603	"Base IOC Sts/Config after %s shutdown is (0x%x)/(0x%x)\n",
4604	(!retval) ? "successful" : "failed", ioc_status,
4605	ioc_config);
4606	}
4607
4608	/**
4609	* mpi3mr_cleanup_ioc - Cleanup controller
4610	* @mrioc: Adapter instance reference
4611	*
4612	* controller cleanup handler, Message unit reset or soft reset
4613	* and shutdown notification is issued to the controller.
4614	*
4615	* Return: Nothing.
4616	*/
4617	void mpi3mr_cleanup_ioc(struct mpi3mr_ioc *mrioc)
4618	{
4619	enum mpi3mr_iocstate ioc_state;
4620
4621	dprint_exit(mrioc, "cleaning up the controller\n");
4622	mpi3mr_ioc_disable_intr(mrioc);
4623
4624	ioc_state = mpi3mr_get_iocstate(mrioc);
4625
4626	if ((!mrioc->unrecoverable) && (!mrioc->reset_in_progress) &&
4627	(ioc_state == MRIOC_STATE_READY)) {
4628	if (mpi3mr_issue_and_process_mur(mrioc,
4629	reset_reason: MPI3MR_RESET_FROM_CTLR_CLEANUP))
4630	mpi3mr_issue_reset(mrioc,
4631	MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET,
4632	reset_reason: MPI3MR_RESET_FROM_MUR_FAILURE);
4633	mpi3mr_issue_ioc_shutdown(mrioc);
4634	}
4635	dprint_exit(mrioc, "controller cleanup completed\n");
4636	}
4637
4638	/**
4639	* mpi3mr_drv_cmd_comp_reset - Flush a internal driver command
4640	* @mrioc: Adapter instance reference
4641	* @cmdptr: Internal command tracker
4642	*
4643	* Complete an internal driver commands with state indicating it
4644	* is completed due to reset.
4645	*
4646	* Return: Nothing.
4647	*/
4648	static inline void mpi3mr_drv_cmd_comp_reset(struct mpi3mr_ioc *mrioc,
4649	struct mpi3mr_drv_cmd *cmdptr)
4650	{
4651	if (cmdptr->state & MPI3MR_CMD_PENDING) {
4652	cmdptr->state |= MPI3MR_CMD_RESET;
4653	cmdptr->state &= ~MPI3MR_CMD_PENDING;
4654	if (cmdptr->is_waiting) {
4655	complete(&cmdptr->done);
4656	cmdptr->is_waiting = 0;
4657	} else if (cmdptr->callback)
4658	cmdptr->callback(mrioc, cmdptr);
4659	}
4660	}
4661
4662	/**
4663	* mpi3mr_flush_drv_cmds - Flush internaldriver commands
4664	* @mrioc: Adapter instance reference
4665	*
4666	* Flush all internal driver commands post reset
4667	*
4668	* Return: Nothing.
4669	*/
4670	void mpi3mr_flush_drv_cmds(struct mpi3mr_ioc *mrioc)
4671	{
4672	struct mpi3mr_drv_cmd *cmdptr;
4673	u8 i;
4674
4675	cmdptr = &mrioc->init_cmds;
4676	mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4677
4678	cmdptr = &mrioc->cfg_cmds;
4679	mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4680
4681	cmdptr = &mrioc->bsg_cmds;
4682	mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4683	cmdptr = &mrioc->host_tm_cmds;
4684	mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4685
4686	for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++) {
4687	cmdptr = &mrioc->dev_rmhs_cmds[i];
4688	mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4689	}
4690
4691	for (i = 0; i < MPI3MR_NUM_EVTACKCMD; i++) {
4692	cmdptr = &mrioc->evtack_cmds[i];
4693	mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4694	}
4695
4696	cmdptr = &mrioc->pel_cmds;
4697	mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4698
4699	cmdptr = &mrioc->pel_abort_cmd;
4700	mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4701
4702	cmdptr = &mrioc->transport_cmds;
4703	mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4704	}
4705
4706	/**
4707	* mpi3mr_pel_wait_post - Issue PEL Wait
4708	* @mrioc: Adapter instance reference
4709	* @drv_cmd: Internal command tracker
4710	*
4711	* Issue PEL Wait MPI request through admin queue and return.
4712	*
4713	* Return: Nothing.
4714	*/
4715	static void mpi3mr_pel_wait_post(struct mpi3mr_ioc *mrioc,
4716	struct mpi3mr_drv_cmd *drv_cmd)
4717	{
4718	struct mpi3_pel_req_action_wait pel_wait;
4719
4720	mrioc->pel_abort_requested = false;
4721
4722	memset(&pel_wait, 0, sizeof(pel_wait));
4723	drv_cmd->state = MPI3MR_CMD_PENDING;
4724	drv_cmd->is_waiting = 0;
4725	drv_cmd->callback = mpi3mr_pel_wait_complete;
4726	drv_cmd->ioc_status = 0;
4727	drv_cmd->ioc_loginfo = 0;
4728	pel_wait.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_PEL_WAIT);
4729	pel_wait.function = MPI3_FUNCTION_PERSISTENT_EVENT_LOG;
4730	pel_wait.action = MPI3_PEL_ACTION_WAIT;
4731	pel_wait.starting_sequence_number = cpu_to_le32(mrioc->pel_newest_seqnum);
4732	pel_wait.locale = cpu_to_le16(mrioc->pel_locale);
4733	pel_wait.class = cpu_to_le16(mrioc->pel_class);
4734	pel_wait.wait_time = MPI3_PEL_WAITTIME_INFINITE_WAIT;
4735	dprint_bsg_info(mrioc, "sending pel_wait seqnum(%d), class(%d), locale(0x%08x)\n",
4736	mrioc->pel_newest_seqnum, mrioc->pel_class, mrioc->pel_locale);
4737
4738	if (mpi3mr_admin_request_post(mrioc, admin_req: &pel_wait, admin_req_sz: sizeof(pel_wait), ignore_reset: 0)) {
4739	dprint_bsg_err(mrioc,
4740	"Issuing PELWait: Admin post failed\n");
4741	drv_cmd->state = MPI3MR_CMD_NOTUSED;
4742	drv_cmd->callback = NULL;
4743	drv_cmd->retry_count = 0;
4744	mrioc->pel_enabled = false;
4745	}
4746	}
4747
4748	/**
4749	* mpi3mr_pel_get_seqnum_post - Issue PEL Get Sequence number
4750	* @mrioc: Adapter instance reference
4751	* @drv_cmd: Internal command tracker
4752	*
4753	* Issue PEL get sequence number MPI request through admin queue
4754	* and return.
4755	*
4756	* Return: 0 on success, non-zero on failure.
4757	*/
4758	int mpi3mr_pel_get_seqnum_post(struct mpi3mr_ioc *mrioc,
4759	struct mpi3mr_drv_cmd *drv_cmd)
4760	{
4761	struct mpi3_pel_req_action_get_sequence_numbers pel_getseq_req;
4762	u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST;
4763	int retval = 0;
4764
4765	memset(&pel_getseq_req, 0, sizeof(pel_getseq_req));
4766	mrioc->pel_cmds.state = MPI3MR_CMD_PENDING;
4767	mrioc->pel_cmds.is_waiting = 0;
4768	mrioc->pel_cmds.ioc_status = 0;
4769	mrioc->pel_cmds.ioc_loginfo = 0;
4770	mrioc->pel_cmds.callback = mpi3mr_pel_get_seqnum_complete;
4771	pel_getseq_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_PEL_WAIT);
4772	pel_getseq_req.function = MPI3_FUNCTION_PERSISTENT_EVENT_LOG;
4773	pel_getseq_req.action = MPI3_PEL_ACTION_GET_SEQNUM;
4774	mpi3mr_add_sg_single(paddr: &pel_getseq_req.sgl, flags: sgl_flags,
4775	length: mrioc->pel_seqnum_sz, dma_addr: mrioc->pel_seqnum_dma);
4776
4777	retval = mpi3mr_admin_request_post(mrioc, admin_req: &pel_getseq_req,
4778	admin_req_sz: sizeof(pel_getseq_req), ignore_reset: 0);
4779	if (retval) {
4780	if (drv_cmd) {
4781	drv_cmd->state = MPI3MR_CMD_NOTUSED;
4782	drv_cmd->callback = NULL;
4783	drv_cmd->retry_count = 0;
4784	}
4785	mrioc->pel_enabled = false;
4786	}
4787
4788	return retval;
4789	}
4790
4791	/**
4792	* mpi3mr_pel_wait_complete - PELWait Completion callback
4793	* @mrioc: Adapter instance reference
4794	* @drv_cmd: Internal command tracker
4795	*
4796	* This is a callback handler for the PELWait request and
4797	* firmware completes a PELWait request when it is aborted or a
4798	* new PEL entry is available. This sends AEN to the application
4799	* and if the PELwait completion is not due to PELAbort then
4800	* this will send a request for new PEL Sequence number
4801	*
4802	* Return: Nothing.
4803	*/
4804	static void mpi3mr_pel_wait_complete(struct mpi3mr_ioc *mrioc,
4805	struct mpi3mr_drv_cmd *drv_cmd)
4806	{
4807	struct mpi3_pel_reply *pel_reply = NULL;
4808	u16 ioc_status, pe_log_status;
4809	bool do_retry = false;
4810
4811	if (drv_cmd->state & MPI3MR_CMD_RESET)
4812	goto cleanup_drv_cmd;
4813
4814	ioc_status = drv_cmd->ioc_status & MPI3_IOCSTATUS_STATUS_MASK;
4815	if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
4816	ioc_err(mrioc, "%s: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
4817	__func__, ioc_status, drv_cmd->ioc_loginfo);
4818	dprint_bsg_err(mrioc,
4819	"pel_wait: failed with ioc_status(0x%04x), log_info(0x%08x)\n",
4820	ioc_status, drv_cmd->ioc_loginfo);
4821	do_retry = true;
4822	}
4823
4824	if (drv_cmd->state & MPI3MR_CMD_REPLY_VALID)
4825	pel_reply = (struct mpi3_pel_reply *)drv_cmd->reply;
4826
4827	if (!pel_reply) {
4828	dprint_bsg_err(mrioc,
4829	"pel_wait: failed due to no reply\n");
4830	goto out_failed;
4831	}
4832
4833	pe_log_status = le16_to_cpu(pel_reply->pe_log_status);
4834	if ((pe_log_status != MPI3_PEL_STATUS_SUCCESS) &&
4835	(pe_log_status != MPI3_PEL_STATUS_ABORTED)) {
4836	ioc_err(mrioc, "%s: Failed pe_log_status(0x%04x)\n",
4837	__func__, pe_log_status);
4838	dprint_bsg_err(mrioc,
4839	"pel_wait: failed due to pel_log_status(0x%04x)\n",
4840	pe_log_status);
4841	do_retry = true;
4842	}
4843
4844	if (do_retry) {
4845	if (drv_cmd->retry_count < MPI3MR_PEL_RETRY_COUNT) {
4846	drv_cmd->retry_count++;
4847	dprint_bsg_err(mrioc, "pel_wait: retrying(%d)\n",
4848	drv_cmd->retry_count);
4849	mpi3mr_pel_wait_post(mrioc, drv_cmd);
4850	return;
4851	}
4852	dprint_bsg_err(mrioc,
4853	"pel_wait: failed after all retries(%d)\n",
4854	drv_cmd->retry_count);
4855	goto out_failed;
4856	}
4857	atomic64_inc(v: &event_counter);
4858	if (!mrioc->pel_abort_requested) {
4859	mrioc->pel_cmds.retry_count = 0;
4860	mpi3mr_pel_get_seqnum_post(mrioc, drv_cmd: &mrioc->pel_cmds);
4861	}
4862
4863	return;
4864	out_failed:
4865	mrioc->pel_enabled = false;
4866	cleanup_drv_cmd:
4867	drv_cmd->state = MPI3MR_CMD_NOTUSED;
4868	drv_cmd->callback = NULL;
4869	drv_cmd->retry_count = 0;
4870	}
4871
4872	/**
4873	* mpi3mr_pel_get_seqnum_complete - PELGetSeqNum Completion callback
4874	* @mrioc: Adapter instance reference
4875	* @drv_cmd: Internal command tracker
4876	*
4877	* This is a callback handler for the PEL get sequence number
4878	* request and a new PEL wait request will be issued to the
4879	* firmware from this
4880	*
4881	* Return: Nothing.
4882	*/
4883	void mpi3mr_pel_get_seqnum_complete(struct mpi3mr_ioc *mrioc,
4884	struct mpi3mr_drv_cmd *drv_cmd)
4885	{
4886	struct mpi3_pel_reply *pel_reply = NULL;
4887	struct mpi3_pel_seq *pel_seqnum_virt;
4888	u16 ioc_status;
4889	bool do_retry = false;
4890
4891	pel_seqnum_virt = (struct mpi3_pel_seq *)mrioc->pel_seqnum_virt;
4892
4893	if (drv_cmd->state & MPI3MR_CMD_RESET)
4894	goto cleanup_drv_cmd;
4895
4896	ioc_status = drv_cmd->ioc_status & MPI3_IOCSTATUS_STATUS_MASK;
4897	if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
4898	dprint_bsg_err(mrioc,
4899	"pel_get_seqnum: failed with ioc_status(0x%04x), log_info(0x%08x)\n",
4900	ioc_status, drv_cmd->ioc_loginfo);
4901	do_retry = true;
4902	}
4903
4904	if (drv_cmd->state & MPI3MR_CMD_REPLY_VALID)
4905	pel_reply = (struct mpi3_pel_reply *)drv_cmd->reply;
4906	if (!pel_reply) {
4907	dprint_bsg_err(mrioc,
4908	"pel_get_seqnum: failed due to no reply\n");
4909	goto out_failed;
4910	}
4911
4912	if (le16_to_cpu(pel_reply->pe_log_status) != MPI3_PEL_STATUS_SUCCESS) {
4913	dprint_bsg_err(mrioc,
4914	"pel_get_seqnum: failed due to pel_log_status(0x%04x)\n",
4915	le16_to_cpu(pel_reply->pe_log_status));
4916	do_retry = true;
4917	}
4918
4919	if (do_retry) {
4920	if (drv_cmd->retry_count < MPI3MR_PEL_RETRY_COUNT) {
4921	drv_cmd->retry_count++;
4922	dprint_bsg_err(mrioc,
4923	"pel_get_seqnum: retrying(%d)\n",
4924	drv_cmd->retry_count);
4925	mpi3mr_pel_get_seqnum_post(mrioc, drv_cmd);
4926	return;
4927	}
4928
4929	dprint_bsg_err(mrioc,
4930	"pel_get_seqnum: failed after all retries(%d)\n",
4931	drv_cmd->retry_count);
4932	goto out_failed;
4933	}
4934	mrioc->pel_newest_seqnum = le32_to_cpu(pel_seqnum_virt->newest) + 1;
4935	drv_cmd->retry_count = 0;
4936	mpi3mr_pel_wait_post(mrioc, drv_cmd);
4937
4938	return;
4939	out_failed:
4940	mrioc->pel_enabled = false;
4941	cleanup_drv_cmd:
4942	drv_cmd->state = MPI3MR_CMD_NOTUSED;
4943	drv_cmd->callback = NULL;
4944	drv_cmd->retry_count = 0;
4945	}
4946
4947	/**
4948	* mpi3mr_soft_reset_handler - Reset the controller
4949	* @mrioc: Adapter instance reference
4950	* @reset_reason: Reset reason code
4951	* @snapdump: Flag to generate snapdump in firmware or not
4952	*
4953	* This is an handler for recovering controller by issuing soft
4954	* reset are diag fault reset. This is a blocking function and
4955	* when one reset is executed if any other resets they will be
4956	* blocked. All BSG requests will be blocked during the reset. If
4957	* controller reset is successful then the controller will be
4958	* reinitalized, otherwise the controller will be marked as not
4959	* recoverable
4960	*
4961	* In snapdump bit is set, the controller is issued with diag
4962	* fault reset so that the firmware can create a snap dump and
4963	* post that the firmware will result in F000 fault and the
4964	* driver will issue soft reset to recover from that.
4965	*
4966	* Return: 0 on success, non-zero on failure.
4967	*/
4968	int mpi3mr_soft_reset_handler(struct mpi3mr_ioc *mrioc,
4969	u32 reset_reason, u8 snapdump)
4970	{
4971	int retval = 0, i;
4972	unsigned long flags;
4973	u32 host_diagnostic, timeout = MPI3_SYSIF_DIAG_SAVE_TIMEOUT * 10;
4974
4975	/* Block the reset handler until diag save in progress*/
4976	dprint_reset(mrioc,
4977	"soft_reset_handler: check and block on diagsave_timeout(%d)\n",
4978	mrioc->diagsave_timeout);
4979	while (mrioc->diagsave_timeout)
4980	ssleep(seconds: 1);
4981	/*
4982	* Block new resets until the currently executing one is finished and
4983	* return the status of the existing reset for all blocked resets
4984	*/
4985	dprint_reset(mrioc, "soft_reset_handler: acquiring reset_mutex\n");
4986	if (!mutex_trylock(lock: &mrioc->reset_mutex)) {
4987	ioc_info(mrioc,
4988	"controller reset triggered by %s is blocked due to another reset in progress\n",
4989	mpi3mr_reset_rc_name(reset_reason));
4990	do {
4991	ssleep(seconds: 1);
4992	} while (mrioc->reset_in_progress == 1);
4993	ioc_info(mrioc,
4994	"returning previous reset result(%d) for the reset triggered by %s\n",
4995	mrioc->prev_reset_result,
4996	mpi3mr_reset_rc_name(reset_reason));
4997	return mrioc->prev_reset_result;
4998	}
4999	ioc_info(mrioc, "controller reset is triggered by %s\n",
5000	mpi3mr_reset_rc_name(reset_reason));
5001
5002	mrioc->device_refresh_on = 0;
5003	mrioc->reset_in_progress = 1;
5004	mrioc->stop_bsgs = 1;
5005	mrioc->prev_reset_result = -1;
5006
5007	if ((!snapdump) && (reset_reason != MPI3MR_RESET_FROM_FAULT_WATCH) &&
5008	(reset_reason != MPI3MR_RESET_FROM_FIRMWARE) &&
5009	(reset_reason != MPI3MR_RESET_FROM_CIACTIV_FAULT)) {
5010	for (i = 0; i < MPI3_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
5011	mrioc->event_masks[i] = -1;
5012
5013	dprint_reset(mrioc, "soft_reset_handler: masking events\n");
5014	mpi3mr_issue_event_notification(mrioc);
5015	}
5016
5017	mpi3mr_wait_for_host_io(mrioc, MPI3MR_RESET_HOST_IOWAIT_TIMEOUT);
5018
5019	mpi3mr_ioc_disable_intr(mrioc);
5020
5021	if (snapdump) {
5022	mpi3mr_set_diagsave(mrioc);
5023	retval = mpi3mr_issue_reset(mrioc,
5024	MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT, reset_reason);
5025	if (!retval) {
5026	do {
5027	host_diagnostic =
5028	readl(addr: &mrioc->sysif_regs->host_diagnostic);
5029	if (!(host_diagnostic &
5030	MPI3_SYSIF_HOST_DIAG_SAVE_IN_PROGRESS))
5031	break;
5032	msleep(msecs: 100);
5033	} while (--timeout);
5034	}
5035	}
5036
5037	retval = mpi3mr_issue_reset(mrioc,
5038	MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET, reset_reason);
5039	if (retval) {
5040	ioc_err(mrioc, "Failed to issue soft reset to the ioc\n");
5041	goto out;
5042	}
5043	if (mrioc->num_io_throttle_group !=
5044	mrioc->facts.max_io_throttle_group) {
5045	ioc_err(mrioc,
5046	"max io throttle group doesn't match old(%d), new(%d)\n",
5047	mrioc->num_io_throttle_group,
5048	mrioc->facts.max_io_throttle_group);
5049	retval = -EPERM;
5050	goto out;
5051	}
5052
5053	mpi3mr_flush_delayed_cmd_lists(mrioc);
5054	mpi3mr_flush_drv_cmds(mrioc);
5055	bitmap_clear(map: mrioc->devrem_bitmap, start: 0, MPI3MR_NUM_DEVRMCMD);
5056	bitmap_clear(map: mrioc->removepend_bitmap, start: 0,
5057	nbits: mrioc->dev_handle_bitmap_bits);
5058	bitmap_clear(map: mrioc->evtack_cmds_bitmap, start: 0, MPI3MR_NUM_EVTACKCMD);
5059	mpi3mr_flush_host_io(mrioc);
5060	mpi3mr_cleanup_fwevt_list(mrioc);
5061	mpi3mr_invalidate_devhandles(mrioc);
5062	mpi3mr_free_enclosure_list(mrioc);
5063
5064	if (mrioc->prepare_for_reset) {
5065	mrioc->prepare_for_reset = 0;
5066	mrioc->prepare_for_reset_timeout_counter = 0;
5067	}
5068	mpi3mr_memset_buffers(mrioc);
5069	retval = mpi3mr_reinit_ioc(mrioc, is_resume: 0);
5070	if (retval) {
5071	pr_err(IOCNAME "reinit after soft reset failed: reason %d\n",
5072	mrioc->name, reset_reason);
5073	goto out;
5074	}
5075	ssleep(MPI3MR_RESET_TOPOLOGY_SETTLE_TIME);
5076
5077	out:
5078	if (!retval) {
5079	mrioc->diagsave_timeout = 0;
5080	mrioc->reset_in_progress = 0;
5081	mrioc->pel_abort_requested = 0;
5082	if (mrioc->pel_enabled) {
5083	mrioc->pel_cmds.retry_count = 0;
5084	mpi3mr_pel_wait_post(mrioc, drv_cmd: &mrioc->pel_cmds);
5085	}
5086
5087	mrioc->device_refresh_on = 0;
5088
5089	mrioc->ts_update_counter = 0;
5090	spin_lock_irqsave(&mrioc->watchdog_lock, flags);
5091	if (mrioc->watchdog_work_q)
5092	queue_delayed_work(wq: mrioc->watchdog_work_q,
5093	dwork: &mrioc->watchdog_work,
5094	delay: msecs_to_jiffies(MPI3MR_WATCHDOG_INTERVAL));
5095	spin_unlock_irqrestore(lock: &mrioc->watchdog_lock, flags);
5096	mrioc->stop_bsgs = 0;
5097	if (mrioc->pel_enabled)
5098	atomic64_inc(v: &event_counter);
5099	} else {
5100	mpi3mr_issue_reset(mrioc,
5101	MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT, reset_reason);
5102	mrioc->device_refresh_on = 0;
5103	mrioc->unrecoverable = 1;
5104	mrioc->reset_in_progress = 0;
5105	retval = -1;
5106	mpi3mr_flush_cmds_for_unrecovered_controller(mrioc);
5107	}
5108	mrioc->prev_reset_result = retval;
5109	mutex_unlock(lock: &mrioc->reset_mutex);
5110	ioc_info(mrioc, "controller reset is %s\n",
5111	((retval == 0) ? "successful" : "failed"));
5112	return retval;
5113	}
5114
5115
5116	/**
5117	* mpi3mr_free_config_dma_memory - free memory for config page
5118	* @mrioc: Adapter instance reference
5119	* @mem_desc: memory descriptor structure
5120	*
5121	* Check whether the size of the buffer specified by the memory
5122	* descriptor is greater than the default page size if so then
5123	* free the memory pointed by the descriptor.
5124	*
5125	* Return: Nothing.
5126	*/
5127	static void mpi3mr_free_config_dma_memory(struct mpi3mr_ioc *mrioc,
5128	struct dma_memory_desc *mem_desc)
5129	{
5130	if ((mem_desc->size > mrioc->cfg_page_sz) && mem_desc->addr) {
5131	dma_free_coherent(dev: &mrioc->pdev->dev, size: mem_desc->size,
5132	cpu_addr: mem_desc->addr, dma_handle: mem_desc->dma_addr);
5133	mem_desc->addr = NULL;
5134	}
5135	}
5136
5137	/**
5138	* mpi3mr_alloc_config_dma_memory - Alloc memory for config page
5139	* @mrioc: Adapter instance reference
5140	* @mem_desc: Memory descriptor to hold dma memory info
5141	*
5142	* This function allocates new dmaable memory or provides the
5143	* default config page dmaable memory based on the memory size
5144	* described by the descriptor.
5145	*
5146	* Return: 0 on success, non-zero on failure.
5147	*/
5148	static int mpi3mr_alloc_config_dma_memory(struct mpi3mr_ioc *mrioc,
5149	struct dma_memory_desc *mem_desc)
5150	{
5151	if (mem_desc->size > mrioc->cfg_page_sz) {
5152	mem_desc->addr = dma_alloc_coherent(dev: &mrioc->pdev->dev,
5153	size: mem_desc->size, dma_handle: &mem_desc->dma_addr, GFP_KERNEL);
5154	if (!mem_desc->addr)
5155	return -ENOMEM;
5156	} else {
5157	mem_desc->addr = mrioc->cfg_page;
5158	mem_desc->dma_addr = mrioc->cfg_page_dma;
5159	memset(mem_desc->addr, 0, mrioc->cfg_page_sz);
5160	}
5161	return 0;
5162	}
5163
5164	/**
5165	* mpi3mr_post_cfg_req - Issue config requests and wait
5166	* @mrioc: Adapter instance reference
5167	* @cfg_req: Configuration request
5168	* @timeout: Timeout in seconds
5169	* @ioc_status: Pointer to return ioc status
5170	*
5171	* A generic function for posting MPI3 configuration request to
5172	* the firmware. This blocks for the completion of request for
5173	* timeout seconds and if the request times out this function
5174	* faults the controller with proper reason code.
5175	*
5176	* On successful completion of the request this function returns
5177	* appropriate ioc status from the firmware back to the caller.
5178	*
5179	* Return: 0 on success, non-zero on failure.
5180	*/
5181	static int mpi3mr_post_cfg_req(struct mpi3mr_ioc *mrioc,
5182	struct mpi3_config_request *cfg_req, int timeout, u16 *ioc_status)
5183	{
5184	int retval = 0;
5185
5186	mutex_lock(&mrioc->cfg_cmds.mutex);
5187	if (mrioc->cfg_cmds.state & MPI3MR_CMD_PENDING) {
5188	retval = -1;
5189	ioc_err(mrioc, "sending config request failed due to command in use\n");
5190	mutex_unlock(lock: &mrioc->cfg_cmds.mutex);
5191	goto out;
5192	}
5193	mrioc->cfg_cmds.state = MPI3MR_CMD_PENDING;
5194	mrioc->cfg_cmds.is_waiting = 1;
5195	mrioc->cfg_cmds.callback = NULL;
5196	mrioc->cfg_cmds.ioc_status = 0;
5197	mrioc->cfg_cmds.ioc_loginfo = 0;
5198
5199	cfg_req->host_tag = cpu_to_le16(MPI3MR_HOSTTAG_CFG_CMDS);
5200	cfg_req->function = MPI3_FUNCTION_CONFIG;
5201
5202	init_completion(x: &mrioc->cfg_cmds.done);
5203	dprint_cfg_info(mrioc, "posting config request\n");
5204	if (mrioc->logging_level & MPI3_DEBUG_CFG_INFO)
5205	dprint_dump(req: cfg_req, sz: sizeof(struct mpi3_config_request),
5206	name_string: "mpi3_cfg_req");
5207	retval = mpi3mr_admin_request_post(mrioc, admin_req: cfg_req, admin_req_sz: sizeof(*cfg_req), ignore_reset: 1);
5208	if (retval) {
5209	ioc_err(mrioc, "posting config request failed\n");
5210	goto out_unlock;
5211	}
5212	wait_for_completion_timeout(x: &mrioc->cfg_cmds.done, timeout: (timeout * HZ));
5213	if (!(mrioc->cfg_cmds.state & MPI3MR_CMD_COMPLETE)) {
5214	mpi3mr_check_rh_fault_ioc(mrioc,
5215	reason_code: MPI3MR_RESET_FROM_CFG_REQ_TIMEOUT);
5216	ioc_err(mrioc, "config request timed out\n");
5217	retval = -1;
5218	goto out_unlock;
5219	}
5220	*ioc_status = mrioc->cfg_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK;
5221	if ((*ioc_status) != MPI3_IOCSTATUS_SUCCESS)
5222	dprint_cfg_err(mrioc,
5223	"cfg_page request returned with ioc_status(0x%04x), log_info(0x%08x)\n",
5224	*ioc_status, mrioc->cfg_cmds.ioc_loginfo);
5225
5226	out_unlock:
5227	mrioc->cfg_cmds.state = MPI3MR_CMD_NOTUSED;
5228	mutex_unlock(lock: &mrioc->cfg_cmds.mutex);
5229
5230	out:
5231	return retval;
5232	}
5233
5234	/**
5235	* mpi3mr_process_cfg_req - config page request processor
5236	* @mrioc: Adapter instance reference
5237	* @cfg_req: Configuration request
5238	* @cfg_hdr: Configuration page header
5239	* @timeout: Timeout in seconds
5240	* @ioc_status: Pointer to return ioc status
5241	* @cfg_buf: Memory pointer to copy config page or header
5242	* @cfg_buf_sz: Size of the memory to get config page or header
5243	*
5244	* This is handler for config page read, write and config page
5245	* header read operations.
5246	*
5247	* This function expects the cfg_req to be populated with page
5248	* type, page number, action for the header read and with page
5249	* address for all other operations.
5250	*
5251	* The cfg_hdr can be passed as null for reading required header
5252	* details for read/write pages the cfg_hdr should point valid
5253	* configuration page header.
5254	*
5255	* This allocates dmaable memory based on the size of the config
5256	* buffer and set the SGE of the cfg_req.
5257	*
5258	* For write actions, the config page data has to be passed in
5259	* the cfg_buf and size of the data has to be mentioned in the
5260	* cfg_buf_sz.
5261	*
5262	* For read/header actions, on successful completion of the
5263	* request with successful ioc_status the data will be copied
5264	* into the cfg_buf limited to a minimum of actual page size and
5265	* cfg_buf_sz
5266	*
5267	*
5268	* Return: 0 on success, non-zero on failure.
5269	*/
5270	static int mpi3mr_process_cfg_req(struct mpi3mr_ioc *mrioc,
5271	struct mpi3_config_request *cfg_req,
5272	struct mpi3_config_page_header *cfg_hdr, int timeout, u16 *ioc_status,
5273	void *cfg_buf, u32 cfg_buf_sz)
5274	{
5275	struct dma_memory_desc mem_desc;
5276	int retval = -1;
5277	u8 invalid_action = 0;
5278	u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST;
5279
5280	memset(&mem_desc, 0, sizeof(struct dma_memory_desc));
5281
5282	if (cfg_req->action == MPI3_CONFIG_ACTION_PAGE_HEADER)
5283	mem_desc.size = sizeof(struct mpi3_config_page_header);
5284	else {
5285	if (!cfg_hdr) {
5286	ioc_err(mrioc, "null config header passed for config action(%d), page_type(0x%02x), page_num(%d)\n",
5287	cfg_req->action, cfg_req->page_type,
5288	cfg_req->page_number);
5289	goto out;
5290	}
5291	switch (cfg_hdr->page_attribute & MPI3_CONFIG_PAGEATTR_MASK) {
5292	case MPI3_CONFIG_PAGEATTR_READ_ONLY:
5293	if (cfg_req->action
5294	!= MPI3_CONFIG_ACTION_READ_CURRENT)
5295	invalid_action = 1;
5296	break;
5297	case MPI3_CONFIG_PAGEATTR_CHANGEABLE:
5298	if ((cfg_req->action ==
5299	MPI3_CONFIG_ACTION_READ_PERSISTENT) ||
5300	(cfg_req->action ==
5301	MPI3_CONFIG_ACTION_WRITE_PERSISTENT))
5302	invalid_action = 1;
5303	break;
5304	case MPI3_CONFIG_PAGEATTR_PERSISTENT:
5305	default:
5306	break;
5307	}
5308	if (invalid_action) {
5309	ioc_err(mrioc,
5310	"config action(%d) is not allowed for page_type(0x%02x), page_num(%d) with page_attribute(0x%02x)\n",
5311	cfg_req->action, cfg_req->page_type,
5312	cfg_req->page_number, cfg_hdr->page_attribute);
5313	goto out;
5314	}
5315	mem_desc.size = le16_to_cpu(cfg_hdr->page_length) * 4;
5316	cfg_req->page_length = cfg_hdr->page_length;
5317	cfg_req->page_version = cfg_hdr->page_version;
5318	}
5319	if (mpi3mr_alloc_config_dma_memory(mrioc, mem_desc: &mem_desc))
5320	goto out;
5321
5322	mpi3mr_add_sg_single(paddr: &cfg_req->sgl, flags: sgl_flags, length: mem_desc.size,
5323	dma_addr: mem_desc.dma_addr);
5324
5325	if ((cfg_req->action == MPI3_CONFIG_ACTION_WRITE_PERSISTENT) ||
5326	(cfg_req->action == MPI3_CONFIG_ACTION_WRITE_CURRENT)) {
5327	memcpy(mem_desc.addr, cfg_buf, min_t(u16, mem_desc.size,
5328	cfg_buf_sz));
5329	dprint_cfg_info(mrioc, "config buffer to be written\n");
5330	if (mrioc->logging_level & MPI3_DEBUG_CFG_INFO)
5331	dprint_dump(req: mem_desc.addr, sz: mem_desc.size, name_string: "cfg_buf");
5332	}
5333
5334	if (mpi3mr_post_cfg_req(mrioc, cfg_req, timeout, ioc_status))
5335	goto out;
5336
5337	retval = 0;
5338	if ((*ioc_status == MPI3_IOCSTATUS_SUCCESS) &&
5339	(cfg_req->action != MPI3_CONFIG_ACTION_WRITE_PERSISTENT) &&
5340	(cfg_req->action != MPI3_CONFIG_ACTION_WRITE_CURRENT)) {
5341	memcpy(cfg_buf, mem_desc.addr, min_t(u16, mem_desc.size,
5342	cfg_buf_sz));
5343	dprint_cfg_info(mrioc, "config buffer read\n");
5344	if (mrioc->logging_level & MPI3_DEBUG_CFG_INFO)
5345	dprint_dump(req: mem_desc.addr, sz: mem_desc.size, name_string: "cfg_buf");
5346	}
5347
5348	out:
5349	mpi3mr_free_config_dma_memory(mrioc, mem_desc: &mem_desc);
5350	return retval;
5351	}
5352
5353	/**
5354	* mpi3mr_cfg_get_dev_pg0 - Read current device page0
5355	* @mrioc: Adapter instance reference
5356	* @ioc_status: Pointer to return ioc status
5357	* @dev_pg0: Pointer to return device page 0
5358	* @pg_sz: Size of the memory allocated to the page pointer
5359	* @form: The form to be used for addressing the page
5360	* @form_spec: Form specific information like device handle
5361	*
5362	* This is handler for config page read for a specific device
5363	* page0. The ioc_status has the controller returned ioc_status.
5364	* This routine doesn't check ioc_status to decide whether the
5365	* page read is success or not and it is the callers
5366	* responsibility.
5367	*
5368	* Return: 0 on success, non-zero on failure.
5369	*/
5370	int mpi3mr_cfg_get_dev_pg0(struct mpi3mr_ioc *mrioc, u16 *ioc_status,
5371	struct mpi3_device_page0 *dev_pg0, u16 pg_sz, u32 form, u32 form_spec)
5372	{
5373	struct mpi3_config_page_header cfg_hdr;
5374	struct mpi3_config_request cfg_req;
5375	u32 page_address;
5376
5377	memset(dev_pg0, 0, pg_sz);
5378	memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5379	memset(&cfg_req, 0, sizeof(cfg_req));
5380
5381	cfg_req.function = MPI3_FUNCTION_CONFIG;
5382	cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5383	cfg_req.page_type = MPI3_CONFIG_PAGETYPE_DEVICE;
5384	cfg_req.page_number = 0;
5385	cfg_req.page_address = 0;
5386
5387	if (mpi3mr_process_cfg_req(mrioc, cfg_req: &cfg_req, NULL,
5388	MPI3MR_INTADMCMD_TIMEOUT, ioc_status, cfg_buf: &cfg_hdr, cfg_buf_sz: sizeof(cfg_hdr))) {
5389	ioc_err(mrioc, "device page0 header read failed\n");
5390	goto out_failed;
5391	}
5392	if (*ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5393	ioc_err(mrioc, "device page0 header read failed with ioc_status(0x%04x)\n",
5394	*ioc_status);
5395	goto out_failed;
5396	}
5397	cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT;
5398	page_address = ((form & MPI3_DEVICE_PGAD_FORM_MASK) |
5399	(form_spec & MPI3_DEVICE_PGAD_HANDLE_MASK));
5400	cfg_req.page_address = cpu_to_le32(page_address);
5401	if (mpi3mr_process_cfg_req(mrioc, cfg_req: &cfg_req, cfg_hdr: &cfg_hdr,
5402	MPI3MR_INTADMCMD_TIMEOUT, ioc_status, cfg_buf: dev_pg0, cfg_buf_sz: pg_sz)) {
5403	ioc_err(mrioc, "device page0 read failed\n");
5404	goto out_failed;
5405	}
5406	return 0;
5407	out_failed:
5408	return -1;
5409	}
5410
5411
5412	/**
5413	* mpi3mr_cfg_get_sas_phy_pg0 - Read current SAS Phy page0
5414	* @mrioc: Adapter instance reference
5415	* @ioc_status: Pointer to return ioc status
5416	* @phy_pg0: Pointer to return SAS Phy page 0
5417	* @pg_sz: Size of the memory allocated to the page pointer
5418	* @form: The form to be used for addressing the page
5419	* @form_spec: Form specific information like phy number
5420	*
5421	* This is handler for config page read for a specific SAS Phy
5422	* page0. The ioc_status has the controller returned ioc_status.
5423	* This routine doesn't check ioc_status to decide whether the
5424	* page read is success or not and it is the callers
5425	* responsibility.
5426	*
5427	* Return: 0 on success, non-zero on failure.
5428	*/
5429	int mpi3mr_cfg_get_sas_phy_pg0(struct mpi3mr_ioc *mrioc, u16 *ioc_status,
5430	struct mpi3_sas_phy_page0 *phy_pg0, u16 pg_sz, u32 form,
5431	u32 form_spec)
5432	{
5433	struct mpi3_config_page_header cfg_hdr;
5434	struct mpi3_config_request cfg_req;
5435	u32 page_address;
5436
5437	memset(phy_pg0, 0, pg_sz);
5438	memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5439	memset(&cfg_req, 0, sizeof(cfg_req));
5440
5441	cfg_req.function = MPI3_FUNCTION_CONFIG;
5442	cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5443	cfg_req.page_type = MPI3_CONFIG_PAGETYPE_SAS_PHY;
5444	cfg_req.page_number = 0;
5445	cfg_req.page_address = 0;
5446
5447	if (mpi3mr_process_cfg_req(mrioc, cfg_req: &cfg_req, NULL,
5448	MPI3MR_INTADMCMD_TIMEOUT, ioc_status, cfg_buf: &cfg_hdr, cfg_buf_sz: sizeof(cfg_hdr))) {
5449	ioc_err(mrioc, "sas phy page0 header read failed\n");
5450	goto out_failed;
5451	}
5452	if (*ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5453	ioc_err(mrioc, "sas phy page0 header read failed with ioc_status(0x%04x)\n",
5454	*ioc_status);
5455	goto out_failed;
5456	}
5457	cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT;
5458	page_address = ((form & MPI3_SAS_PHY_PGAD_FORM_MASK) |
5459	(form_spec & MPI3_SAS_PHY_PGAD_PHY_NUMBER_MASK));
5460	cfg_req.page_address = cpu_to_le32(page_address);
5461	if (mpi3mr_process_cfg_req(mrioc, cfg_req: &cfg_req, cfg_hdr: &cfg_hdr,
5462	MPI3MR_INTADMCMD_TIMEOUT, ioc_status, cfg_buf: phy_pg0, cfg_buf_sz: pg_sz)) {
5463	ioc_err(mrioc, "sas phy page0 read failed\n");
5464	goto out_failed;
5465	}
5466	return 0;
5467	out_failed:
5468	return -1;
5469	}
5470
5471	/**
5472	* mpi3mr_cfg_get_sas_phy_pg1 - Read current SAS Phy page1
5473	* @mrioc: Adapter instance reference
5474	* @ioc_status: Pointer to return ioc status
5475	* @phy_pg1: Pointer to return SAS Phy page 1
5476	* @pg_sz: Size of the memory allocated to the page pointer
5477	* @form: The form to be used for addressing the page
5478	* @form_spec: Form specific information like phy number
5479	*
5480	* This is handler for config page read for a specific SAS Phy
5481	* page1. The ioc_status has the controller returned ioc_status.
5482	* This routine doesn't check ioc_status to decide whether the
5483	* page read is success or not and it is the callers
5484	* responsibility.
5485	*
5486	* Return: 0 on success, non-zero on failure.
5487	*/
5488	int mpi3mr_cfg_get_sas_phy_pg1(struct mpi3mr_ioc *mrioc, u16 *ioc_status,
5489	struct mpi3_sas_phy_page1 *phy_pg1, u16 pg_sz, u32 form,
5490	u32 form_spec)
5491	{
5492	struct mpi3_config_page_header cfg_hdr;
5493	struct mpi3_config_request cfg_req;
5494	u32 page_address;
5495
5496	memset(phy_pg1, 0, pg_sz);
5497	memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5498	memset(&cfg_req, 0, sizeof(cfg_req));
5499
5500	cfg_req.function = MPI3_FUNCTION_CONFIG;
5501	cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5502	cfg_req.page_type = MPI3_CONFIG_PAGETYPE_SAS_PHY;
5503	cfg_req.page_number = 1;
5504	cfg_req.page_address = 0;
5505
5506	if (mpi3mr_process_cfg_req(mrioc, cfg_req: &cfg_req, NULL,
5507	MPI3MR_INTADMCMD_TIMEOUT, ioc_status, cfg_buf: &cfg_hdr, cfg_buf_sz: sizeof(cfg_hdr))) {
5508	ioc_err(mrioc, "sas phy page1 header read failed\n");
5509	goto out_failed;
5510	}
5511	if (*ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5512	ioc_err(mrioc, "sas phy page1 header read failed with ioc_status(0x%04x)\n",
5513	*ioc_status);
5514	goto out_failed;
5515	}
5516	cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT;
5517	page_address = ((form & MPI3_SAS_PHY_PGAD_FORM_MASK) |
5518	(form_spec & MPI3_SAS_PHY_PGAD_PHY_NUMBER_MASK));
5519	cfg_req.page_address = cpu_to_le32(page_address);
5520	if (mpi3mr_process_cfg_req(mrioc, cfg_req: &cfg_req, cfg_hdr: &cfg_hdr,
5521	MPI3MR_INTADMCMD_TIMEOUT, ioc_status, cfg_buf: phy_pg1, cfg_buf_sz: pg_sz)) {
5522	ioc_err(mrioc, "sas phy page1 read failed\n");
5523	goto out_failed;
5524	}
5525	return 0;
5526	out_failed:
5527	return -1;
5528	}
5529
5530
5531	/**
5532	* mpi3mr_cfg_get_sas_exp_pg0 - Read current SAS Expander page0
5533	* @mrioc: Adapter instance reference
5534	* @ioc_status: Pointer to return ioc status
5535	* @exp_pg0: Pointer to return SAS Expander page 0
5536	* @pg_sz: Size of the memory allocated to the page pointer
5537	* @form: The form to be used for addressing the page
5538	* @form_spec: Form specific information like device handle
5539	*
5540	* This is handler for config page read for a specific SAS
5541	* Expander page0. The ioc_status has the controller returned
5542	* ioc_status. This routine doesn't check ioc_status to decide
5543	* whether the page read is success or not and it is the callers
5544	* responsibility.
5545	*
5546	* Return: 0 on success, non-zero on failure.
5547	*/
5548	int mpi3mr_cfg_get_sas_exp_pg0(struct mpi3mr_ioc *mrioc, u16 *ioc_status,
5549	struct mpi3_sas_expander_page0 *exp_pg0, u16 pg_sz, u32 form,
5550	u32 form_spec)
5551	{
5552	struct mpi3_config_page_header cfg_hdr;
5553	struct mpi3_config_request cfg_req;
5554	u32 page_address;
5555
5556	memset(exp_pg0, 0, pg_sz);
5557	memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5558	memset(&cfg_req, 0, sizeof(cfg_req));
5559
5560	cfg_req.function = MPI3_FUNCTION_CONFIG;
5561	cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5562	cfg_req.page_type = MPI3_CONFIG_PAGETYPE_SAS_EXPANDER;
5563	cfg_req.page_number = 0;
5564	cfg_req.page_address = 0;
5565
5566	if (mpi3mr_process_cfg_req(mrioc, cfg_req: &cfg_req, NULL,
5567	MPI3MR_INTADMCMD_TIMEOUT, ioc_status, cfg_buf: &cfg_hdr, cfg_buf_sz: sizeof(cfg_hdr))) {
5568	ioc_err(mrioc, "expander page0 header read failed\n");
5569	goto out_failed;
5570	}
5571	if (*ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5572	ioc_err(mrioc, "expander page0 header read failed with ioc_status(0x%04x)\n",
5573	*ioc_status);
5574	goto out_failed;
5575	}
5576	cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT;
5577	page_address = ((form & MPI3_SAS_EXPAND_PGAD_FORM_MASK) |
5578	(form_spec & (MPI3_SAS_EXPAND_PGAD_PHYNUM_MASK |
5579	MPI3_SAS_EXPAND_PGAD_HANDLE_MASK)));
5580	cfg_req.page_address = cpu_to_le32(page_address);
5581	if (mpi3mr_process_cfg_req(mrioc, cfg_req: &cfg_req, cfg_hdr: &cfg_hdr,
5582	MPI3MR_INTADMCMD_TIMEOUT, ioc_status, cfg_buf: exp_pg0, cfg_buf_sz: pg_sz)) {
5583	ioc_err(mrioc, "expander page0 read failed\n");
5584	goto out_failed;
5585	}
5586	return 0;
5587	out_failed:
5588	return -1;
5589	}
5590
5591	/**
5592	* mpi3mr_cfg_get_sas_exp_pg1 - Read current SAS Expander page1
5593	* @mrioc: Adapter instance reference
5594	* @ioc_status: Pointer to return ioc status
5595	* @exp_pg1: Pointer to return SAS Expander page 1
5596	* @pg_sz: Size of the memory allocated to the page pointer
5597	* @form: The form to be used for addressing the page
5598	* @form_spec: Form specific information like phy number
5599	*
5600	* This is handler for config page read for a specific SAS
5601	* Expander page1. The ioc_status has the controller returned
5602	* ioc_status. This routine doesn't check ioc_status to decide
5603	* whether the page read is success or not and it is the callers
5604	* responsibility.
5605	*
5606	* Return: 0 on success, non-zero on failure.
5607	*/
5608	int mpi3mr_cfg_get_sas_exp_pg1(struct mpi3mr_ioc *mrioc, u16 *ioc_status,
5609	struct mpi3_sas_expander_page1 *exp_pg1, u16 pg_sz, u32 form,
5610	u32 form_spec)
5611	{
5612	struct mpi3_config_page_header cfg_hdr;
5613	struct mpi3_config_request cfg_req;
5614	u32 page_address;
5615
5616	memset(exp_pg1, 0, pg_sz);
5617	memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5618	memset(&cfg_req, 0, sizeof(cfg_req));
5619
5620	cfg_req.function = MPI3_FUNCTION_CONFIG;
5621	cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5622	cfg_req.page_type = MPI3_CONFIG_PAGETYPE_SAS_EXPANDER;
5623	cfg_req.page_number = 1;
5624	cfg_req.page_address = 0;
5625
5626	if (mpi3mr_process_cfg_req(mrioc, cfg_req: &cfg_req, NULL,
5627	MPI3MR_INTADMCMD_TIMEOUT, ioc_status, cfg_buf: &cfg_hdr, cfg_buf_sz: sizeof(cfg_hdr))) {
5628	ioc_err(mrioc, "expander page1 header read failed\n");
5629	goto out_failed;
5630	}
5631	if (*ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5632	ioc_err(mrioc, "expander page1 header read failed with ioc_status(0x%04x)\n",
5633	*ioc_status);
5634	goto out_failed;
5635	}
5636	cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT;
5637	page_address = ((form & MPI3_SAS_EXPAND_PGAD_FORM_MASK) |
5638	(form_spec & (MPI3_SAS_EXPAND_PGAD_PHYNUM_MASK |
5639	MPI3_SAS_EXPAND_PGAD_HANDLE_MASK)));
5640	cfg_req.page_address = cpu_to_le32(page_address);
5641	if (mpi3mr_process_cfg_req(mrioc, cfg_req: &cfg_req, cfg_hdr: &cfg_hdr,
5642	MPI3MR_INTADMCMD_TIMEOUT, ioc_status, cfg_buf: exp_pg1, cfg_buf_sz: pg_sz)) {
5643	ioc_err(mrioc, "expander page1 read failed\n");
5644	goto out_failed;
5645	}
5646	return 0;
5647	out_failed:
5648	return -1;
5649	}
5650
5651	/**
5652	* mpi3mr_cfg_get_enclosure_pg0 - Read current Enclosure page0
5653	* @mrioc: Adapter instance reference
5654	* @ioc_status: Pointer to return ioc status
5655	* @encl_pg0: Pointer to return Enclosure page 0
5656	* @pg_sz: Size of the memory allocated to the page pointer
5657	* @form: The form to be used for addressing the page
5658	* @form_spec: Form specific information like device handle
5659	*
5660	* This is handler for config page read for a specific Enclosure
5661	* page0. The ioc_status has the controller returned ioc_status.
5662	* This routine doesn't check ioc_status to decide whether the
5663	* page read is success or not and it is the callers
5664	* responsibility.
5665	*
5666	* Return: 0 on success, non-zero on failure.
5667	*/
5668	int mpi3mr_cfg_get_enclosure_pg0(struct mpi3mr_ioc *mrioc, u16 *ioc_status,
5669	struct mpi3_enclosure_page0 *encl_pg0, u16 pg_sz, u32 form,
5670	u32 form_spec)
5671	{
5672	struct mpi3_config_page_header cfg_hdr;
5673	struct mpi3_config_request cfg_req;
5674	u32 page_address;
5675
5676	memset(encl_pg0, 0, pg_sz);
5677	memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5678	memset(&cfg_req, 0, sizeof(cfg_req));
5679
5680	cfg_req.function = MPI3_FUNCTION_CONFIG;
5681	cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5682	cfg_req.page_type = MPI3_CONFIG_PAGETYPE_ENCLOSURE;
5683	cfg_req.page_number = 0;
5684	cfg_req.page_address = 0;
5685
5686	if (mpi3mr_process_cfg_req(mrioc, cfg_req: &cfg_req, NULL,
5687	MPI3MR_INTADMCMD_TIMEOUT, ioc_status, cfg_buf: &cfg_hdr, cfg_buf_sz: sizeof(cfg_hdr))) {
5688	ioc_err(mrioc, "enclosure page0 header read failed\n");
5689	goto out_failed;
5690	}
5691	if (*ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5692	ioc_err(mrioc, "enclosure page0 header read failed with ioc_status(0x%04x)\n",
5693	*ioc_status);
5694	goto out_failed;
5695	}
5696	cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT;
5697	page_address = ((form & MPI3_ENCLOS_PGAD_FORM_MASK) |
5698	(form_spec & MPI3_ENCLOS_PGAD_HANDLE_MASK));
5699	cfg_req.page_address = cpu_to_le32(page_address);
5700	if (mpi3mr_process_cfg_req(mrioc, cfg_req: &cfg_req, cfg_hdr: &cfg_hdr,
5701	MPI3MR_INTADMCMD_TIMEOUT, ioc_status, cfg_buf: encl_pg0, cfg_buf_sz: pg_sz)) {
5702	ioc_err(mrioc, "enclosure page0 read failed\n");
5703	goto out_failed;
5704	}
5705	return 0;
5706	out_failed:
5707	return -1;
5708	}
5709
5710
5711	/**
5712	* mpi3mr_cfg_get_sas_io_unit_pg0 - Read current SASIOUnit page0
5713	* @mrioc: Adapter instance reference
5714	* @sas_io_unit_pg0: Pointer to return SAS IO Unit page 0
5715	* @pg_sz: Size of the memory allocated to the page pointer
5716	*
5717	* This is handler for config page read for the SAS IO Unit
5718	* page0. This routine checks ioc_status to decide whether the
5719	* page read is success or not.
5720	*
5721	* Return: 0 on success, non-zero on failure.
5722	*/
5723	int mpi3mr_cfg_get_sas_io_unit_pg0(struct mpi3mr_ioc *mrioc,
5724	struct mpi3_sas_io_unit_page0 *sas_io_unit_pg0, u16 pg_sz)
5725	{
5726	struct mpi3_config_page_header cfg_hdr;
5727	struct mpi3_config_request cfg_req;
5728	u16 ioc_status = 0;
5729
5730	memset(sas_io_unit_pg0, 0, pg_sz);
5731	memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5732	memset(&cfg_req, 0, sizeof(cfg_req));
5733
5734	cfg_req.function = MPI3_FUNCTION_CONFIG;
5735	cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5736	cfg_req.page_type = MPI3_CONFIG_PAGETYPE_SAS_IO_UNIT;
5737	cfg_req.page_number = 0;
5738	cfg_req.page_address = 0;
5739
5740	if (mpi3mr_process_cfg_req(mrioc, cfg_req: &cfg_req, NULL,
5741	MPI3MR_INTADMCMD_TIMEOUT, ioc_status: &ioc_status, cfg_buf: &cfg_hdr, cfg_buf_sz: sizeof(cfg_hdr))) {
5742	ioc_err(mrioc, "sas io unit page0 header read failed\n");
5743	goto out_failed;
5744	}
5745	if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5746	ioc_err(mrioc, "sas io unit page0 header read failed with ioc_status(0x%04x)\n",
5747	ioc_status);
5748	goto out_failed;
5749	}
5750	cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT;
5751
5752	if (mpi3mr_process_cfg_req(mrioc, cfg_req: &cfg_req, cfg_hdr: &cfg_hdr,
5753	MPI3MR_INTADMCMD_TIMEOUT, ioc_status: &ioc_status, cfg_buf: sas_io_unit_pg0, cfg_buf_sz: pg_sz)) {
5754	ioc_err(mrioc, "sas io unit page0 read failed\n");
5755	goto out_failed;
5756	}
5757	if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5758	ioc_err(mrioc, "sas io unit page0 read failed with ioc_status(0x%04x)\n",
5759	ioc_status);
5760	goto out_failed;
5761	}
5762	return 0;
5763	out_failed:
5764	return -1;
5765	}
5766
5767	/**
5768	* mpi3mr_cfg_get_sas_io_unit_pg1 - Read current SASIOUnit page1
5769	* @mrioc: Adapter instance reference
5770	* @sas_io_unit_pg1: Pointer to return SAS IO Unit page 1
5771	* @pg_sz: Size of the memory allocated to the page pointer
5772	*
5773	* This is handler for config page read for the SAS IO Unit
5774	* page1. This routine checks ioc_status to decide whether the
5775	* page read is success or not.
5776	*
5777	* Return: 0 on success, non-zero on failure.
5778	*/
5779	int mpi3mr_cfg_get_sas_io_unit_pg1(struct mpi3mr_ioc *mrioc,
5780	struct mpi3_sas_io_unit_page1 *sas_io_unit_pg1, u16 pg_sz)
5781	{
5782	struct mpi3_config_page_header cfg_hdr;
5783	struct mpi3_config_request cfg_req;
5784	u16 ioc_status = 0;
5785
5786	memset(sas_io_unit_pg1, 0, pg_sz);
5787	memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5788	memset(&cfg_req, 0, sizeof(cfg_req));
5789
5790	cfg_req.function = MPI3_FUNCTION_CONFIG;
5791	cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5792	cfg_req.page_type = MPI3_CONFIG_PAGETYPE_SAS_IO_UNIT;
5793	cfg_req.page_number = 1;
5794	cfg_req.page_address = 0;
5795
5796	if (mpi3mr_process_cfg_req(mrioc, cfg_req: &cfg_req, NULL,
5797	MPI3MR_INTADMCMD_TIMEOUT, ioc_status: &ioc_status, cfg_buf: &cfg_hdr, cfg_buf_sz: sizeof(cfg_hdr))) {
5798	ioc_err(mrioc, "sas io unit page1 header read failed\n");
5799	goto out_failed;
5800	}
5801	if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5802	ioc_err(mrioc, "sas io unit page1 header read failed with ioc_status(0x%04x)\n",
5803	ioc_status);
5804	goto out_failed;
5805	}
5806	cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT;
5807
5808	if (mpi3mr_process_cfg_req(mrioc, cfg_req: &cfg_req, cfg_hdr: &cfg_hdr,
5809	MPI3MR_INTADMCMD_TIMEOUT, ioc_status: &ioc_status, cfg_buf: sas_io_unit_pg1, cfg_buf_sz: pg_sz)) {
5810	ioc_err(mrioc, "sas io unit page1 read failed\n");
5811	goto out_failed;
5812	}
5813	if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5814	ioc_err(mrioc, "sas io unit page1 read failed with ioc_status(0x%04x)\n",
5815	ioc_status);
5816	goto out_failed;
5817	}
5818	return 0;
5819	out_failed:
5820	return -1;
5821	}
5822
5823	/**
5824	* mpi3mr_cfg_set_sas_io_unit_pg1 - Write SASIOUnit page1
5825	* @mrioc: Adapter instance reference
5826	* @sas_io_unit_pg1: Pointer to the SAS IO Unit page 1 to write
5827	* @pg_sz: Size of the memory allocated to the page pointer
5828	*
5829	* This is handler for config page write for the SAS IO Unit
5830	* page1. This routine checks ioc_status to decide whether the
5831	* page read is success or not. This will modify both current
5832	* and persistent page.
5833	*
5834	* Return: 0 on success, non-zero on failure.
5835	*/
5836	int mpi3mr_cfg_set_sas_io_unit_pg1(struct mpi3mr_ioc *mrioc,
5837	struct mpi3_sas_io_unit_page1 *sas_io_unit_pg1, u16 pg_sz)
5838	{
5839	struct mpi3_config_page_header cfg_hdr;
5840	struct mpi3_config_request cfg_req;
5841	u16 ioc_status = 0;
5842
5843	memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5844	memset(&cfg_req, 0, sizeof(cfg_req));
5845
5846	cfg_req.function = MPI3_FUNCTION_CONFIG;
5847	cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5848	cfg_req.page_type = MPI3_CONFIG_PAGETYPE_SAS_IO_UNIT;
5849	cfg_req.page_number = 1;
5850	cfg_req.page_address = 0;
5851
5852	if (mpi3mr_process_cfg_req(mrioc, cfg_req: &cfg_req, NULL,
5853	MPI3MR_INTADMCMD_TIMEOUT, ioc_status: &ioc_status, cfg_buf: &cfg_hdr, cfg_buf_sz: sizeof(cfg_hdr))) {
5854	ioc_err(mrioc, "sas io unit page1 header read failed\n");
5855	goto out_failed;
5856	}
5857	if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5858	ioc_err(mrioc, "sas io unit page1 header read failed with ioc_status(0x%04x)\n",
5859	ioc_status);
5860	goto out_failed;
5861	}
5862	cfg_req.action = MPI3_CONFIG_ACTION_WRITE_CURRENT;
5863
5864	if (mpi3mr_process_cfg_req(mrioc, cfg_req: &cfg_req, cfg_hdr: &cfg_hdr,
5865	MPI3MR_INTADMCMD_TIMEOUT, ioc_status: &ioc_status, cfg_buf: sas_io_unit_pg1, cfg_buf_sz: pg_sz)) {
5866	ioc_err(mrioc, "sas io unit page1 write current failed\n");
5867	goto out_failed;
5868	}
5869	if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5870	ioc_err(mrioc, "sas io unit page1 write current failed with ioc_status(0x%04x)\n",
5871	ioc_status);
5872	goto out_failed;
5873	}
5874
5875	cfg_req.action = MPI3_CONFIG_ACTION_WRITE_PERSISTENT;
5876
5877	if (mpi3mr_process_cfg_req(mrioc, cfg_req: &cfg_req, cfg_hdr: &cfg_hdr,
5878	MPI3MR_INTADMCMD_TIMEOUT, ioc_status: &ioc_status, cfg_buf: sas_io_unit_pg1, cfg_buf_sz: pg_sz)) {
5879	ioc_err(mrioc, "sas io unit page1 write persistent failed\n");
5880	goto out_failed;
5881	}
5882	if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5883	ioc_err(mrioc, "sas io unit page1 write persistent failed with ioc_status(0x%04x)\n",
5884	ioc_status);
5885	goto out_failed;
5886	}
5887	return 0;
5888	out_failed:
5889	return -1;
5890	}
5891
5892	/**
5893	* mpi3mr_cfg_get_driver_pg1 - Read current Driver page1
5894	* @mrioc: Adapter instance reference
5895	* @driver_pg1: Pointer to return Driver page 1
5896	* @pg_sz: Size of the memory allocated to the page pointer
5897	*
5898	* This is handler for config page read for the Driver page1.
5899	* This routine checks ioc_status to decide whether the page
5900	* read is success or not.
5901	*
5902	* Return: 0 on success, non-zero on failure.
5903	*/
5904	int mpi3mr_cfg_get_driver_pg1(struct mpi3mr_ioc *mrioc,
5905	struct mpi3_driver_page1 *driver_pg1, u16 pg_sz)
5906	{
5907	struct mpi3_config_page_header cfg_hdr;
5908	struct mpi3_config_request cfg_req;
5909	u16 ioc_status = 0;
5910
5911	memset(driver_pg1, 0, pg_sz);
5912	memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5913	memset(&cfg_req, 0, sizeof(cfg_req));
5914
5915	cfg_req.function = MPI3_FUNCTION_CONFIG;
5916	cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5917	cfg_req.page_type = MPI3_CONFIG_PAGETYPE_DRIVER;
5918	cfg_req.page_number = 1;
5919	cfg_req.page_address = 0;
5920
5921	if (mpi3mr_process_cfg_req(mrioc, cfg_req: &cfg_req, NULL,
5922	MPI3MR_INTADMCMD_TIMEOUT, ioc_status: &ioc_status, cfg_buf: &cfg_hdr, cfg_buf_sz: sizeof(cfg_hdr))) {
5923	ioc_err(mrioc, "driver page1 header read failed\n");
5924	goto out_failed;
5925	}
5926	if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5927	ioc_err(mrioc, "driver page1 header read failed with ioc_status(0x%04x)\n",
5928	ioc_status);
5929	goto out_failed;
5930	}
5931	cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT;
5932
5933	if (mpi3mr_process_cfg_req(mrioc, cfg_req: &cfg_req, cfg_hdr: &cfg_hdr,
5934	MPI3MR_INTADMCMD_TIMEOUT, ioc_status: &ioc_status, cfg_buf: driver_pg1, cfg_buf_sz: pg_sz)) {
5935	ioc_err(mrioc, "driver page1 read failed\n");
5936	goto out_failed;
5937	}
5938	if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5939	ioc_err(mrioc, "driver page1 read failed with ioc_status(0x%04x)\n",
5940	ioc_status);
5941	goto out_failed;
5942	}
5943	return 0;
5944	out_failed:
5945	return -1;
5946	}
5947