1	/*
2	3w-sas.c -- LSI 3ware SAS/SATA-RAID Controller device driver for Linux.
3
4	Written By: Adam Radford <aradford@gmail.com>
5
6	Copyright (C) 2009 LSI Corporation.
7
8	This program is free software; you can redistribute it and/or modify
9	it under the terms of the GNU General Public License as published by
10	the Free Software Foundation; version 2 of the License.
11
12	This program is distributed in the hope that it will be useful,
13	but WITHOUT ANY WARRANTY; without even the implied warranty of
14	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15	GNU General Public License for more details.
16
17	NO WARRANTY
18	THE PROGRAM IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR
19	CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT
20	LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT,
21	MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is
22	solely responsible for determining the appropriateness of using and
23	distributing the Program and assumes all risks associated with its
24	exercise of rights under this Agreement, including but not limited to
25	the risks and costs of program errors, damage to or loss of data,
26	programs or equipment, and unavailability or interruption of operations.
27
28	DISCLAIMER OF LIABILITY
29	NEITHER RECIPIENT NOR ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY
30	DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
31	DAMAGES (INCLUDING WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND
32	ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
33	TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
34	USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED
35	HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES
36
37	You should have received a copy of the GNU General Public License
38	along with this program; if not, write to the Free Software
39	Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
40
41	Controllers supported by this driver:
42
43	LSI 3ware 9750 6Gb/s SAS/SATA-RAID
44
45	Bugs/Comments/Suggestions should be mailed to:
46	aradford@gmail.com
47
48	History
49	-------
50	3.26.02.000 - Initial driver release.
51	*/
52
53	#include <linux/module.h>
54	#include <linux/reboot.h>
55	#include <linux/spinlock.h>
56	#include <linux/interrupt.h>
57	#include <linux/moduleparam.h>
58	#include <linux/errno.h>
59	#include <linux/types.h>
60	#include <linux/delay.h>
61	#include <linux/pci.h>
62	#include <linux/time.h>
63	#include <linux/mutex.h>
64	#include <linux/slab.h>
65	#include <asm/io.h>
66	#include <asm/irq.h>
67	#include <linux/uaccess.h>
68	#include <scsi/scsi.h>
69	#include <scsi/scsi_host.h>
70	#include <scsi/scsi_tcq.h>
71	#include <scsi/scsi_cmnd.h>
72	#include "3w-sas.h"
73
74	/* Globals */
75	#define TW_DRIVER_VERSION "3.26.02.000"
76	static DEFINE_MUTEX(twl_chrdev_mutex);
77	static TW_Device_Extension *twl_device_extension_list[TW_MAX_SLOT];
78	static unsigned int twl_device_extension_count;
79	static int twl_major = -1;
80	extern struct timezone sys_tz;
81
82	/* Module parameters */
83	MODULE_AUTHOR ("LSI");
84	MODULE_DESCRIPTION ("LSI 3ware SAS/SATA-RAID Linux Driver");
85	MODULE_LICENSE("GPL");
86	MODULE_VERSION(TW_DRIVER_VERSION);
87
88	static int use_msi;
89	module_param(use_msi, int, S_IRUGO);
90	MODULE_PARM_DESC(use_msi, "Use Message Signaled Interrupts. Default: 0");
91
92	/* Function prototypes */
93	static int twl_reset_device_extension(TW_Device_Extension *tw_dev, int ioctl_reset);
94
95	/* Functions */
96
97	/* This function returns AENs through sysfs */
98	static ssize_t twl_sysfs_aen_read(struct file *filp, struct kobject *kobj,
99	struct bin_attribute *bin_attr,
100	char *outbuf, loff_t offset, size_t count)
101	{
102	struct device *dev = container_of(kobj, struct device, kobj);
103	struct Scsi_Host *shost = class_to_shost(dev);
104	TW_Device_Extension *tw_dev = (TW_Device_Extension *)shost->hostdata;
105	unsigned long flags = 0;
106	ssize_t ret;
107
108	if (!capable(CAP_SYS_ADMIN))
109	return -EACCES;
110
111	spin_lock_irqsave(tw_dev->host->host_lock, flags);
112	ret = memory_read_from_buffer(to: outbuf, count, ppos: &offset, from: tw_dev->event_queue[0], available: sizeof(TW_Event) * TW_Q_LENGTH);
113	spin_unlock_irqrestore(lock: tw_dev->host->host_lock, flags);
114
115	return ret;
116	} /* End twl_sysfs_aen_read() */
117
118	/* aen_read sysfs attribute initializer */
119	static struct bin_attribute twl_sysfs_aen_read_attr = {
120	.attr = {
121	.name = "3ware_aen_read",
122	.mode = S_IRUSR,
123	},
124	.size = 0,
125	.read = twl_sysfs_aen_read
126	};
127
128	/* This function returns driver compatibility info through sysfs */
129	static ssize_t twl_sysfs_compat_info(struct file *filp, struct kobject *kobj,
130	struct bin_attribute *bin_attr,
131	char *outbuf, loff_t offset, size_t count)
132	{
133	struct device *dev = container_of(kobj, struct device, kobj);
134	struct Scsi_Host *shost = class_to_shost(dev);
135	TW_Device_Extension *tw_dev = (TW_Device_Extension *)shost->hostdata;
136	unsigned long flags = 0;
137	ssize_t ret;
138
139	if (!capable(CAP_SYS_ADMIN))
140	return -EACCES;
141
142	spin_lock_irqsave(tw_dev->host->host_lock, flags);
143	ret = memory_read_from_buffer(to: outbuf, count, ppos: &offset, from: &tw_dev->tw_compat_info, available: sizeof(TW_Compatibility_Info));
144	spin_unlock_irqrestore(lock: tw_dev->host->host_lock, flags);
145
146	return ret;
147	} /* End twl_sysfs_compat_info() */
148
149	/* compat_info sysfs attribute initializer */
150	static struct bin_attribute twl_sysfs_compat_info_attr = {
151	.attr = {
152	.name = "3ware_compat_info",
153	.mode = S_IRUSR,
154	},
155	.size = 0,
156	.read = twl_sysfs_compat_info
157	};
158
159	/* Show some statistics about the card */
160	static ssize_t twl_show_stats(struct device *dev,
161	struct device_attribute *attr, char *buf)
162	{
163	struct Scsi_Host *host = class_to_shost(dev);
164	TW_Device_Extension *tw_dev = (TW_Device_Extension *)host->hostdata;
165	unsigned long flags = 0;
166	ssize_t len;
167
168	spin_lock_irqsave(tw_dev->host->host_lock, flags);
169	len = sysfs_emit(buf, fmt: "3w-sas Driver version: %s\n"
170	"Current commands posted: %4d\n"
171	"Max commands posted: %4d\n"
172	"Last sgl length: %4d\n"
173	"Max sgl length: %4d\n"
174	"Last sector count: %4d\n"
175	"Max sector count: %4d\n"
176	"SCSI Host Resets: %4d\n"
177	"AEN's: %4d\n",
178	TW_DRIVER_VERSION,
179	tw_dev->posted_request_count,
180	tw_dev->max_posted_request_count,
181	tw_dev->sgl_entries,
182	tw_dev->max_sgl_entries,
183	tw_dev->sector_count,
184	tw_dev->max_sector_count,
185	tw_dev->num_resets,
186	tw_dev->aen_count);
187	spin_unlock_irqrestore(lock: tw_dev->host->host_lock, flags);
188	return len;
189	} /* End twl_show_stats() */
190
191	/* stats sysfs attribute initializer */
192	static struct device_attribute twl_host_stats_attr = {
193	.attr = {
194	.name = "3ware_stats",
195	.mode = S_IRUGO,
196	},
197	.show = twl_show_stats
198	};
199
200	/* Host attributes initializer */
201	static struct attribute *twl_host_attrs[] = {
202	&twl_host_stats_attr.attr,
203	NULL,
204	};
205
206	ATTRIBUTE_GROUPS(twl_host);
207
208	/* This function will look up an AEN severity string */
209	static char *twl_aen_severity_lookup(unsigned char severity_code)
210	{
211	char *retval = NULL;
212
213	if ((severity_code < (unsigned char) TW_AEN_SEVERITY_ERROR) ||
214	(severity_code > (unsigned char) TW_AEN_SEVERITY_DEBUG))
215	goto out;
216
217	retval = twl_aen_severity_table[severity_code];
218	out:
219	return retval;
220	} /* End twl_aen_severity_lookup() */
221
222	/* This function will queue an event */
223	static void twl_aen_queue_event(TW_Device_Extension *tw_dev, TW_Command_Apache_Header *header)
224	{
225	u32 local_time;
226	TW_Event *event;
227	unsigned short aen;
228	char host[16];
229	char *error_str;
230
231	tw_dev->aen_count++;
232
233	/* Fill out event info */
234	event = tw_dev->event_queue[tw_dev->error_index];
235
236	host[0] = '\0';
237	if (tw_dev->host)
238	sprintf(buf: host, fmt: " scsi%d:", tw_dev->host->host_no);
239
240	aen = le16_to_cpu(header->status_block.error);
241	memset(event, 0, sizeof(TW_Event));
242
243	event->severity = TW_SEV_OUT(header->status_block.severity__reserved);
244	/* event->time_stamp_sec overflows in y2106 */
245	local_time = (u32)(ktime_get_real_seconds() - (sys_tz.tz_minuteswest * 60));
246	event->time_stamp_sec = local_time;
247	event->aen_code = aen;
248	event->retrieved = TW_AEN_NOT_RETRIEVED;
249	event->sequence_id = tw_dev->error_sequence_id;
250	tw_dev->error_sequence_id++;
251
252	/* Check for embedded error string */
253	error_str = &(header->err_specific_desc[strlen(header->err_specific_desc)+1]);
254
255	header->err_specific_desc[sizeof(header->err_specific_desc) - 1] = '\0';
256	event->parameter_len = strlen(header->err_specific_desc);
257	memcpy(event->parameter_data, header->err_specific_desc, event->parameter_len + 1 + strlen(error_str));
258	if (event->severity != TW_AEN_SEVERITY_DEBUG)
259	printk(KERN_WARNING "3w-sas:%s AEN: %s (0x%02X:0x%04X): %s:%s.\n",
260	host,
261	twl_aen_severity_lookup(TW_SEV_OUT(header->status_block.severity__reserved)),
262	TW_MESSAGE_SOURCE_CONTROLLER_EVENT, aen, error_str,
263	header->err_specific_desc);
264	else
265	tw_dev->aen_count--;
266
267	tw_dev->error_index = (tw_dev->error_index + 1 ) % TW_Q_LENGTH;
268	} /* End twl_aen_queue_event() */
269
270	/* This function will attempt to post a command packet to the board */
271	static int twl_post_command_packet(TW_Device_Extension *tw_dev, int request_id)
272	{
273	dma_addr_t command_que_value;
274
275	command_que_value = tw_dev->command_packet_phys[request_id];
276	command_que_value += TW_COMMAND_OFFSET;
277
278	/* First write upper 4 bytes */
279	writel(val: (u32)((u64)command_que_value >> 32), TWL_HIBQPH_REG_ADDR(tw_dev));
280	/* Then the lower 4 bytes */
281	writel(val: (u32)(command_que_value | TWL_PULL_MODE), TWL_HIBQPL_REG_ADDR(tw_dev));
282
283	tw_dev->state[request_id] = TW_S_POSTED;
284	tw_dev->posted_request_count++;
285	if (tw_dev->posted_request_count > tw_dev->max_posted_request_count)
286	tw_dev->max_posted_request_count = tw_dev->posted_request_count;
287
288	return 0;
289	} /* End twl_post_command_packet() */
290
291	/* This function hands scsi cdb's to the firmware */
292	static int twl_scsiop_execute_scsi(TW_Device_Extension *tw_dev, int request_id,
293	unsigned char *cdb, int use_sg,
294	TW_SG_Entry_ISO *sglistarg)
295	{
296	TW_Command_Full *full_command_packet;
297	TW_Command_Apache *command_packet;
298	int i, sg_count;
299	struct scsi_cmnd *srb = NULL;
300	struct scatterlist *sg;
301	int retval = 1;
302
303	if (tw_dev->srb[request_id])
304	srb = tw_dev->srb[request_id];
305
306	/* Initialize command packet */
307	full_command_packet = tw_dev->command_packet_virt[request_id];
308	full_command_packet->header.header_desc.size_header = 128;
309	full_command_packet->header.status_block.error = 0;
310	full_command_packet->header.status_block.severity__reserved = 0;
311
312	command_packet = &full_command_packet->command.newcommand;
313	command_packet->status = 0;
314	command_packet->opcode__reserved = TW_OPRES_IN(0, TW_OP_EXECUTE_SCSI);
315
316	/* We forced 16 byte cdb use earlier */
317	if (!cdb)
318	memcpy(command_packet->cdb, srb->cmnd, TW_MAX_CDB_LEN);
319	else
320	memcpy(command_packet->cdb, cdb, TW_MAX_CDB_LEN);
321
322	if (srb) {
323	command_packet->unit = srb->device->id;
324	command_packet->request_id__lunl =
325	cpu_to_le16(TW_REQ_LUN_IN(srb->device->lun, request_id));
326	} else {
327	command_packet->request_id__lunl =
328	cpu_to_le16(TW_REQ_LUN_IN(0, request_id));
329	command_packet->unit = 0;
330	}
331
332	command_packet->sgl_offset = 16;
333
334	if (!sglistarg) {
335	/* Map sglist from scsi layer to cmd packet */
336	if (scsi_sg_count(cmd: srb)) {
337	sg_count = scsi_dma_map(cmd: srb);
338	if (sg_count <= 0)
339	goto out;
340
341	scsi_for_each_sg(srb, sg, sg_count, i) {
342	command_packet->sg_list[i].address = TW_CPU_TO_SGL(sg_dma_address(sg));
343	command_packet->sg_list[i].length = TW_CPU_TO_SGL(sg_dma_len(sg));
344	}
345	command_packet->sgl_entries__lunh = cpu_to_le16(TW_REQ_LUN_IN((srb->device->lun >> 4), scsi_sg_count(tw_dev->srb[request_id])));
346	}
347	} else {
348	/* Internal cdb post */
349	for (i = 0; i < use_sg; i++) {
350	command_packet->sg_list[i].address = TW_CPU_TO_SGL(sglistarg[i].address);
351	command_packet->sg_list[i].length = TW_CPU_TO_SGL(sglistarg[i].length);
352	}
353	command_packet->sgl_entries__lunh = cpu_to_le16(TW_REQ_LUN_IN(0, use_sg));
354	}
355
356	/* Update some stats */
357	if (srb) {
358	tw_dev->sector_count = scsi_bufflen(cmd: srb) / 512;
359	if (tw_dev->sector_count > tw_dev->max_sector_count)
360	tw_dev->max_sector_count = tw_dev->sector_count;
361	tw_dev->sgl_entries = scsi_sg_count(cmd: srb);
362	if (tw_dev->sgl_entries > tw_dev->max_sgl_entries)
363	tw_dev->max_sgl_entries = tw_dev->sgl_entries;
364	}
365
366	/* Now post the command to the board */
367	retval = twl_post_command_packet(tw_dev, request_id);
368
369	out:
370	return retval;
371	} /* End twl_scsiop_execute_scsi() */
372
373	/* This function will read the aen queue from the isr */
374	static int twl_aen_read_queue(TW_Device_Extension *tw_dev, int request_id)
375	{
376	unsigned char cdb[TW_MAX_CDB_LEN];
377	TW_SG_Entry_ISO sglist[1];
378	TW_Command_Full *full_command_packet;
379	int retval = 1;
380
381	full_command_packet = tw_dev->command_packet_virt[request_id];
382	memset(full_command_packet, 0, sizeof(TW_Command_Full));
383
384	/* Initialize cdb */
385	memset(&cdb, 0, TW_MAX_CDB_LEN);
386	cdb[0] = REQUEST_SENSE; /* opcode */
387	cdb[4] = TW_ALLOCATION_LENGTH; /* allocation length */
388
389	/* Initialize sglist */
390	memset(&sglist, 0, sizeof(TW_SG_Entry_ISO));
391	sglist[0].length = TW_SECTOR_SIZE;
392	sglist[0].address = tw_dev->generic_buffer_phys[request_id];
393
394	/* Mark internal command */
395	tw_dev->srb[request_id] = NULL;
396
397	/* Now post the command packet */
398	if (twl_scsiop_execute_scsi(tw_dev, request_id, cdb, use_sg: 1, sglistarg: sglist)) {
399	TW_PRINTK(tw_dev->host, TW_DRIVER, 0x2, "Post failed while reading AEN queue");
400	goto out;
401	}
402	retval = 0;
403	out:
404	return retval;
405	} /* End twl_aen_read_queue() */
406
407	/* This function will sync firmware time with the host time */
408	static void twl_aen_sync_time(TW_Device_Extension *tw_dev, int request_id)
409	{
410	u32 schedulertime;
411	TW_Command_Full *full_command_packet;
412	TW_Command *command_packet;
413	TW_Param_Apache *param;
414	time64_t local_time;
415
416	/* Fill out the command packet */
417	full_command_packet = tw_dev->command_packet_virt[request_id];
418	memset(full_command_packet, 0, sizeof(TW_Command_Full));
419	command_packet = &full_command_packet->command.oldcommand;
420	command_packet->opcode__sgloffset = TW_OPSGL_IN(2, TW_OP_SET_PARAM);
421	command_packet->request_id = request_id;
422	command_packet->byte8_offset.param.sgl[0].address = TW_CPU_TO_SGL(tw_dev->generic_buffer_phys[request_id]);
423	command_packet->byte8_offset.param.sgl[0].length = TW_CPU_TO_SGL(TW_SECTOR_SIZE);
424	command_packet->size = TW_COMMAND_SIZE;
425	command_packet->byte6_offset.parameter_count = cpu_to_le16(1);
426
427	/* Setup the param */
428	param = (TW_Param_Apache *)tw_dev->generic_buffer_virt[request_id];
429	memset(param, 0, TW_SECTOR_SIZE);
430	param->table_id = cpu_to_le16(TW_TIMEKEEP_TABLE | 0x8000); /* Controller time keep table */
431	param->parameter_id = cpu_to_le16(0x3); /* SchedulerTime */
432	param->parameter_size_bytes = cpu_to_le16(4);
433
434	/* Convert system time in UTC to local time seconds since last
435	Sunday 12:00AM */
436	local_time = (ktime_get_real_seconds() - (sys_tz.tz_minuteswest * 60));
437	div_u64_rem(dividend: local_time - (3 * 86400), divisor: 604800, remainder: &schedulertime);
438	schedulertime = cpu_to_le32(schedulertime);
439
440	memcpy(param->data, &schedulertime, sizeof(u32));
441
442	/* Mark internal command */
443	tw_dev->srb[request_id] = NULL;
444
445	/* Now post the command */
446	twl_post_command_packet(tw_dev, request_id);
447	} /* End twl_aen_sync_time() */
448
449	/* This function will assign an available request id */
450	static void twl_get_request_id(TW_Device_Extension *tw_dev, int *request_id)
451	{
452	*request_id = tw_dev->free_queue[tw_dev->free_head];
453	tw_dev->free_head = (tw_dev->free_head + 1) % TW_Q_LENGTH;
454	tw_dev->state[*request_id] = TW_S_STARTED;
455	} /* End twl_get_request_id() */
456
457	/* This function will free a request id */
458	static void twl_free_request_id(TW_Device_Extension *tw_dev, int request_id)
459	{
460	tw_dev->free_queue[tw_dev->free_tail] = request_id;
461	tw_dev->state[request_id] = TW_S_FINISHED;
462	tw_dev->free_tail = (tw_dev->free_tail + 1) % TW_Q_LENGTH;
463	} /* End twl_free_request_id() */
464
465	/* This function will complete an aen request from the isr */
466	static int twl_aen_complete(TW_Device_Extension *tw_dev, int request_id)
467	{
468	TW_Command_Full *full_command_packet;
469	TW_Command *command_packet;
470	TW_Command_Apache_Header *header;
471	unsigned short aen;
472	int retval = 1;
473
474	header = (TW_Command_Apache_Header *)tw_dev->generic_buffer_virt[request_id];
475	tw_dev->posted_request_count--;
476	aen = le16_to_cpu(header->status_block.error);
477	full_command_packet = tw_dev->command_packet_virt[request_id];
478	command_packet = &full_command_packet->command.oldcommand;
479
480	/* First check for internal completion of set param for time sync */
481	if (TW_OP_OUT(command_packet->opcode__sgloffset) == TW_OP_SET_PARAM) {
482	/* Keep reading the queue in case there are more aen's */
483	if (twl_aen_read_queue(tw_dev, request_id))
484	goto out2;
485	else {
486	retval = 0;
487	goto out;
488	}
489	}
490
491	switch (aen) {
492	case TW_AEN_QUEUE_EMPTY:
493	/* Quit reading the queue if this is the last one */
494	break;
495	case TW_AEN_SYNC_TIME_WITH_HOST:
496	twl_aen_sync_time(tw_dev, request_id);
497	retval = 0;
498	goto out;
499	default:
500	twl_aen_queue_event(tw_dev, header);
501
502	/* If there are more aen's, keep reading the queue */
503	if (twl_aen_read_queue(tw_dev, request_id))
504	goto out2;
505	else {
506	retval = 0;
507	goto out;
508	}
509	}
510	retval = 0;
511	out2:
512	tw_dev->state[request_id] = TW_S_COMPLETED;
513	twl_free_request_id(tw_dev, request_id);
514	clear_bit(TW_IN_ATTENTION_LOOP, addr: &tw_dev->flags);
515	out:
516	return retval;
517	} /* End twl_aen_complete() */
518
519	/* This function will poll for a response */
520	static int twl_poll_response(TW_Device_Extension *tw_dev, int request_id, int seconds)
521	{
522	unsigned long before;
523	dma_addr_t mfa;
524	u32 regh, regl;
525	u32 response;
526	int retval = 1;
527	int found = 0;
528
529	before = jiffies;
530
531	while (!found) {
532	if (sizeof(dma_addr_t) > 4) {
533	regh = readl(TWL_HOBQPH_REG_ADDR(tw_dev));
534	regl = readl(TWL_HOBQPL_REG_ADDR(tw_dev));
535	mfa = ((u64)regh << 32) | regl;
536	} else
537	mfa = readl(TWL_HOBQPL_REG_ADDR(tw_dev));
538
539	response = (u32)mfa;
540
541	if (TW_RESID_OUT(response) == request_id)
542	found = 1;
543
544	if (time_after(jiffies, before + HZ * seconds))
545	goto out;
546
547	msleep(msecs: 50);
548	}
549	retval = 0;
550	out:
551	return retval;
552	} /* End twl_poll_response() */
553
554	/* This function will drain the aen queue */
555	static int twl_aen_drain_queue(TW_Device_Extension *tw_dev, int no_check_reset)
556	{
557	int request_id = 0;
558	unsigned char cdb[TW_MAX_CDB_LEN];
559	TW_SG_Entry_ISO sglist[1];
560	int finished = 0, count = 0;
561	TW_Command_Full *full_command_packet;
562	TW_Command_Apache_Header *header;
563	unsigned short aen;
564	int first_reset = 0, queue = 0, retval = 1;
565
566	if (no_check_reset)
567	first_reset = 0;
568	else
569	first_reset = 1;
570
571	full_command_packet = tw_dev->command_packet_virt[request_id];
572	memset(full_command_packet, 0, sizeof(TW_Command_Full));
573
574	/* Initialize cdb */
575	memset(&cdb, 0, TW_MAX_CDB_LEN);
576	cdb[0] = REQUEST_SENSE; /* opcode */
577	cdb[4] = TW_ALLOCATION_LENGTH; /* allocation length */
578
579	/* Initialize sglist */
580	memset(&sglist, 0, sizeof(TW_SG_Entry_ISO));
581	sglist[0].length = TW_SECTOR_SIZE;
582	sglist[0].address = tw_dev->generic_buffer_phys[request_id];
583
584	/* Mark internal command */
585	tw_dev->srb[request_id] = NULL;
586
587	do {
588	/* Send command to the board */
589	if (twl_scsiop_execute_scsi(tw_dev, request_id, cdb, use_sg: 1, sglistarg: sglist)) {
590	TW_PRINTK(tw_dev->host, TW_DRIVER, 0x3, "Error posting request sense");
591	goto out;
592	}
593
594	/* Now poll for completion */
595	if (twl_poll_response(tw_dev, request_id, seconds: 30)) {
596	TW_PRINTK(tw_dev->host, TW_DRIVER, 0x4, "No valid response while draining AEN queue");
597	tw_dev->posted_request_count--;
598	goto out;
599	}
600
601	tw_dev->posted_request_count--;
602	header = (TW_Command_Apache_Header *)tw_dev->generic_buffer_virt[request_id];
603	aen = le16_to_cpu(header->status_block.error);
604	queue = 0;
605	count++;
606
607	switch (aen) {
608	case TW_AEN_QUEUE_EMPTY:
609	if (first_reset != 1)
610	goto out;
611	else
612	finished = 1;
613	break;
614	case TW_AEN_SOFT_RESET:
615	if (first_reset == 0)
616	first_reset = 1;
617	else
618	queue = 1;
619	break;
620	case TW_AEN_SYNC_TIME_WITH_HOST:
621	break;
622	default:
623	queue = 1;
624	}
625
626	/* Now queue an event info */
627	if (queue)
628	twl_aen_queue_event(tw_dev, header);
629	} while ((finished == 0) && (count < TW_MAX_AEN_DRAIN));
630
631	if (count == TW_MAX_AEN_DRAIN)
632	goto out;
633
634	retval = 0;
635	out:
636	tw_dev->state[request_id] = TW_S_INITIAL;
637	return retval;
638	} /* End twl_aen_drain_queue() */
639
640	/* This function will allocate memory and check if it is correctly aligned */
641	static int twl_allocate_memory(TW_Device_Extension *tw_dev, int size, int which)
642	{
643	int i;
644	dma_addr_t dma_handle;
645	unsigned long *cpu_addr;
646	int retval = 1;
647
648	cpu_addr = dma_alloc_coherent(dev: &tw_dev->tw_pci_dev->dev,
649	size: size * TW_Q_LENGTH, dma_handle: &dma_handle,
650	GFP_KERNEL);
651	if (!cpu_addr) {
652	TW_PRINTK(tw_dev->host, TW_DRIVER, 0x5, "Memory allocation failed");
653	goto out;
654	}
655
656	for (i = 0; i < TW_Q_LENGTH; i++) {
657	switch(which) {
658	case 0:
659	tw_dev->command_packet_phys[i] = dma_handle+(i*size);
660	tw_dev->command_packet_virt[i] = (TW_Command_Full *)((unsigned char *)cpu_addr + (i*size));
661	break;
662	case 1:
663	tw_dev->generic_buffer_phys[i] = dma_handle+(i*size);
664	tw_dev->generic_buffer_virt[i] = (unsigned long *)((unsigned char *)cpu_addr + (i*size));
665	break;
666	case 2:
667	tw_dev->sense_buffer_phys[i] = dma_handle+(i*size);
668	tw_dev->sense_buffer_virt[i] = (TW_Command_Apache_Header *)((unsigned char *)cpu_addr + (i*size));
669	break;
670	}
671	}
672	retval = 0;
673	out:
674	return retval;
675	} /* End twl_allocate_memory() */
676
677	/* This function will load the request id and various sgls for ioctls */
678	static void twl_load_sgl(TW_Device_Extension *tw_dev, TW_Command_Full *full_command_packet, int request_id, dma_addr_t dma_handle, int length)
679	{
680	TW_Command *oldcommand;
681	TW_Command_Apache *newcommand;
682	TW_SG_Entry_ISO *sgl;
683	unsigned int pae = 0;
684
685	if ((sizeof(long) < 8) && (sizeof(dma_addr_t) > 4))
686	pae = 1;
687
688	if (TW_OP_OUT(full_command_packet->command.newcommand.opcode__reserved) == TW_OP_EXECUTE_SCSI) {
689	newcommand = &full_command_packet->command.newcommand;
690	newcommand->request_id__lunl =
691	cpu_to_le16(TW_REQ_LUN_IN(TW_LUN_OUT(newcommand->request_id__lunl), request_id));
692	if (length) {
693	newcommand->sg_list[0].address = TW_CPU_TO_SGL(dma_handle + sizeof(TW_Ioctl_Buf_Apache));
694	newcommand->sg_list[0].length = TW_CPU_TO_SGL(length);
695	}
696	newcommand->sgl_entries__lunh =
697	cpu_to_le16(TW_REQ_LUN_IN(TW_LUN_OUT(newcommand->sgl_entries__lunh), length ? 1 : 0));
698	} else {
699	oldcommand = &full_command_packet->command.oldcommand;
700	oldcommand->request_id = request_id;
701
702	if (TW_SGL_OUT(oldcommand->opcode__sgloffset)) {
703	/* Load the sg list */
704	sgl = (TW_SG_Entry_ISO *)((u32 *)oldcommand+oldcommand->size - (sizeof(TW_SG_Entry_ISO)/4) + pae + (sizeof(dma_addr_t) > 4 ? 1 : 0));
705	sgl->address = TW_CPU_TO_SGL(dma_handle + sizeof(TW_Ioctl_Buf_Apache));
706	sgl->length = TW_CPU_TO_SGL(length);
707	oldcommand->size += pae;
708	oldcommand->size += sizeof(dma_addr_t) > 4 ? 1 : 0;
709	}
710	}
711	} /* End twl_load_sgl() */
712
713	/* This function handles ioctl for the character device
714	This interface is used by smartmontools open source software */
715	static long twl_chrdev_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
716	{
717	long timeout;
718	unsigned long *cpu_addr, data_buffer_length_adjusted = 0, flags = 0;
719	dma_addr_t dma_handle;
720	int request_id = 0;
721	TW_Ioctl_Driver_Command driver_command;
722	struct inode *inode = file_inode(f: file);
723	TW_Ioctl_Buf_Apache *tw_ioctl;
724	TW_Command_Full *full_command_packet;
725	TW_Device_Extension *tw_dev = twl_device_extension_list[iminor(inode)];
726	int retval = -EFAULT;
727	void __user *argp = (void __user *)arg;
728
729	mutex_lock(&twl_chrdev_mutex);
730
731	/* Only let one of these through at a time */
732	if (mutex_lock_interruptible(&tw_dev->ioctl_lock)) {
733	retval = -EINTR;
734	goto out;
735	}
736
737	/* First copy down the driver command */
738	if (copy_from_user(to: &driver_command, from: argp, n: sizeof(TW_Ioctl_Driver_Command)))
739	goto out2;
740
741	/* Check data buffer size */
742	if (driver_command.buffer_length > TW_MAX_SECTORS * 2048) {
743	retval = -EINVAL;
744	goto out2;
745	}
746
747	/* Hardware can only do multiple of 512 byte transfers */
748	data_buffer_length_adjusted = (driver_command.buffer_length + 511) & ~511;
749
750	/* Now allocate ioctl buf memory */
751	cpu_addr = dma_alloc_coherent(dev: &tw_dev->tw_pci_dev->dev, size: data_buffer_length_adjusted + sizeof(TW_Ioctl_Buf_Apache), dma_handle: &dma_handle, GFP_KERNEL);
752	if (!cpu_addr) {
753	retval = -ENOMEM;
754	goto out2;
755	}
756
757	tw_ioctl = (TW_Ioctl_Buf_Apache *)cpu_addr;
758
759	/* Now copy down the entire ioctl */
760	if (copy_from_user(to: tw_ioctl, from: argp, n: driver_command.buffer_length + sizeof(TW_Ioctl_Buf_Apache)))
761	goto out3;
762
763	/* See which ioctl we are doing */
764	switch (cmd) {
765	case TW_IOCTL_FIRMWARE_PASS_THROUGH:
766	spin_lock_irqsave(tw_dev->host->host_lock, flags);
767	twl_get_request_id(tw_dev, request_id: &request_id);
768
769	/* Flag internal command */
770	tw_dev->srb[request_id] = NULL;
771
772	/* Flag chrdev ioctl */
773	tw_dev->chrdev_request_id = request_id;
774
775	full_command_packet = (TW_Command_Full *)&tw_ioctl->firmware_command;
776
777	/* Load request id and sglist for both command types */
778	twl_load_sgl(tw_dev, full_command_packet, request_id, dma_handle, length: data_buffer_length_adjusted);
779
780	memcpy(tw_dev->command_packet_virt[request_id], &(tw_ioctl->firmware_command), sizeof(TW_Command_Full));
781
782	/* Now post the command packet to the controller */
783	twl_post_command_packet(tw_dev, request_id);
784	spin_unlock_irqrestore(lock: tw_dev->host->host_lock, flags);
785
786	timeout = TW_IOCTL_CHRDEV_TIMEOUT*HZ;
787
788	/* Now wait for command to complete */
789	timeout = wait_event_timeout(tw_dev->ioctl_wqueue, tw_dev->chrdev_request_id == TW_IOCTL_CHRDEV_FREE, timeout);
790
791	/* We timed out, and didn't get an interrupt */
792	if (tw_dev->chrdev_request_id != TW_IOCTL_CHRDEV_FREE) {
793	/* Now we need to reset the board */
794	printk(KERN_WARNING "3w-sas: scsi%d: WARNING: (0x%02X:0x%04X): Character ioctl (0x%x) timed out, resetting card.\n",
795	tw_dev->host->host_no, TW_DRIVER, 0x6,
796	cmd);
797	retval = -EIO;
798	twl_reset_device_extension(tw_dev, ioctl_reset: 1);
799	goto out3;
800	}
801
802	/* Now copy in the command packet response */
803	memcpy(&(tw_ioctl->firmware_command), tw_dev->command_packet_virt[request_id], sizeof(TW_Command_Full));
804
805	/* Now complete the io */
806	spin_lock_irqsave(tw_dev->host->host_lock, flags);
807	tw_dev->posted_request_count--;
808	tw_dev->state[request_id] = TW_S_COMPLETED;
809	twl_free_request_id(tw_dev, request_id);
810	spin_unlock_irqrestore(lock: tw_dev->host->host_lock, flags);
811	break;
812	default:
813	retval = -ENOTTY;
814	goto out3;
815	}
816
817	/* Now copy the entire response to userspace */
818	if (copy_to_user(to: argp, from: tw_ioctl, n: sizeof(TW_Ioctl_Buf_Apache) + driver_command.buffer_length) == 0)
819	retval = 0;
820	out3:
821	/* Now free ioctl buf memory */
822	dma_free_coherent(dev: &tw_dev->tw_pci_dev->dev, size: data_buffer_length_adjusted + sizeof(TW_Ioctl_Buf_Apache), cpu_addr, dma_handle);
823	out2:
824	mutex_unlock(lock: &tw_dev->ioctl_lock);
825	out:
826	mutex_unlock(lock: &twl_chrdev_mutex);
827	return retval;
828	} /* End twl_chrdev_ioctl() */
829
830	/* This function handles open for the character device */
831	static int twl_chrdev_open(struct inode *inode, struct file *file)
832	{
833	unsigned int minor_number;
834	int retval = -ENODEV;
835
836	if (!capable(CAP_SYS_ADMIN)) {
837	retval = -EACCES;
838	goto out;
839	}
840
841	minor_number = iminor(inode);
842	if (minor_number >= twl_device_extension_count)
843	goto out;
844	retval = 0;
845	out:
846	return retval;
847	} /* End twl_chrdev_open() */
848
849	/* File operations struct for character device */
850	static const struct file_operations twl_fops = {
851	.owner = THIS_MODULE,
852	.unlocked_ioctl = twl_chrdev_ioctl,
853	.open = twl_chrdev_open,
854	.release = NULL,
855	.llseek = noop_llseek,
856	};
857
858	/* This function passes sense data from firmware to scsi layer */
859	static int twl_fill_sense(TW_Device_Extension *tw_dev, int i, int request_id, int copy_sense, int print_host)
860	{
861	TW_Command_Apache_Header *header;
862	TW_Command_Full *full_command_packet;
863	unsigned short error;
864	char *error_str;
865
866	header = tw_dev->sense_buffer_virt[i];
867	full_command_packet = tw_dev->command_packet_virt[request_id];
868
869	/* Get embedded firmware error string */
870	error_str = &(header->err_specific_desc[strlen(header->err_specific_desc) + 1]);
871
872	/* Don't print error for Logical unit not supported during rollcall */
873	error = le16_to_cpu(header->status_block.error);
874	if ((error != TW_ERROR_LOGICAL_UNIT_NOT_SUPPORTED) && (error != TW_ERROR_UNIT_OFFLINE) && (error != TW_ERROR_INVALID_FIELD_IN_CDB)) {
875	if (print_host)
876	printk(KERN_WARNING "3w-sas: scsi%d: ERROR: (0x%02X:0x%04X): %s:%s.\n",
877	tw_dev->host->host_no,
878	TW_MESSAGE_SOURCE_CONTROLLER_ERROR,
879	header->status_block.error,
880	error_str,
881	header->err_specific_desc);
882	else
883	printk(KERN_WARNING "3w-sas: ERROR: (0x%02X:0x%04X): %s:%s.\n",
884	TW_MESSAGE_SOURCE_CONTROLLER_ERROR,
885	header->status_block.error,
886	error_str,
887	header->err_specific_desc);
888	}
889
890	if (copy_sense) {
891	memcpy(tw_dev->srb[request_id]->sense_buffer, header->sense_data, TW_SENSE_DATA_LENGTH);
892	tw_dev->srb[request_id]->result = (full_command_packet->command.newcommand.status << 1);
893	goto out;
894	}
895	out:
896	return 1;
897	} /* End twl_fill_sense() */
898
899	/* This function will free up device extension resources */
900	static void twl_free_device_extension(TW_Device_Extension *tw_dev)
901	{
902	if (tw_dev->command_packet_virt[0])
903	dma_free_coherent(dev: &tw_dev->tw_pci_dev->dev,
904	size: sizeof(TW_Command_Full)*TW_Q_LENGTH,
905	cpu_addr: tw_dev->command_packet_virt[0],
906	dma_handle: tw_dev->command_packet_phys[0]);
907
908	if (tw_dev->generic_buffer_virt[0])
909	dma_free_coherent(dev: &tw_dev->tw_pci_dev->dev,
910	TW_SECTOR_SIZE*TW_Q_LENGTH,
911	cpu_addr: tw_dev->generic_buffer_virt[0],
912	dma_handle: tw_dev->generic_buffer_phys[0]);
913
914	if (tw_dev->sense_buffer_virt[0])
915	dma_free_coherent(dev: &tw_dev->tw_pci_dev->dev,
916	size: sizeof(TW_Command_Apache_Header)*
917	TW_Q_LENGTH,
918	cpu_addr: tw_dev->sense_buffer_virt[0],
919	dma_handle: tw_dev->sense_buffer_phys[0]);
920
921	kfree(objp: tw_dev->event_queue[0]);
922	} /* End twl_free_device_extension() */
923
924	/* This function will get parameter table entries from the firmware */
925	static void *twl_get_param(TW_Device_Extension *tw_dev, int request_id, int table_id, int parameter_id, int parameter_size_bytes)
926	{
927	TW_Command_Full *full_command_packet;
928	TW_Command *command_packet;
929	TW_Param_Apache *param;
930	void *retval = NULL;
931
932	/* Setup the command packet */
933	full_command_packet = tw_dev->command_packet_virt[request_id];
934	memset(full_command_packet, 0, sizeof(TW_Command_Full));
935	command_packet = &full_command_packet->command.oldcommand;
936
937	command_packet->opcode__sgloffset = TW_OPSGL_IN(2, TW_OP_GET_PARAM);
938	command_packet->size = TW_COMMAND_SIZE;
939	command_packet->request_id = request_id;
940	command_packet->byte6_offset.block_count = cpu_to_le16(1);
941
942	/* Now setup the param */
943	param = (TW_Param_Apache *)tw_dev->generic_buffer_virt[request_id];
944	memset(param, 0, TW_SECTOR_SIZE);
945	param->table_id = cpu_to_le16(table_id | 0x8000);
946	param->parameter_id = cpu_to_le16(parameter_id);
947	param->parameter_size_bytes = cpu_to_le16(parameter_size_bytes);
948
949	command_packet->byte8_offset.param.sgl[0].address = TW_CPU_TO_SGL(tw_dev->generic_buffer_phys[request_id]);
950	command_packet->byte8_offset.param.sgl[0].length = TW_CPU_TO_SGL(TW_SECTOR_SIZE);
951
952	/* Post the command packet to the board */
953	twl_post_command_packet(tw_dev, request_id);
954
955	/* Poll for completion */
956	if (twl_poll_response(tw_dev, request_id, seconds: 30))
957	TW_PRINTK(tw_dev->host, TW_DRIVER, 0x7, "No valid response during get param")
958	else
959	retval = (void *)&(param->data[0]);
960
961	tw_dev->posted_request_count--;
962	tw_dev->state[request_id] = TW_S_INITIAL;
963
964	return retval;
965	} /* End twl_get_param() */
966
967	/* This function will send an initconnection command to controller */
968	static int twl_initconnection(TW_Device_Extension *tw_dev, int message_credits,
969	u32 set_features, unsigned short current_fw_srl,
970	unsigned short current_fw_arch_id,
971	unsigned short current_fw_branch,
972	unsigned short current_fw_build,
973	unsigned short *fw_on_ctlr_srl,
974	unsigned short *fw_on_ctlr_arch_id,
975	unsigned short *fw_on_ctlr_branch,
976	unsigned short *fw_on_ctlr_build,
977	u32 *init_connect_result)
978	{
979	TW_Command_Full *full_command_packet;
980	TW_Initconnect *tw_initconnect;
981	int request_id = 0, retval = 1;
982
983	/* Initialize InitConnection command packet */
984	full_command_packet = tw_dev->command_packet_virt[request_id];
985	memset(full_command_packet, 0, sizeof(TW_Command_Full));
986	full_command_packet->header.header_desc.size_header = 128;
987
988	tw_initconnect = (TW_Initconnect *)&full_command_packet->command.oldcommand;
989	tw_initconnect->opcode__reserved = TW_OPRES_IN(0, TW_OP_INIT_CONNECTION);
990	tw_initconnect->request_id = request_id;
991	tw_initconnect->message_credits = cpu_to_le16(message_credits);
992	tw_initconnect->features = set_features;
993
994	/* Turn on 64-bit sgl support if we need to */
995	tw_initconnect->features |= sizeof(dma_addr_t) > 4 ? 1 : 0;
996
997	tw_initconnect->features = cpu_to_le32(tw_initconnect->features);
998
999	if (set_features & TW_EXTENDED_INIT_CONNECT) {
1000	tw_initconnect->size = TW_INIT_COMMAND_PACKET_SIZE_EXTENDED;
1001	tw_initconnect->fw_srl = cpu_to_le16(current_fw_srl);
1002	tw_initconnect->fw_arch_id = cpu_to_le16(current_fw_arch_id);
1003	tw_initconnect->fw_branch = cpu_to_le16(current_fw_branch);
1004	tw_initconnect->fw_build = cpu_to_le16(current_fw_build);
1005	} else
1006	tw_initconnect->size = TW_INIT_COMMAND_PACKET_SIZE;
1007
1008	/* Send command packet to the board */
1009	twl_post_command_packet(tw_dev, request_id);
1010
1011	/* Poll for completion */
1012	if (twl_poll_response(tw_dev, request_id, seconds: 30)) {
1013	TW_PRINTK(tw_dev->host, TW_DRIVER, 0x8, "No valid response during init connection");
1014	} else {
1015	if (set_features & TW_EXTENDED_INIT_CONNECT) {
1016	*fw_on_ctlr_srl = le16_to_cpu(tw_initconnect->fw_srl);
1017	*fw_on_ctlr_arch_id = le16_to_cpu(tw_initconnect->fw_arch_id);
1018	*fw_on_ctlr_branch = le16_to_cpu(tw_initconnect->fw_branch);
1019	*fw_on_ctlr_build = le16_to_cpu(tw_initconnect->fw_build);
1020	*init_connect_result = le32_to_cpu(tw_initconnect->result);
1021	}
1022	retval = 0;
1023	}
1024
1025	tw_dev->posted_request_count--;
1026	tw_dev->state[request_id] = TW_S_INITIAL;
1027
1028	return retval;
1029	} /* End twl_initconnection() */
1030
1031	/* This function will initialize the fields of a device extension */
1032	static int twl_initialize_device_extension(TW_Device_Extension *tw_dev)
1033	{
1034	int i, retval = 1;
1035
1036	/* Initialize command packet buffers */
1037	if (twl_allocate_memory(tw_dev, size: sizeof(TW_Command_Full), which: 0)) {
1038	TW_PRINTK(tw_dev->host, TW_DRIVER, 0x9, "Command packet memory allocation failed");
1039	goto out;
1040	}
1041
1042	/* Initialize generic buffer */
1043	if (twl_allocate_memory(tw_dev, TW_SECTOR_SIZE, which: 1)) {
1044	TW_PRINTK(tw_dev->host, TW_DRIVER, 0xa, "Generic memory allocation failed");
1045	goto out;
1046	}
1047
1048	/* Allocate sense buffers */
1049	if (twl_allocate_memory(tw_dev, size: sizeof(TW_Command_Apache_Header), which: 2)) {
1050	TW_PRINTK(tw_dev->host, TW_DRIVER, 0xb, "Sense buffer allocation failed");
1051	goto out;
1052	}
1053
1054	/* Allocate event info space */
1055	tw_dev->event_queue[0] = kcalloc(TW_Q_LENGTH, size: sizeof(TW_Event), GFP_KERNEL);
1056	if (!tw_dev->event_queue[0]) {
1057	TW_PRINTK(tw_dev->host, TW_DRIVER, 0xc, "Event info memory allocation failed");
1058	goto out;
1059	}
1060
1061	for (i = 0; i < TW_Q_LENGTH; i++) {
1062	tw_dev->event_queue[i] = (TW_Event *)((unsigned char *)tw_dev->event_queue[0] + (i * sizeof(TW_Event)));
1063	tw_dev->free_queue[i] = i;
1064	tw_dev->state[i] = TW_S_INITIAL;
1065	}
1066
1067	tw_dev->free_head = TW_Q_START;
1068	tw_dev->free_tail = TW_Q_START;
1069	tw_dev->error_sequence_id = 1;
1070	tw_dev->chrdev_request_id = TW_IOCTL_CHRDEV_FREE;
1071
1072	mutex_init(&tw_dev->ioctl_lock);
1073	init_waitqueue_head(&tw_dev->ioctl_wqueue);
1074
1075	retval = 0;
1076	out:
1077	return retval;
1078	} /* End twl_initialize_device_extension() */
1079
1080	/* This function will handle attention interrupts */
1081	static int twl_handle_attention_interrupt(TW_Device_Extension *tw_dev)
1082	{
1083	int retval = 1;
1084	u32 request_id, doorbell;
1085
1086	/* Read doorbell status */
1087	doorbell = readl(TWL_HOBDB_REG_ADDR(tw_dev));
1088
1089	/* Check for controller errors */
1090	if (doorbell & TWL_DOORBELL_CONTROLLER_ERROR) {
1091	TW_PRINTK(tw_dev->host, TW_DRIVER, 0xd, "Microcontroller Error: clearing");
1092	goto out;
1093	}
1094
1095	/* Check if we need to perform an AEN drain */
1096	if (doorbell & TWL_DOORBELL_ATTENTION_INTERRUPT) {
1097	if (!(test_and_set_bit(TW_IN_ATTENTION_LOOP, addr: &tw_dev->flags))) {
1098	twl_get_request_id(tw_dev, request_id: &request_id);
1099	if (twl_aen_read_queue(tw_dev, request_id)) {
1100	tw_dev->state[request_id] = TW_S_COMPLETED;
1101	twl_free_request_id(tw_dev, request_id);
1102	clear_bit(TW_IN_ATTENTION_LOOP, addr: &tw_dev->flags);
1103	}
1104	}
1105	}
1106
1107	retval = 0;
1108	out:
1109	/* Clear doorbell interrupt */
1110	TWL_CLEAR_DB_INTERRUPT(tw_dev);
1111
1112	/* Make sure the clear was flushed by reading it back */
1113	readl(TWL_HOBDBC_REG_ADDR(tw_dev));
1114
1115	return retval;
1116	} /* End twl_handle_attention_interrupt() */
1117
1118	/* Interrupt service routine */
1119	static irqreturn_t twl_interrupt(int irq, void *dev_instance)
1120	{
1121	TW_Device_Extension *tw_dev = (TW_Device_Extension *)dev_instance;
1122	int i, handled = 0, error = 0;
1123	dma_addr_t mfa = 0;
1124	u32 reg, regl, regh, response, request_id = 0;
1125	struct scsi_cmnd *cmd;
1126	TW_Command_Full *full_command_packet;
1127
1128	spin_lock(lock: tw_dev->host->host_lock);
1129
1130	/* Read host interrupt status */
1131	reg = readl(TWL_HISTAT_REG_ADDR(tw_dev));
1132
1133	/* Check if this is our interrupt, otherwise bail */
1134	if (!(reg & TWL_HISTATUS_VALID_INTERRUPT))
1135	goto twl_interrupt_bail;
1136
1137	handled = 1;
1138
1139	/* If we are resetting, bail */
1140	if (test_bit(TW_IN_RESET, &tw_dev->flags))
1141	goto twl_interrupt_bail;
1142
1143	/* Attention interrupt */
1144	if (reg & TWL_HISTATUS_ATTENTION_INTERRUPT) {
1145	if (twl_handle_attention_interrupt(tw_dev)) {
1146	TWL_MASK_INTERRUPTS(tw_dev);
1147	goto twl_interrupt_bail;
1148	}
1149	}
1150
1151	/* Response interrupt */
1152	while (reg & TWL_HISTATUS_RESPONSE_INTERRUPT) {
1153	if (sizeof(dma_addr_t) > 4) {
1154	regh = readl(TWL_HOBQPH_REG_ADDR(tw_dev));
1155	regl = readl(TWL_HOBQPL_REG_ADDR(tw_dev));
1156	mfa = ((u64)regh << 32) | regl;
1157	} else
1158	mfa = readl(TWL_HOBQPL_REG_ADDR(tw_dev));
1159
1160	error = 0;
1161	response = (u32)mfa;
1162
1163	/* Check for command packet error */
1164	if (!TW_NOTMFA_OUT(response)) {
1165	for (i=0;i<TW_Q_LENGTH;i++) {
1166	if (tw_dev->sense_buffer_phys[i] == mfa) {
1167	request_id = le16_to_cpu(tw_dev->sense_buffer_virt[i]->header_desc.request_id);
1168	if (tw_dev->srb[request_id] != NULL)
1169	error = twl_fill_sense(tw_dev, i, request_id, copy_sense: 1, print_host: 1);
1170	else {
1171	/* Skip ioctl error prints */
1172	if (request_id != tw_dev->chrdev_request_id)
1173	error = twl_fill_sense(tw_dev, i, request_id, copy_sense: 0, print_host: 1);
1174	else
1175	memcpy(tw_dev->command_packet_virt[request_id], tw_dev->sense_buffer_virt[i], sizeof(TW_Command_Apache_Header));
1176	}
1177
1178	/* Now re-post the sense buffer */
1179	writel(val: (u32)((u64)tw_dev->sense_buffer_phys[i] >> 32), TWL_HOBQPH_REG_ADDR(tw_dev));
1180	writel(val: (u32)tw_dev->sense_buffer_phys[i], TWL_HOBQPL_REG_ADDR(tw_dev));
1181	break;
1182	}
1183	}
1184	} else
1185	request_id = TW_RESID_OUT(response);
1186
1187	full_command_packet = tw_dev->command_packet_virt[request_id];
1188
1189	/* Check for correct state */
1190	if (tw_dev->state[request_id] != TW_S_POSTED) {
1191	if (tw_dev->srb[request_id] != NULL) {
1192	TW_PRINTK(tw_dev->host, TW_DRIVER, 0xe, "Received a request id that wasn't posted");
1193	TWL_MASK_INTERRUPTS(tw_dev);
1194	goto twl_interrupt_bail;
1195	}
1196	}
1197
1198	/* Check for internal command completion */
1199	if (tw_dev->srb[request_id] == NULL) {
1200	if (request_id != tw_dev->chrdev_request_id) {
1201	if (twl_aen_complete(tw_dev, request_id))
1202	TW_PRINTK(tw_dev->host, TW_DRIVER, 0xf, "Error completing AEN during attention interrupt");
1203	} else {
1204	tw_dev->chrdev_request_id = TW_IOCTL_CHRDEV_FREE;
1205	wake_up(&tw_dev->ioctl_wqueue);
1206	}
1207	} else {
1208	cmd = tw_dev->srb[request_id];
1209
1210	if (!error)
1211	cmd->result = (DID_OK << 16);
1212
1213	/* Report residual bytes for single sgl */
1214	if ((scsi_sg_count(cmd) <= 1) && (full_command_packet->command.newcommand.status == 0)) {
1215	if (full_command_packet->command.newcommand.sg_list[0].length < scsi_bufflen(cmd: tw_dev->srb[request_id]))
1216	scsi_set_resid(cmd, resid: scsi_bufflen(cmd) - full_command_packet->command.newcommand.sg_list[0].length);
1217	}
1218
1219	/* Now complete the io */
1220	scsi_dma_unmap(cmd);
1221	scsi_done(cmd);
1222	tw_dev->state[request_id] = TW_S_COMPLETED;
1223	twl_free_request_id(tw_dev, request_id);
1224	tw_dev->posted_request_count--;
1225	}
1226
1227	/* Check for another response interrupt */
1228	reg = readl(TWL_HISTAT_REG_ADDR(tw_dev));
1229	}
1230
1231	twl_interrupt_bail:
1232	spin_unlock(lock: tw_dev->host->host_lock);
1233	return IRQ_RETVAL(handled);
1234	} /* End twl_interrupt() */
1235
1236	/* This function will poll for a register change */
1237	static int twl_poll_register(TW_Device_Extension *tw_dev, void *reg, u32 value, u32 result, int seconds)
1238	{
1239	unsigned long before;
1240	int retval = 1;
1241	u32 reg_value;
1242
1243	reg_value = readl(addr: reg);
1244	before = jiffies;
1245
1246	while ((reg_value & value) != result) {
1247	reg_value = readl(addr: reg);
1248	if (time_after(jiffies, before + HZ * seconds))
1249	goto out;
1250	msleep(msecs: 50);
1251	}
1252	retval = 0;
1253	out:
1254	return retval;
1255	} /* End twl_poll_register() */
1256
1257	/* This function will reset a controller */
1258	static int twl_reset_sequence(TW_Device_Extension *tw_dev, int soft_reset)
1259	{
1260	int retval = 1;
1261	int i = 0;
1262	u32 status = 0;
1263	unsigned short fw_on_ctlr_srl = 0, fw_on_ctlr_arch_id = 0;
1264	unsigned short fw_on_ctlr_branch = 0, fw_on_ctlr_build = 0;
1265	u32 init_connect_result = 0;
1266	int tries = 0;
1267	int do_soft_reset = soft_reset;
1268
1269	while (tries < TW_MAX_RESET_TRIES) {
1270	/* Do a soft reset if one is needed */
1271	if (do_soft_reset) {
1272	TWL_SOFT_RESET(tw_dev);
1273
1274	/* Make sure controller is in a good state */
1275	if (twl_poll_register(tw_dev, TWL_SCRPD3_REG_ADDR(tw_dev), TWL_CONTROLLER_READY, result: 0x0, seconds: 30)) {
1276	TW_PRINTK(tw_dev->host, TW_DRIVER, 0x10, "Controller never went non-ready during reset sequence");
1277	tries++;
1278	continue;
1279	}
1280	if (twl_poll_register(tw_dev, TWL_SCRPD3_REG_ADDR(tw_dev), TWL_CONTROLLER_READY, TWL_CONTROLLER_READY, seconds: 60)) {
1281	TW_PRINTK(tw_dev->host, TW_DRIVER, 0x11, "Controller not ready during reset sequence");
1282	tries++;
1283	continue;
1284	}
1285	}
1286
1287	/* Initconnect */
1288	if (twl_initconnection(tw_dev, TW_INIT_MESSAGE_CREDITS,
1289	TW_EXTENDED_INIT_CONNECT, TW_CURRENT_DRIVER_SRL,
1290	TW_9750_ARCH_ID, TW_CURRENT_DRIVER_BRANCH,
1291	TW_CURRENT_DRIVER_BUILD, fw_on_ctlr_srl: &fw_on_ctlr_srl,
1292	fw_on_ctlr_arch_id: &fw_on_ctlr_arch_id, fw_on_ctlr_branch: &fw_on_ctlr_branch,
1293	fw_on_ctlr_build: &fw_on_ctlr_build, init_connect_result: &init_connect_result)) {
1294	TW_PRINTK(tw_dev->host, TW_DRIVER, 0x12, "Initconnection failed while checking SRL");
1295	do_soft_reset = 1;
1296	tries++;
1297	continue;
1298	}
1299
1300	/* Load sense buffers */
1301	while (i < TW_Q_LENGTH) {
1302	writel(val: (u32)((u64)tw_dev->sense_buffer_phys[i] >> 32), TWL_HOBQPH_REG_ADDR(tw_dev));
1303	writel(val: (u32)tw_dev->sense_buffer_phys[i], TWL_HOBQPL_REG_ADDR(tw_dev));
1304
1305	/* Check status for over-run after each write */
1306	status = readl(TWL_STATUS_REG_ADDR(tw_dev));
1307	if (!(status & TWL_STATUS_OVERRUN_SUBMIT))
1308	i++;
1309	}
1310
1311	/* Now check status */
1312	status = readl(TWL_STATUS_REG_ADDR(tw_dev));
1313	if (status) {
1314	TW_PRINTK(tw_dev->host, TW_DRIVER, 0x13, "Bad controller status after loading sense buffers");
1315	do_soft_reset = 1;
1316	tries++;
1317	continue;
1318	}
1319
1320	/* Drain the AEN queue */
1321	if (twl_aen_drain_queue(tw_dev, no_check_reset: soft_reset)) {
1322	TW_PRINTK(tw_dev->host, TW_DRIVER, 0x14, "AEN drain failed during reset sequence");
1323	do_soft_reset = 1;
1324	tries++;
1325	continue;
1326	}
1327
1328	/* Load rest of compatibility struct */
1329	strscpy(tw_dev->tw_compat_info.driver_version, TW_DRIVER_VERSION,
1330	sizeof(tw_dev->tw_compat_info.driver_version));
1331	tw_dev->tw_compat_info.driver_srl_high = TW_CURRENT_DRIVER_SRL;
1332	tw_dev->tw_compat_info.driver_branch_high = TW_CURRENT_DRIVER_BRANCH;
1333	tw_dev->tw_compat_info.driver_build_high = TW_CURRENT_DRIVER_BUILD;
1334	tw_dev->tw_compat_info.driver_srl_low = TW_BASE_FW_SRL;
1335	tw_dev->tw_compat_info.driver_branch_low = TW_BASE_FW_BRANCH;
1336	tw_dev->tw_compat_info.driver_build_low = TW_BASE_FW_BUILD;
1337	tw_dev->tw_compat_info.fw_on_ctlr_srl = fw_on_ctlr_srl;
1338	tw_dev->tw_compat_info.fw_on_ctlr_branch = fw_on_ctlr_branch;
1339	tw_dev->tw_compat_info.fw_on_ctlr_build = fw_on_ctlr_build;
1340
1341	/* If we got here, controller is in a good state */
1342	retval = 0;
1343	goto out;
1344	}
1345	out:
1346	return retval;
1347	} /* End twl_reset_sequence() */
1348
1349	/* This function will reset a device extension */
1350	static int twl_reset_device_extension(TW_Device_Extension *tw_dev, int ioctl_reset)
1351	{
1352	int i = 0, retval = 1;
1353	unsigned long flags = 0;
1354
1355	/* Block SCSI requests while we are resetting */
1356	if (ioctl_reset)
1357	scsi_block_requests(tw_dev->host);
1358
1359	set_bit(TW_IN_RESET, addr: &tw_dev->flags);
1360	TWL_MASK_INTERRUPTS(tw_dev);
1361	TWL_CLEAR_DB_INTERRUPT(tw_dev);
1362
1363	spin_lock_irqsave(tw_dev->host->host_lock, flags);
1364
1365	/* Abort all requests that are in progress */
1366	for (i = 0; i < TW_Q_LENGTH; i++) {
1367	if ((tw_dev->state[i] != TW_S_FINISHED) &&
1368	(tw_dev->state[i] != TW_S_INITIAL) &&
1369	(tw_dev->state[i] != TW_S_COMPLETED)) {
1370	struct scsi_cmnd *cmd = tw_dev->srb[i];
1371
1372	if (cmd) {
1373	cmd->result = (DID_RESET << 16);
1374	scsi_dma_unmap(cmd);
1375	scsi_done(cmd);
1376	}
1377	}
1378	}
1379
1380	/* Reset queues and counts */
1381	for (i = 0; i < TW_Q_LENGTH; i++) {
1382	tw_dev->free_queue[i] = i;
1383	tw_dev->state[i] = TW_S_INITIAL;
1384	}
1385	tw_dev->free_head = TW_Q_START;
1386	tw_dev->free_tail = TW_Q_START;
1387	tw_dev->posted_request_count = 0;
1388
1389	spin_unlock_irqrestore(lock: tw_dev->host->host_lock, flags);
1390
1391	if (twl_reset_sequence(tw_dev, soft_reset: 1))
1392	goto out;
1393
1394	TWL_UNMASK_INTERRUPTS(tw_dev);
1395
1396	clear_bit(TW_IN_RESET, addr: &tw_dev->flags);
1397	tw_dev->chrdev_request_id = TW_IOCTL_CHRDEV_FREE;
1398
1399	retval = 0;
1400	out:
1401	if (ioctl_reset)
1402	scsi_unblock_requests(tw_dev->host);
1403	return retval;
1404	} /* End twl_reset_device_extension() */
1405
1406	/* This funciton returns unit geometry in cylinders/heads/sectors */
1407	static int twl_scsi_biosparam(struct scsi_device *sdev, struct block_device *bdev, sector_t capacity, int geom[])
1408	{
1409	int heads, sectors;
1410
1411	if (capacity >= 0x200000) {
1412	heads = 255;
1413	sectors = 63;
1414	} else {
1415	heads = 64;
1416	sectors = 32;
1417	}
1418
1419	geom[0] = heads;
1420	geom[1] = sectors;
1421	geom[2] = sector_div(capacity, heads * sectors); /* cylinders */
1422
1423	return 0;
1424	} /* End twl_scsi_biosparam() */
1425
1426	/* This is the new scsi eh reset function */
1427	static int twl_scsi_eh_reset(struct scsi_cmnd *SCpnt)
1428	{
1429	TW_Device_Extension *tw_dev = NULL;
1430	int retval = FAILED;
1431
1432	tw_dev = (TW_Device_Extension *)SCpnt->device->host->hostdata;
1433
1434	tw_dev->num_resets++;
1435
1436	sdev_printk(KERN_WARNING, SCpnt->device,
1437	"WARNING: (0x%02X:0x%04X): Command (0x%x) timed out, resetting card.\n",
1438	TW_DRIVER, 0x2c, SCpnt->cmnd[0]);
1439
1440	/* Make sure we are not issuing an ioctl or resetting from ioctl */
1441	mutex_lock(&tw_dev->ioctl_lock);
1442
1443	/* Now reset the card and some of the device extension data */
1444	if (twl_reset_device_extension(tw_dev, ioctl_reset: 0)) {
1445	TW_PRINTK(tw_dev->host, TW_DRIVER, 0x15, "Controller reset failed during scsi host reset");
1446	goto out;
1447	}
1448
1449	retval = SUCCESS;
1450	out:
1451	mutex_unlock(lock: &tw_dev->ioctl_lock);
1452	return retval;
1453	} /* End twl_scsi_eh_reset() */
1454
1455	/* This is the main scsi queue function to handle scsi opcodes */
1456	static int twl_scsi_queue_lck(struct scsi_cmnd *SCpnt)
1457	{
1458	void (*done)(struct scsi_cmnd *) = scsi_done;
1459	int request_id, retval;
1460	TW_Device_Extension *tw_dev = (TW_Device_Extension *)SCpnt->device->host->hostdata;
1461
1462	/* If we are resetting due to timed out ioctl, report as busy */
1463	if (test_bit(TW_IN_RESET, &tw_dev->flags)) {
1464	retval = SCSI_MLQUEUE_HOST_BUSY;
1465	goto out;
1466	}
1467
1468	/* Get a free request id */
1469	twl_get_request_id(tw_dev, request_id: &request_id);
1470
1471	/* Save the scsi command for use by the ISR */
1472	tw_dev->srb[request_id] = SCpnt;
1473
1474	retval = twl_scsiop_execute_scsi(tw_dev, request_id, NULL, use_sg: 0, NULL);
1475	if (retval) {
1476	tw_dev->state[request_id] = TW_S_COMPLETED;
1477	twl_free_request_id(tw_dev, request_id);
1478	SCpnt->result = (DID_ERROR << 16);
1479	done(SCpnt);
1480	retval = 0;
1481	}
1482	out:
1483	return retval;
1484	} /* End twl_scsi_queue() */
1485
1486	static DEF_SCSI_QCMD(twl_scsi_queue)
1487
1488	/* This function tells the controller to shut down */
1489	static void __twl_shutdown(TW_Device_Extension *tw_dev)
1490	{
1491	/* Disable interrupts */
1492	TWL_MASK_INTERRUPTS(tw_dev);
1493
1494	/* Free up the IRQ */
1495	free_irq(tw_dev->tw_pci_dev->irq, tw_dev);
1496
1497	printk(KERN_WARNING "3w-sas: Shutting down host %d.\n", tw_dev->host->host_no);
1498
1499	/* Tell the card we are shutting down */
1500	if (twl_initconnection(tw_dev, message_credits: 1, set_features: 0, current_fw_srl: 0, current_fw_arch_id: 0, current_fw_branch: 0, current_fw_build: 0, NULL, NULL, NULL, NULL, NULL)) {
1501	TW_PRINTK(tw_dev->host, TW_DRIVER, 0x16, "Connection shutdown failed");
1502	} else {
1503	printk(KERN_WARNING "3w-sas: Shutdown complete.\n");
1504	}
1505
1506	/* Clear doorbell interrupt just before exit */
1507	TWL_CLEAR_DB_INTERRUPT(tw_dev);
1508	} /* End __twl_shutdown() */
1509
1510	/* Wrapper for __twl_shutdown */
1511	static void twl_shutdown(struct pci_dev *pdev)
1512	{
1513	struct Scsi_Host *host = pci_get_drvdata(pdev);
1514	TW_Device_Extension *tw_dev;
1515
1516	if (!host)
1517	return;
1518
1519	tw_dev = (TW_Device_Extension *)host->hostdata;
1520
1521	if (tw_dev->online)
1522	__twl_shutdown(tw_dev);
1523	} /* End twl_shutdown() */
1524
1525	/* This function configures unit settings when a unit is coming on-line */
1526	static int twl_slave_configure(struct scsi_device *sdev)
1527	{
1528	/* Force 60 second timeout */
1529	blk_queue_rq_timeout(sdev->request_queue, 60 * HZ);
1530
1531	return 0;
1532	} /* End twl_slave_configure() */
1533
1534	static const struct scsi_host_template driver_template = {
1535	.module = THIS_MODULE,
1536	.name = "3w-sas",
1537	.queuecommand = twl_scsi_queue,
1538	.eh_host_reset_handler = twl_scsi_eh_reset,
1539	.bios_param = twl_scsi_biosparam,
1540	.change_queue_depth = scsi_change_queue_depth,
1541	.can_queue = TW_Q_LENGTH-2,
1542	.slave_configure = twl_slave_configure,
1543	.this_id = -1,
1544	.sg_tablesize = TW_LIBERATOR_MAX_SGL_LENGTH,
1545	.max_sectors = TW_MAX_SECTORS,
1546	.cmd_per_lun = TW_MAX_CMDS_PER_LUN,
1547	.shost_groups = twl_host_groups,
1548	.emulated = 1,
1549	.no_write_same = 1,
1550	};
1551
1552	/* This function will probe and initialize a card */
1553	static int twl_probe(struct pci_dev *pdev, const struct pci_device_id *dev_id)
1554	{
1555	struct Scsi_Host *host = NULL;
1556	TW_Device_Extension *tw_dev;
1557	int retval = -ENODEV;
1558	int *ptr_phycount, phycount=0;
1559
1560	retval = pci_enable_device(dev: pdev);
1561	if (retval) {
1562	TW_PRINTK(host, TW_DRIVER, 0x17, "Failed to enable pci device");
1563	goto out_disable_device;
1564	}
1565
1566	pci_set_master(dev: pdev);
1567	pci_try_set_mwi(dev: pdev);
1568
1569	retval = dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(64));
1570	if (retval) {
1571	TW_PRINTK(host, TW_DRIVER, 0x18, "Failed to set dma mask");
1572	retval = -ENODEV;
1573	goto out_disable_device;
1574	}
1575
1576	host = scsi_host_alloc(&driver_template, sizeof(TW_Device_Extension));
1577	if (!host) {
1578	TW_PRINTK(host, TW_DRIVER, 0x19, "Failed to allocate memory for device extension");
1579	retval = -ENOMEM;
1580	goto out_disable_device;
1581	}
1582	tw_dev = shost_priv(shost: host);
1583
1584	/* Save values to device extension */
1585	tw_dev->host = host;
1586	tw_dev->tw_pci_dev = pdev;
1587
1588	if (twl_initialize_device_extension(tw_dev)) {
1589	TW_PRINTK(tw_dev->host, TW_DRIVER, 0x1a, "Failed to initialize device extension");
1590	retval = -ENOMEM;
1591	goto out_free_device_extension;
1592	}
1593
1594	/* Request IO regions */
1595	retval = pci_request_regions(pdev, "3w-sas");
1596	if (retval) {
1597	TW_PRINTK(tw_dev->host, TW_DRIVER, 0x1b, "Failed to get mem region");
1598	goto out_free_device_extension;
1599	}
1600
1601	/* Save base address, use region 1 */
1602	tw_dev->base_addr = pci_iomap(dev: pdev, bar: 1, max: 0);
1603	if (!tw_dev->base_addr) {
1604	TW_PRINTK(tw_dev->host, TW_DRIVER, 0x1c, "Failed to ioremap");
1605	retval = -ENOMEM;
1606	goto out_release_mem_region;
1607	}
1608
1609	/* Disable interrupts on the card */
1610	TWL_MASK_INTERRUPTS(tw_dev);
1611
1612	/* Initialize the card */
1613	if (twl_reset_sequence(tw_dev, soft_reset: 0)) {
1614	TW_PRINTK(tw_dev->host, TW_DRIVER, 0x1d, "Controller reset failed during probe");
1615	retval = -ENOMEM;
1616	goto out_iounmap;
1617	}
1618
1619	/* Set host specific parameters */
1620	host->max_id = TW_MAX_UNITS;
1621	host->max_cmd_len = TW_MAX_CDB_LEN;
1622	host->max_lun = TW_MAX_LUNS;
1623	host->max_channel = 0;
1624
1625	/* Register the card with the kernel SCSI layer */
1626	retval = scsi_add_host(host, dev: &pdev->dev);
1627	if (retval) {
1628	TW_PRINTK(tw_dev->host, TW_DRIVER, 0x1e, "scsi add host failed");
1629	goto out_iounmap;
1630	}
1631
1632	pci_set_drvdata(pdev, data: host);
1633
1634	printk(KERN_WARNING "3w-sas: scsi%d: Found an LSI 3ware %s Controller at 0x%llx, IRQ: %d.\n",
1635	host->host_no,
1636	(char *)twl_get_param(tw_dev, 1, TW_VERSION_TABLE,
1637	TW_PARAM_MODEL, TW_PARAM_MODEL_LENGTH),
1638	(u64)pci_resource_start(pdev, 1), pdev->irq);
1639
1640	ptr_phycount = twl_get_param(tw_dev, request_id: 2, TW_PARAM_PHY_SUMMARY_TABLE,
1641	TW_PARAM_PHYCOUNT, TW_PARAM_PHYCOUNT_LENGTH);
1642	if (ptr_phycount)
1643	phycount = le32_to_cpu(*(int *)ptr_phycount);
1644
1645	printk(KERN_WARNING "3w-sas: scsi%d: Firmware %s, BIOS %s, Phys: %d.\n",
1646	host->host_no,
1647	(char *)twl_get_param(tw_dev, 1, TW_VERSION_TABLE,
1648	TW_PARAM_FWVER, TW_PARAM_FWVER_LENGTH),
1649	(char *)twl_get_param(tw_dev, 2, TW_VERSION_TABLE,
1650	TW_PARAM_BIOSVER, TW_PARAM_BIOSVER_LENGTH),
1651	phycount);
1652
1653	/* Try to enable MSI */
1654	if (use_msi && !pci_enable_msi(dev: pdev))
1655	set_bit(TW_USING_MSI, addr: &tw_dev->flags);
1656
1657	/* Now setup the interrupt handler */
1658	retval = request_irq(irq: pdev->irq, handler: twl_interrupt, IRQF_SHARED, name: "3w-sas", dev: tw_dev);
1659	if (retval) {
1660	TW_PRINTK(tw_dev->host, TW_DRIVER, 0x1f, "Error requesting IRQ");
1661	goto out_remove_host;
1662	}
1663
1664	twl_device_extension_list[twl_device_extension_count] = tw_dev;
1665	twl_device_extension_count++;
1666
1667	/* Re-enable interrupts on the card */
1668	TWL_UNMASK_INTERRUPTS(tw_dev);
1669
1670	/* Finally, scan the host */
1671	scsi_scan_host(host);
1672
1673	/* Add sysfs binary files */
1674	if (sysfs_create_bin_file(kobj: &host->shost_dev.kobj, attr: &twl_sysfs_aen_read_attr))
1675	TW_PRINTK(tw_dev->host, TW_DRIVER, 0x20, "Failed to create sysfs binary file: 3ware_aen_read");
1676	if (sysfs_create_bin_file(kobj: &host->shost_dev.kobj, attr: &twl_sysfs_compat_info_attr))
1677	TW_PRINTK(tw_dev->host, TW_DRIVER, 0x21, "Failed to create sysfs binary file: 3ware_compat_info");
1678
1679	if (twl_major == -1) {
1680	if ((twl_major = register_chrdev (major: 0, name: "twl", fops: &twl_fops)) < 0)
1681	TW_PRINTK(host, TW_DRIVER, 0x22, "Failed to register character device");
1682	}
1683	tw_dev->online = 1;
1684	return 0;
1685
1686	out_remove_host:
1687	if (test_bit(TW_USING_MSI, &tw_dev->flags))
1688	pci_disable_msi(dev: pdev);
1689	scsi_remove_host(host);
1690	out_iounmap:
1691	iounmap(addr: tw_dev->base_addr);
1692	out_release_mem_region:
1693	pci_release_regions(pdev);
1694	out_free_device_extension:
1695	twl_free_device_extension(tw_dev);
1696	scsi_host_put(t: host);
1697	out_disable_device:
1698	pci_disable_device(dev: pdev);
1699
1700	return retval;
1701	} /* End twl_probe() */
1702
1703	/* This function is called to remove a device */
1704	static void twl_remove(struct pci_dev *pdev)
1705	{
1706	struct Scsi_Host *host = pci_get_drvdata(pdev);
1707	TW_Device_Extension *tw_dev;
1708
1709	if (!host)
1710	return;
1711
1712	tw_dev = (TW_Device_Extension *)host->hostdata;
1713
1714	if (!tw_dev->online)
1715	return;
1716
1717	/* Remove sysfs binary files */
1718	sysfs_remove_bin_file(kobj: &host->shost_dev.kobj, attr: &twl_sysfs_aen_read_attr);
1719	sysfs_remove_bin_file(kobj: &host->shost_dev.kobj, attr: &twl_sysfs_compat_info_attr);
1720
1721	scsi_remove_host(tw_dev->host);
1722
1723	/* Unregister character device */
1724	if (twl_major >= 0) {
1725	unregister_chrdev(major: twl_major, name: "twl");
1726	twl_major = -1;
1727	}
1728
1729	/* Shutdown the card */
1730	__twl_shutdown(tw_dev);
1731
1732	/* Disable MSI if enabled */
1733	if (test_bit(TW_USING_MSI, &tw_dev->flags))
1734	pci_disable_msi(dev: pdev);
1735
1736	/* Free IO remapping */
1737	iounmap(addr: tw_dev->base_addr);
1738
1739	/* Free up the mem region */
1740	pci_release_regions(pdev);
1741
1742	/* Free up device extension resources */
1743	twl_free_device_extension(tw_dev);
1744
1745	scsi_host_put(t: tw_dev->host);
1746	pci_disable_device(dev: pdev);
1747	twl_device_extension_count--;
1748	} /* End twl_remove() */
1749
1750	/* This function is called on PCI suspend */
1751	static int __maybe_unused twl_suspend(struct device *dev)
1752	{
1753	struct Scsi_Host *host = dev_get_drvdata(dev);
1754	TW_Device_Extension *tw_dev = (TW_Device_Extension *)host->hostdata;
1755
1756	printk(KERN_WARNING "3w-sas: Suspending host %d.\n", tw_dev->host->host_no);
1757	/* Disable interrupts */
1758	TWL_MASK_INTERRUPTS(tw_dev);
1759
1760	free_irq(tw_dev->tw_pci_dev->irq, tw_dev);
1761
1762	/* Tell the card we are shutting down */
1763	if (twl_initconnection(tw_dev, message_credits: 1, set_features: 0, current_fw_srl: 0, current_fw_arch_id: 0, current_fw_branch: 0, current_fw_build: 0, NULL, NULL, NULL, NULL, NULL)) {
1764	TW_PRINTK(tw_dev->host, TW_DRIVER, 0x23, "Connection shutdown failed during suspend");
1765	} else {
1766	printk(KERN_WARNING "3w-sas: Suspend complete.\n");
1767	}
1768
1769	/* Clear doorbell interrupt */
1770	TWL_CLEAR_DB_INTERRUPT(tw_dev);
1771
1772	return 0;
1773	} /* End twl_suspend() */
1774
1775	/* This function is called on PCI resume */
1776	static int __maybe_unused twl_resume(struct device *dev)
1777	{
1778	int retval = 0;
1779	struct pci_dev *pdev = to_pci_dev(dev);
1780	struct Scsi_Host *host = pci_get_drvdata(pdev);
1781	TW_Device_Extension *tw_dev = (TW_Device_Extension *)host->hostdata;
1782
1783	printk(KERN_WARNING "3w-sas: Resuming host %d.\n", tw_dev->host->host_no);
1784	pci_try_set_mwi(dev: pdev);
1785
1786	retval = dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(64));
1787	if (retval) {
1788	TW_PRINTK(host, TW_DRIVER, 0x25, "Failed to set dma mask during resume");
1789	retval = -ENODEV;
1790	goto out_disable_device;
1791	}
1792
1793	/* Initialize the card */
1794	if (twl_reset_sequence(tw_dev, soft_reset: 0)) {
1795	retval = -ENODEV;
1796	goto out_disable_device;
1797	}
1798
1799	/* Now setup the interrupt handler */
1800	retval = request_irq(irq: pdev->irq, handler: twl_interrupt, IRQF_SHARED, name: "3w-sas", dev: tw_dev);
1801	if (retval) {
1802	TW_PRINTK(tw_dev->host, TW_DRIVER, 0x26, "Error requesting IRQ during resume");
1803	retval = -ENODEV;
1804	goto out_disable_device;
1805	}
1806
1807	/* Now enable MSI if enabled */
1808	if (test_bit(TW_USING_MSI, &tw_dev->flags))
1809	pci_enable_msi(dev: pdev);
1810
1811	/* Re-enable interrupts on the card */
1812	TWL_UNMASK_INTERRUPTS(tw_dev);
1813
1814	printk(KERN_WARNING "3w-sas: Resume complete.\n");
1815	return 0;
1816
1817	out_disable_device:
1818	scsi_remove_host(host);
1819
1820	return retval;
1821	} /* End twl_resume() */
1822
1823	/* PCI Devices supported by this driver */
1824	static struct pci_device_id twl_pci_tbl[] = {
1825	{ PCI_VDEVICE(3WARE, PCI_DEVICE_ID_3WARE_9750) },
1826	{ }
1827	};
1828	MODULE_DEVICE_TABLE(pci, twl_pci_tbl);
1829
1830	static SIMPLE_DEV_PM_OPS(twl_pm_ops, twl_suspend, twl_resume);
1831
1832	/* pci_driver initializer */
1833	static struct pci_driver twl_driver = {
1834	.name = "3w-sas",
1835	.id_table = twl_pci_tbl,
1836	.probe = twl_probe,
1837	.remove = twl_remove,
1838	.driver.pm = &twl_pm_ops,
1839	.shutdown = twl_shutdown
1840	};
1841
1842	/* This function is called on driver initialization */
1843	static int __init twl_init(void)
1844	{
1845	printk(KERN_INFO "LSI 3ware SAS/SATA-RAID Controller device driver for Linux v%s.\n", TW_DRIVER_VERSION);
1846
1847	return pci_register_driver(&twl_driver);
1848	} /* End twl_init() */
1849
1850	/* This function is called on driver exit */
1851	static void __exit twl_exit(void)
1852	{
1853	pci_unregister_driver(dev: &twl_driver);
1854	} /* End twl_exit() */
1855
1856	module_init(twl_init);
1857	module_exit(twl_exit);
1858
1859