1	// SPDX-License-Identifier: GPL-2.0+
2	/*
3	* Driver for the PLX NET2280 USB device controller.
4	* Specs and errata are available from <http://www.plxtech.com>.
5	*
6	* PLX Technology Inc. (formerly NetChip Technology) supported the
7	* development of this driver.
8	*
9	*
10	* CODE STATUS HIGHLIGHTS
11	*
12	* This driver should work well with most "gadget" drivers, including
13	* the Mass Storage, Serial, and Ethernet/RNDIS gadget drivers
14	* as well as Gadget Zero and Gadgetfs.
15	*
16	* DMA is enabled by default.
17	*
18	* MSI is enabled by default. The legacy IRQ is used if MSI couldn't
19	* be enabled.
20	*
21	* Note that almost all the errata workarounds here are only needed for
22	* rev1 chips. Rev1a silicon (0110) fixes almost all of them.
23	*/
24
25	/*
26	* Copyright (C) 2003 David Brownell
27	* Copyright (C) 2003-2005 PLX Technology, Inc.
28	* Copyright (C) 2014 Ricardo Ribalda - Qtechnology/AS
29	*
30	* Modified Seth Levy 2005 PLX Technology, Inc. to provide compatibility
31	* with 2282 chip
32	*
33	* Modified Ricardo Ribalda Qtechnology AS to provide compatibility
34	* with usb 338x chip. Based on PLX driver
35	*/
36
37	#include <linux/module.h>
38	#include <linux/pci.h>
39	#include <linux/dma-mapping.h>
40	#include <linux/kernel.h>
41	#include <linux/delay.h>
42	#include <linux/ioport.h>
43	#include <linux/slab.h>
44	#include <linux/errno.h>
45	#include <linux/init.h>
46	#include <linux/timer.h>
47	#include <linux/list.h>
48	#include <linux/interrupt.h>
49	#include <linux/moduleparam.h>
50	#include <linux/device.h>
51	#include <linux/usb/ch9.h>
52	#include <linux/usb/gadget.h>
53	#include <linux/prefetch.h>
54	#include <linux/io.h>
55	#include <linux/iopoll.h>
56
57	#include <asm/byteorder.h>
58	#include <asm/irq.h>
59	#include <asm/unaligned.h>
60
61	#define DRIVER_DESC "PLX NET228x/USB338x USB Peripheral Controller"
62	#define DRIVER_VERSION "2005 Sept 27/v3.0"
63
64	#define EP_DONTUSE 13 /* nonzero */
65
66	#define USE_RDK_LEDS /* GPIO pins control three LEDs */
67
68
69	static const char driver_name[] = "net2280";
70	static const char driver_desc[] = DRIVER_DESC;
71
72	static const u32 ep_bit[9] = { 0, 17, 2, 19, 4, 1, 18, 3, 20 };
73	static const char ep0name[] = "ep0";
74
75	#define EP_INFO(_name, _caps) \
76	{ \
77	.name = _name, \
78	.caps = _caps, \
79	}
80
81	static const struct {
82	const char *name;
83	const struct usb_ep_caps caps;
84	} ep_info_dft[] = { /* Default endpoint configuration */
85	EP_INFO(ep0name,
86	USB_EP_CAPS(USB_EP_CAPS_TYPE_CONTROL, USB_EP_CAPS_DIR_ALL)),
87	EP_INFO("ep-a",
88	USB_EP_CAPS(USB_EP_CAPS_TYPE_ALL, USB_EP_CAPS_DIR_ALL)),
89	EP_INFO("ep-b",
90	USB_EP_CAPS(USB_EP_CAPS_TYPE_ALL, USB_EP_CAPS_DIR_ALL)),
91	EP_INFO("ep-c",
92	USB_EP_CAPS(USB_EP_CAPS_TYPE_ALL, USB_EP_CAPS_DIR_ALL)),
93	EP_INFO("ep-d",
94	USB_EP_CAPS(USB_EP_CAPS_TYPE_ALL, USB_EP_CAPS_DIR_ALL)),
95	EP_INFO("ep-e",
96	USB_EP_CAPS(USB_EP_CAPS_TYPE_ALL, USB_EP_CAPS_DIR_ALL)),
97	EP_INFO("ep-f",
98	USB_EP_CAPS(USB_EP_CAPS_TYPE_ALL, USB_EP_CAPS_DIR_ALL)),
99	EP_INFO("ep-g",
100	USB_EP_CAPS(USB_EP_CAPS_TYPE_ALL, USB_EP_CAPS_DIR_ALL)),
101	EP_INFO("ep-h",
102	USB_EP_CAPS(USB_EP_CAPS_TYPE_ALL, USB_EP_CAPS_DIR_ALL)),
103	}, ep_info_adv[] = { /* Endpoints for usb3380 advance mode */
104	EP_INFO(ep0name,
105	USB_EP_CAPS(USB_EP_CAPS_TYPE_CONTROL, USB_EP_CAPS_DIR_ALL)),
106	EP_INFO("ep1in",
107	USB_EP_CAPS(USB_EP_CAPS_TYPE_ALL, USB_EP_CAPS_DIR_IN)),
108	EP_INFO("ep2out",
109	USB_EP_CAPS(USB_EP_CAPS_TYPE_ALL, USB_EP_CAPS_DIR_OUT)),
110	EP_INFO("ep3in",
111	USB_EP_CAPS(USB_EP_CAPS_TYPE_ALL, USB_EP_CAPS_DIR_IN)),
112	EP_INFO("ep4out",
113	USB_EP_CAPS(USB_EP_CAPS_TYPE_ALL, USB_EP_CAPS_DIR_OUT)),
114	EP_INFO("ep1out",
115	USB_EP_CAPS(USB_EP_CAPS_TYPE_ALL, USB_EP_CAPS_DIR_OUT)),
116	EP_INFO("ep2in",
117	USB_EP_CAPS(USB_EP_CAPS_TYPE_ALL, USB_EP_CAPS_DIR_IN)),
118	EP_INFO("ep3out",
119	USB_EP_CAPS(USB_EP_CAPS_TYPE_ALL, USB_EP_CAPS_DIR_OUT)),
120	EP_INFO("ep4in",
121	USB_EP_CAPS(USB_EP_CAPS_TYPE_ALL, USB_EP_CAPS_DIR_IN)),
122	};
123
124	#undef EP_INFO
125
126	/* mode 0 == ep-{a,b,c,d} 1K fifo each
127	* mode 1 == ep-{a,b} 2K fifo each, ep-{c,d} unavailable
128	* mode 2 == ep-a 2K fifo, ep-{b,c} 1K each, ep-d unavailable
129	*/
130	static ushort fifo_mode;
131
132	/* "modprobe net2280 fifo_mode=1" etc */
133	module_param(fifo_mode, ushort, 0644);
134
135	/* enable_suspend -- When enabled, the driver will respond to
136	* USB suspend requests by powering down the NET2280. Otherwise,
137	* USB suspend requests will be ignored. This is acceptable for
138	* self-powered devices
139	*/
140	static bool enable_suspend;
141
142	/* "modprobe net2280 enable_suspend=1" etc */
143	module_param(enable_suspend, bool, 0444);
144
145	#define DIR_STRING(bAddress) (((bAddress) & USB_DIR_IN) ? "in" : "out")
146
147	static char *type_string(u8 bmAttributes)
148	{
149	switch ((bmAttributes) & USB_ENDPOINT_XFERTYPE_MASK) {
150	case USB_ENDPOINT_XFER_BULK: return "bulk";
151	case USB_ENDPOINT_XFER_ISOC: return "iso";
152	case USB_ENDPOINT_XFER_INT: return "intr";
153	}
154	return "control";
155	}
156
157	#include "net2280.h"
158
159	#define valid_bit cpu_to_le32(BIT(VALID_BIT))
160	#define dma_done_ie cpu_to_le32(BIT(DMA_DONE_INTERRUPT_ENABLE))
161
162	static void ep_clear_seqnum(struct net2280_ep *ep);
163	static void stop_activity(struct net2280 *dev,
164	struct usb_gadget_driver *driver);
165	static void ep0_start(struct net2280 *dev);
166
167	/*-------------------------------------------------------------------------*/
168	static inline void enable_pciirqenb(struct net2280_ep *ep)
169	{
170	u32 tmp = readl(addr: &ep->dev->regs->pciirqenb0);
171
172	if (ep->dev->quirks & PLX_LEGACY)
173	tmp |= BIT(ep->num);
174	else
175	tmp |= BIT(ep_bit[ep->num]);
176	writel(val: tmp, addr: &ep->dev->regs->pciirqenb0);
177
178	return;
179	}
180
181	static int
182	net2280_enable(struct usb_ep *_ep, const struct usb_endpoint_descriptor *desc)
183	{
184	struct net2280 *dev;
185	struct net2280_ep *ep;
186	u32 max;
187	u32 tmp = 0;
188	u32 type;
189	unsigned long flags;
190	static const u32 ep_key[9] = { 1, 0, 1, 0, 1, 1, 0, 1, 0 };
191	int ret = 0;
192
193	ep = container_of(_ep, struct net2280_ep, ep);
194	if (!_ep || !desc || ep->desc || _ep->name == ep0name ||
195	desc->bDescriptorType != USB_DT_ENDPOINT) {
196	pr_err("%s: failed at line=%d\n", __func__, __LINE__);
197	return -EINVAL;
198	}
199	dev = ep->dev;
200	if (!dev->driver || dev->gadget.speed == USB_SPEED_UNKNOWN) {
201	ret = -ESHUTDOWN;
202	goto print_err;
203	}
204
205	/* erratum 0119 workaround ties up an endpoint number */
206	if ((desc->bEndpointAddress & 0x0f) == EP_DONTUSE) {
207	ret = -EDOM;
208	goto print_err;
209	}
210
211	if (dev->quirks & PLX_PCIE) {
212	if ((desc->bEndpointAddress & 0x0f) >= 0x0c) {
213	ret = -EDOM;
214	goto print_err;
215	}
216	ep->is_in = !!usb_endpoint_dir_in(epd: desc);
217	if (dev->enhanced_mode && ep->is_in && ep_key[ep->num]) {
218	ret = -EINVAL;
219	goto print_err;
220	}
221	}
222
223	/* sanity check ep-e/ep-f since their fifos are small */
224	max = usb_endpoint_maxp(epd: desc);
225	if (ep->num > 4 && max > 64 && (dev->quirks & PLX_LEGACY)) {
226	ret = -ERANGE;
227	goto print_err;
228	}
229
230	spin_lock_irqsave(&dev->lock, flags);
231	_ep->maxpacket = max;
232	ep->desc = desc;
233
234	/* ep_reset() has already been called */
235	ep->stopped = 0;
236	ep->wedged = 0;
237	ep->out_overflow = 0;
238
239	/* set speed-dependent max packet; may kick in high bandwidth */
240	set_max_speed(ep, max);
241
242	/* set type, direction, address; reset fifo counters */
243	writel(BIT(FIFO_FLUSH), addr: &ep->regs->ep_stat);
244
245	if ((dev->quirks & PLX_PCIE) && dev->enhanced_mode) {
246	tmp = readl(addr: &ep->cfg->ep_cfg);
247	/* If USB ep number doesn't match hardware ep number */
248	if ((tmp & 0xf) != usb_endpoint_num(epd: desc)) {
249	ret = -EINVAL;
250	spin_unlock_irqrestore(lock: &dev->lock, flags);
251	goto print_err;
252	}
253	if (ep->is_in)
254	tmp &= ~USB3380_EP_CFG_MASK_IN;
255	else
256	tmp &= ~USB3380_EP_CFG_MASK_OUT;
257	}
258	type = (desc->bmAttributes & USB_ENDPOINT_XFERTYPE_MASK);
259	if (type == USB_ENDPOINT_XFER_INT) {
260	/* erratum 0105 workaround prevents hs NYET */
261	if (dev->chiprev == 0100 &&
262	dev->gadget.speed == USB_SPEED_HIGH &&
263	!(desc->bEndpointAddress & USB_DIR_IN))
264	writel(BIT(CLEAR_NAK_OUT_PACKETS_MODE),
265	addr: &ep->regs->ep_rsp);
266	} else if (type == USB_ENDPOINT_XFER_BULK) {
267	/* catch some particularly blatant driver bugs */
268	if ((dev->gadget.speed == USB_SPEED_SUPER && max != 1024) ||
269	(dev->gadget.speed == USB_SPEED_HIGH && max != 512) ||
270	(dev->gadget.speed == USB_SPEED_FULL && max > 64)) {
271	spin_unlock_irqrestore(lock: &dev->lock, flags);
272	ret = -ERANGE;
273	goto print_err;
274	}
275	}
276	ep->is_iso = (type == USB_ENDPOINT_XFER_ISOC);
277	/* Enable this endpoint */
278	if (dev->quirks & PLX_LEGACY) {
279	tmp |= type << ENDPOINT_TYPE;
280	tmp |= desc->bEndpointAddress;
281	/* default full fifo lines */
282	tmp |= (4 << ENDPOINT_BYTE_COUNT);
283	tmp |= BIT(ENDPOINT_ENABLE);
284	ep->is_in = (tmp & USB_DIR_IN) != 0;
285	} else {
286	/* In Legacy mode, only OUT endpoints are used */
287	if (dev->enhanced_mode && ep->is_in) {
288	tmp |= type << IN_ENDPOINT_TYPE;
289	tmp |= BIT(IN_ENDPOINT_ENABLE);
290	} else {
291	tmp |= type << OUT_ENDPOINT_TYPE;
292	tmp |= BIT(OUT_ENDPOINT_ENABLE);
293	tmp |= (ep->is_in << ENDPOINT_DIRECTION);
294	}
295
296	tmp |= (4 << ENDPOINT_BYTE_COUNT);
297	if (!dev->enhanced_mode)
298	tmp |= usb_endpoint_num(epd: desc);
299	tmp |= (ep->ep.maxburst << MAX_BURST_SIZE);
300	}
301
302	/* Make sure all the registers are written before ep_rsp*/
303	wmb();
304
305	/* for OUT transfers, block the rx fifo until a read is posted */
306	if (!ep->is_in)
307	writel(BIT(SET_NAK_OUT_PACKETS), addr: &ep->regs->ep_rsp);
308	else if (!(dev->quirks & PLX_2280)) {
309	/* Added for 2282, Don't use nak packets on an in endpoint,
310	* this was ignored on 2280
311	*/
312	writel(BIT(CLEAR_NAK_OUT_PACKETS) |
313	BIT(CLEAR_NAK_OUT_PACKETS_MODE), addr: &ep->regs->ep_rsp);
314	}
315
316	if (dev->quirks & PLX_PCIE)
317	ep_clear_seqnum(ep);
318	writel(val: tmp, addr: &ep->cfg->ep_cfg);
319
320	/* enable irqs */
321	if (!ep->dma) { /* pio, per-packet */
322	enable_pciirqenb(ep);
323
324	tmp = BIT(DATA_PACKET_RECEIVED_INTERRUPT_ENABLE) |
325	BIT(DATA_PACKET_TRANSMITTED_INTERRUPT_ENABLE);
326	if (dev->quirks & PLX_2280)
327	tmp |= readl(addr: &ep->regs->ep_irqenb);
328	writel(val: tmp, addr: &ep->regs->ep_irqenb);
329	} else { /* dma, per-request */
330	tmp = BIT((8 + ep->num)); /* completion */
331	tmp |= readl(addr: &dev->regs->pciirqenb1);
332	writel(val: tmp, addr: &dev->regs->pciirqenb1);
333
334	/* for short OUT transfers, dma completions can't
335	* advance the queue; do it pio-style, by hand.
336	* NOTE erratum 0112 workaround #2
337	*/
338	if ((desc->bEndpointAddress & USB_DIR_IN) == 0) {
339	tmp = BIT(SHORT_PACKET_TRANSFERRED_INTERRUPT_ENABLE);
340	writel(val: tmp, addr: &ep->regs->ep_irqenb);
341
342	enable_pciirqenb(ep);
343	}
344	}
345
346	tmp = desc->bEndpointAddress;
347	ep_dbg(dev, "enabled %s (ep%d%s-%s) %s max %04x\n",
348	_ep->name, tmp & 0x0f, DIR_STRING(tmp),
349	type_string(desc->bmAttributes),
350	ep->dma ? "dma" : "pio", max);
351
352	/* pci writes may still be posted */
353	spin_unlock_irqrestore(lock: &dev->lock, flags);
354	return ret;
355
356	print_err:
357	dev_err(&ep->dev->pdev->dev, "%s: error=%d\n", __func__, ret);
358	return ret;
359	}
360
361	static int handshake(u32 __iomem *ptr, u32 mask, u32 done, int usec)
362	{
363	u32 result;
364	int ret;
365
366	ret = readl_poll_timeout_atomic(ptr, result,
367	((result & mask) == done ||
368	result == U32_MAX),
369	1, usec);
370	if (result == U32_MAX) /* device unplugged */
371	return -ENODEV;
372
373	return ret;
374	}
375
376	static const struct usb_ep_ops net2280_ep_ops;
377
378	static void ep_reset_228x(struct net2280_regs __iomem *regs,
379	struct net2280_ep *ep)
380	{
381	u32 tmp;
382
383	ep->desc = NULL;
384	INIT_LIST_HEAD(list: &ep->queue);
385
386	usb_ep_set_maxpacket_limit(ep: &ep->ep, maxpacket_limit: ~0);
387	ep->ep.ops = &net2280_ep_ops;
388
389	/* disable the dma, irqs, endpoint... */
390	if (ep->dma) {
391	writel(val: 0, addr: &ep->dma->dmactl);
392	writel(BIT(DMA_SCATTER_GATHER_DONE_INTERRUPT) |
393	BIT(DMA_TRANSACTION_DONE_INTERRUPT) |
394	BIT(DMA_ABORT),
395	addr: &ep->dma->dmastat);
396
397	tmp = readl(addr: &regs->pciirqenb0);
398	tmp &= ~BIT(ep->num);
399	writel(val: tmp, addr: &regs->pciirqenb0);
400	} else {
401	tmp = readl(addr: &regs->pciirqenb1);
402	tmp &= ~BIT((8 + ep->num)); /* completion */
403	writel(val: tmp, addr: &regs->pciirqenb1);
404	}
405	writel(val: 0, addr: &ep->regs->ep_irqenb);
406
407	/* init to our chosen defaults, notably so that we NAK OUT
408	* packets until the driver queues a read (+note erratum 0112)
409	*/
410	if (!ep->is_in || (ep->dev->quirks & PLX_2280)) {
411	tmp = BIT(SET_NAK_OUT_PACKETS_MODE) |
412	BIT(SET_NAK_OUT_PACKETS) |
413	BIT(CLEAR_EP_HIDE_STATUS_PHASE) |
414	BIT(CLEAR_INTERRUPT_MODE);
415	} else {
416	/* added for 2282 */
417	tmp = BIT(CLEAR_NAK_OUT_PACKETS_MODE) |
418	BIT(CLEAR_NAK_OUT_PACKETS) |
419	BIT(CLEAR_EP_HIDE_STATUS_PHASE) |
420	BIT(CLEAR_INTERRUPT_MODE);
421	}
422
423	if (ep->num != 0) {
424	tmp |= BIT(CLEAR_ENDPOINT_TOGGLE) |
425	BIT(CLEAR_ENDPOINT_HALT);
426	}
427	writel(val: tmp, addr: &ep->regs->ep_rsp);
428
429	/* scrub most status bits, and flush any fifo state */
430	if (ep->dev->quirks & PLX_2280)
431	tmp = BIT(FIFO_OVERFLOW) |
432	BIT(FIFO_UNDERFLOW);
433	else
434	tmp = 0;
435
436	writel(val: tmp | BIT(TIMEOUT) |
437	BIT(USB_STALL_SENT) |
438	BIT(USB_IN_NAK_SENT) |
439	BIT(USB_IN_ACK_RCVD) |
440	BIT(USB_OUT_PING_NAK_SENT) |
441	BIT(USB_OUT_ACK_SENT) |
442	BIT(FIFO_FLUSH) |
443	BIT(SHORT_PACKET_OUT_DONE_INTERRUPT) |
444	BIT(SHORT_PACKET_TRANSFERRED_INTERRUPT) |
445	BIT(DATA_PACKET_RECEIVED_INTERRUPT) |
446	BIT(DATA_PACKET_TRANSMITTED_INTERRUPT) |
447	BIT(DATA_OUT_PING_TOKEN_INTERRUPT) |
448	BIT(DATA_IN_TOKEN_INTERRUPT),
449	addr: &ep->regs->ep_stat);
450
451	/* fifo size is handled separately */
452	}
453
454	static void ep_reset_338x(struct net2280_regs __iomem *regs,
455	struct net2280_ep *ep)
456	{
457	u32 tmp, dmastat;
458
459	ep->desc = NULL;
460	INIT_LIST_HEAD(list: &ep->queue);
461
462	usb_ep_set_maxpacket_limit(ep: &ep->ep, maxpacket_limit: ~0);
463	ep->ep.ops = &net2280_ep_ops;
464
465	/* disable the dma, irqs, endpoint... */
466	if (ep->dma) {
467	writel(val: 0, addr: &ep->dma->dmactl);
468	writel(BIT(DMA_ABORT_DONE_INTERRUPT) |
469	BIT(DMA_PAUSE_DONE_INTERRUPT) |
470	BIT(DMA_SCATTER_GATHER_DONE_INTERRUPT) |
471	BIT(DMA_TRANSACTION_DONE_INTERRUPT),
472	/* | BIT(DMA_ABORT), */
473	addr: &ep->dma->dmastat);
474
475	dmastat = readl(addr: &ep->dma->dmastat);
476	if (dmastat == 0x5002) {
477	ep_warn(ep->dev, "The dmastat return = %x!!\n",
478	dmastat);
479	writel(val: 0x5a, addr: &ep->dma->dmastat);
480	}
481
482	tmp = readl(addr: &regs->pciirqenb0);
483	tmp &= ~BIT(ep_bit[ep->num]);
484	writel(val: tmp, addr: &regs->pciirqenb0);
485	} else {
486	if (ep->num < 5) {
487	tmp = readl(addr: &regs->pciirqenb1);
488	tmp &= ~BIT((8 + ep->num)); /* completion */
489	writel(val: tmp, addr: &regs->pciirqenb1);
490	}
491	}
492	writel(val: 0, addr: &ep->regs->ep_irqenb);
493
494	writel(BIT(SHORT_PACKET_OUT_DONE_INTERRUPT) |
495	BIT(SHORT_PACKET_TRANSFERRED_INTERRUPT) |
496	BIT(FIFO_OVERFLOW) |
497	BIT(DATA_PACKET_RECEIVED_INTERRUPT) |
498	BIT(DATA_PACKET_TRANSMITTED_INTERRUPT) |
499	BIT(DATA_OUT_PING_TOKEN_INTERRUPT) |
500	BIT(DATA_IN_TOKEN_INTERRUPT), addr: &ep->regs->ep_stat);
501
502	tmp = readl(addr: &ep->cfg->ep_cfg);
503	if (ep->is_in)
504	tmp &= ~USB3380_EP_CFG_MASK_IN;
505	else
506	tmp &= ~USB3380_EP_CFG_MASK_OUT;
507	writel(val: tmp, addr: &ep->cfg->ep_cfg);
508	}
509
510	static void nuke(struct net2280_ep *);
511
512	static int net2280_disable(struct usb_ep *_ep)
513	{
514	struct net2280_ep *ep;
515	unsigned long flags;
516
517	ep = container_of(_ep, struct net2280_ep, ep);
518	if (!_ep || _ep->name == ep0name) {
519	pr_err("%s: Invalid ep=%p\n", __func__, _ep);
520	return -EINVAL;
521	}
522	spin_lock_irqsave(&ep->dev->lock, flags);
523	nuke(ep);
524
525	if (ep->dev->quirks & PLX_PCIE)
526	ep_reset_338x(regs: ep->dev->regs, ep);
527	else
528	ep_reset_228x(regs: ep->dev->regs, ep);
529
530	ep_vdbg(ep->dev, "disabled %s %s\n",
531	ep->dma ? "dma" : "pio", _ep->name);
532
533	/* synch memory views with the device */
534	(void)readl(addr: &ep->cfg->ep_cfg);
535
536	if (!ep->dma && ep->num >= 1 && ep->num <= 4)
537	ep->dma = &ep->dev->dma[ep->num - 1];
538
539	spin_unlock_irqrestore(lock: &ep->dev->lock, flags);
540	return 0;
541	}
542
543	/*-------------------------------------------------------------------------*/
544
545	static struct usb_request
546	*net2280_alloc_request(struct usb_ep *_ep, gfp_t gfp_flags)
547	{
548	struct net2280_ep *ep;
549	struct net2280_request *req;
550
551	if (!_ep) {
552	pr_err("%s: Invalid ep\n", __func__);
553	return NULL;
554	}
555	ep = container_of(_ep, struct net2280_ep, ep);
556
557	req = kzalloc(size: sizeof(*req), flags: gfp_flags);
558	if (!req)
559	return NULL;
560
561	INIT_LIST_HEAD(list: &req->queue);
562
563	/* this dma descriptor may be swapped with the previous dummy */
564	if (ep->dma) {
565	struct net2280_dma *td;
566
567	td = dma_pool_alloc(pool: ep->dev->requests, mem_flags: gfp_flags,
568	handle: &req->td_dma);
569	if (!td) {
570	kfree(objp: req);
571	return NULL;
572	}
573	td->dmacount = 0; /* not VALID */
574	td->dmadesc = td->dmaaddr;
575	req->td = td;
576	}
577	return &req->req;
578	}
579
580	static void net2280_free_request(struct usb_ep *_ep, struct usb_request *_req)
581	{
582	struct net2280_ep *ep;
583	struct net2280_request *req;
584
585	ep = container_of(_ep, struct net2280_ep, ep);
586	if (!_ep || !_req) {
587	dev_err(&ep->dev->pdev->dev, "%s: Invalid ep=%p or req=%p\n",
588	__func__, _ep, _req);
589	return;
590	}
591
592	req = container_of(_req, struct net2280_request, req);
593	WARN_ON(!list_empty(&req->queue));
594	if (req->td)
595	dma_pool_free(pool: ep->dev->requests, vaddr: req->td, addr: req->td_dma);
596	kfree(objp: req);
597	}
598
599	/*-------------------------------------------------------------------------*/
600
601	/* load a packet into the fifo we use for usb IN transfers.
602	* works for all endpoints.
603	*
604	* NOTE: pio with ep-a..ep-d could stuff multiple packets into the fifo
605	* at a time, but this code is simpler because it knows it only writes
606	* one packet. ep-a..ep-d should use dma instead.
607	*/
608	static void write_fifo(struct net2280_ep *ep, struct usb_request *req)
609	{
610	struct net2280_ep_regs __iomem *regs = ep->regs;
611	u8 *buf;
612	u32 tmp;
613	unsigned count, total;
614
615	/* INVARIANT: fifo is currently empty. (testable) */
616
617	if (req) {
618	buf = req->buf + req->actual;
619	prefetch(buf);
620	total = req->length - req->actual;
621	} else {
622	total = 0;
623	buf = NULL;
624	}
625
626	/* write just one packet at a time */
627	count = ep->ep.maxpacket;
628	if (count > total) /* min() cannot be used on a bitfield */
629	count = total;
630
631	ep_vdbg(ep->dev, "write %s fifo (IN) %d bytes%s req %p\n",
632	ep->ep.name, count,
633	(count != ep->ep.maxpacket) ? " (short)" : "",
634	req);
635	while (count >= 4) {
636	/* NOTE be careful if you try to align these. fifo lines
637	* should normally be full (4 bytes) and successive partial
638	* lines are ok only in certain cases.
639	*/
640	tmp = get_unaligned((u32 *)buf);
641	cpu_to_le32s(&tmp);
642	writel(val: tmp, addr: &regs->ep_data);
643	buf += 4;
644	count -= 4;
645	}
646
647	/* last fifo entry is "short" unless we wrote a full packet.
648	* also explicitly validate last word in (periodic) transfers
649	* when maxpacket is not a multiple of 4 bytes.
650	*/
651	if (count || total < ep->ep.maxpacket) {
652	tmp = count ? get_unaligned((u32 *)buf) : count;
653	cpu_to_le32s(&tmp);
654	set_fifo_bytecount(ep, count: count & 0x03);
655	writel(val: tmp, addr: &regs->ep_data);
656	}
657
658	/* pci writes may still be posted */
659	}
660
661	/* work around erratum 0106: PCI and USB race over the OUT fifo.
662	* caller guarantees chiprev 0100, out endpoint is NAKing, and
663	* there's no real data in the fifo.
664	*
665	* NOTE: also used in cases where that erratum doesn't apply:
666	* where the host wrote "too much" data to us.
667	*/
668	static void out_flush(struct net2280_ep *ep)
669	{
670	u32 __iomem *statp;
671	u32 tmp;
672
673	statp = &ep->regs->ep_stat;
674
675	tmp = readl(addr: statp);
676	if (tmp & BIT(NAK_OUT_PACKETS)) {
677	ep_dbg(ep->dev, "%s %s %08x !NAK\n",
678	ep->ep.name, __func__, tmp);
679	writel(BIT(SET_NAK_OUT_PACKETS), addr: &ep->regs->ep_rsp);
680	}
681
682	writel(BIT(DATA_OUT_PING_TOKEN_INTERRUPT) |
683	BIT(DATA_PACKET_RECEIVED_INTERRUPT),
684	addr: statp);
685	writel(BIT(FIFO_FLUSH), addr: statp);
686	/* Make sure that stap is written */
687	mb();
688	tmp = readl(addr: statp);
689	if (tmp & BIT(DATA_OUT_PING_TOKEN_INTERRUPT) &&
690	/* high speed did bulk NYET; fifo isn't filling */
691	ep->dev->gadget.speed == USB_SPEED_FULL) {
692	unsigned usec;
693
694	usec = 50; /* 64 byte bulk/interrupt */
695	handshake(ptr: statp, BIT(USB_OUT_PING_NAK_SENT),
696	BIT(USB_OUT_PING_NAK_SENT), usec);
697	/* NAK done; now CLEAR_NAK_OUT_PACKETS is safe */
698	}
699	}
700
701	/* unload packet(s) from the fifo we use for usb OUT transfers.
702	* returns true iff the request completed, because of short packet
703	* or the request buffer having filled with full packets.
704	*
705	* for ep-a..ep-d this will read multiple packets out when they
706	* have been accepted.
707	*/
708	static int read_fifo(struct net2280_ep *ep, struct net2280_request *req)
709	{
710	struct net2280_ep_regs __iomem *regs = ep->regs;
711	u8 *buf = req->req.buf + req->req.actual;
712	unsigned count, tmp, is_short;
713	unsigned cleanup = 0, prevent = 0;
714
715	/* erratum 0106 ... packets coming in during fifo reads might
716	* be incompletely rejected. not all cases have workarounds.
717	*/
718	if (ep->dev->chiprev == 0x0100 &&
719	ep->dev->gadget.speed == USB_SPEED_FULL) {
720	udelay(1);
721	tmp = readl(addr: &ep->regs->ep_stat);
722	if ((tmp & BIT(NAK_OUT_PACKETS)))
723	cleanup = 1;
724	else if ((tmp & BIT(FIFO_FULL))) {
725	start_out_naking(ep);
726	prevent = 1;
727	}
728	/* else: hope we don't see the problem */
729	}
730
731	/* never overflow the rx buffer. the fifo reads packets until
732	* it sees a short one; we might not be ready for them all.
733	*/
734	prefetchw(x: buf);
735	count = readl(addr: &regs->ep_avail);
736	if (unlikely(count == 0)) {
737	udelay(1);
738	tmp = readl(addr: &ep->regs->ep_stat);
739	count = readl(addr: &regs->ep_avail);
740	/* handled that data already? */
741	if (count == 0 && (tmp & BIT(NAK_OUT_PACKETS)) == 0)
742	return 0;
743	}
744
745	tmp = req->req.length - req->req.actual;
746	if (count > tmp) {
747	/* as with DMA, data overflow gets flushed */
748	if ((tmp % ep->ep.maxpacket) != 0) {
749	ep_err(ep->dev,
750	"%s out fifo %d bytes, expected %d\n",
751	ep->ep.name, count, tmp);
752	req->req.status = -EOVERFLOW;
753	cleanup = 1;
754	/* NAK_OUT_PACKETS will be set, so flushing is safe;
755	* the next read will start with the next packet
756	*/
757	} /* else it's a ZLP, no worries */
758	count = tmp;
759	}
760	req->req.actual += count;
761
762	is_short = (count == 0) || ((count % ep->ep.maxpacket) != 0);
763
764	ep_vdbg(ep->dev, "read %s fifo (OUT) %d bytes%s%s%s req %p %d/%d\n",
765	ep->ep.name, count, is_short ? " (short)" : "",
766	cleanup ? " flush" : "", prevent ? " nak" : "",
767	req, req->req.actual, req->req.length);
768
769	while (count >= 4) {
770	tmp = readl(addr: &regs->ep_data);
771	cpu_to_le32s(&tmp);
772	put_unaligned(tmp, (u32 *)buf);
773	buf += 4;
774	count -= 4;
775	}
776	if (count) {
777	tmp = readl(addr: &regs->ep_data);
778	/* LE conversion is implicit here: */
779	do {
780	*buf++ = (u8) tmp;
781	tmp >>= 8;
782	} while (--count);
783	}
784	if (cleanup)
785	out_flush(ep);
786	if (prevent) {
787	writel(BIT(CLEAR_NAK_OUT_PACKETS), addr: &ep->regs->ep_rsp);
788	(void) readl(addr: &ep->regs->ep_rsp);
789	}
790
791	return is_short || req->req.actual == req->req.length;
792	}
793
794	/* fill out dma descriptor to match a given request */
795	static void fill_dma_desc(struct net2280_ep *ep,
796	struct net2280_request *req, int valid)
797	{
798	struct net2280_dma *td = req->td;
799	u32 dmacount = req->req.length;
800
801	/* don't let DMA continue after a short OUT packet,
802	* so overruns can't affect the next transfer.
803	* in case of overruns on max-size packets, we can't
804	* stop the fifo from filling but we can flush it.
805	*/
806	if (ep->is_in)
807	dmacount |= BIT(DMA_DIRECTION);
808	if ((!ep->is_in && (dmacount % ep->ep.maxpacket) != 0) ||
809	!(ep->dev->quirks & PLX_2280))
810	dmacount |= BIT(END_OF_CHAIN);
811
812	req->valid = valid;
813	if (valid)
814	dmacount |= BIT(VALID_BIT);
815	dmacount |= BIT(DMA_DONE_INTERRUPT_ENABLE);
816
817	/* td->dmadesc = previously set by caller */
818	td->dmaaddr = cpu_to_le32 (req->req.dma);
819
820	/* 2280 may be polling VALID_BIT through ep->dma->dmadesc */
821	wmb();
822	td->dmacount = cpu_to_le32(dmacount);
823	}
824
825	static const u32 dmactl_default =
826	BIT(DMA_SCATTER_GATHER_DONE_INTERRUPT) |
827	BIT(DMA_CLEAR_COUNT_ENABLE) |
828	/* erratum 0116 workaround part 1 (use POLLING) */
829	(POLL_100_USEC << DESCRIPTOR_POLLING_RATE) |
830	BIT(DMA_VALID_BIT_POLLING_ENABLE) |
831	BIT(DMA_VALID_BIT_ENABLE) |
832	BIT(DMA_SCATTER_GATHER_ENABLE) |
833	/* erratum 0116 workaround part 2 (no AUTOSTART) */
834	BIT(DMA_ENABLE);
835
836	static inline void spin_stop_dma(struct net2280_dma_regs __iomem *dma)
837	{
838	handshake(ptr: &dma->dmactl, BIT(DMA_ENABLE), done: 0, usec: 50);
839	}
840
841	static inline void stop_dma(struct net2280_dma_regs __iomem *dma)
842	{
843	writel(readl(addr: &dma->dmactl) & ~BIT(DMA_ENABLE), addr: &dma->dmactl);
844	spin_stop_dma(dma);
845	}
846
847	static void start_queue(struct net2280_ep *ep, u32 dmactl, u32 td_dma)
848	{
849	struct net2280_dma_regs __iomem *dma = ep->dma;
850	unsigned int tmp = BIT(VALID_BIT) | (ep->is_in << DMA_DIRECTION);
851
852	if (!(ep->dev->quirks & PLX_2280))
853	tmp |= BIT(END_OF_CHAIN);
854
855	writel(val: tmp, addr: &dma->dmacount);
856	writel(readl(addr: &dma->dmastat), addr: &dma->dmastat);
857
858	writel(val: td_dma, addr: &dma->dmadesc);
859	if (ep->dev->quirks & PLX_PCIE)
860	dmactl |= BIT(DMA_REQUEST_OUTSTANDING);
861	writel(val: dmactl, addr: &dma->dmactl);
862
863	/* erratum 0116 workaround part 3: pci arbiter away from net2280 */
864	(void) readl(addr: &ep->dev->pci->pcimstctl);
865
866	writel(BIT(DMA_START), addr: &dma->dmastat);
867	}
868
869	static void start_dma(struct net2280_ep *ep, struct net2280_request *req)
870	{
871	u32 tmp;
872	struct net2280_dma_regs __iomem *dma = ep->dma;
873
874	/* FIXME can't use DMA for ZLPs */
875
876	/* on this path we "know" there's no dma active (yet) */
877	WARN_ON(readl(&dma->dmactl) & BIT(DMA_ENABLE));
878	writel(val: 0, addr: &ep->dma->dmactl);
879
880	/* previous OUT packet might have been short */
881	if (!ep->is_in && (readl(addr: &ep->regs->ep_stat) &
882	BIT(NAK_OUT_PACKETS))) {
883	writel(BIT(SHORT_PACKET_TRANSFERRED_INTERRUPT),
884	addr: &ep->regs->ep_stat);
885
886	tmp = readl(addr: &ep->regs->ep_avail);
887	if (tmp) {
888	writel(readl(addr: &dma->dmastat), addr: &dma->dmastat);
889
890	/* transfer all/some fifo data */
891	writel(val: req->req.dma, addr: &dma->dmaaddr);
892	tmp = min(tmp, req->req.length);
893
894	/* dma irq, faking scatterlist status */
895	req->td->dmacount = cpu_to_le32(req->req.length - tmp);
896	writel(BIT(DMA_DONE_INTERRUPT_ENABLE) | tmp,
897	addr: &dma->dmacount);
898	req->td->dmadesc = 0;
899	req->valid = 1;
900
901	writel(BIT(DMA_ENABLE), addr: &dma->dmactl);
902	writel(BIT(DMA_START), addr: &dma->dmastat);
903	return;
904	}
905	stop_out_naking(ep);
906	}
907
908	tmp = dmactl_default;
909
910	/* force packet boundaries between dma requests, but prevent the
911	* controller from automagically writing a last "short" packet
912	* (zero length) unless the driver explicitly said to do that.
913	*/
914	if (ep->is_in) {
915	if (likely((req->req.length % ep->ep.maxpacket) ||
916	req->req.zero)){
917	tmp |= BIT(DMA_FIFO_VALIDATE);
918	ep->in_fifo_validate = 1;
919	} else
920	ep->in_fifo_validate = 0;
921	}
922
923	/* init req->td, pointing to the current dummy */
924	req->td->dmadesc = cpu_to_le32 (ep->td_dma);
925	fill_dma_desc(ep, req, valid: 1);
926
927	req->td->dmacount |= cpu_to_le32(BIT(END_OF_CHAIN));
928
929	start_queue(ep, dmactl: tmp, td_dma: req->td_dma);
930	}
931
932	static inline void
933	queue_dma(struct net2280_ep *ep, struct net2280_request *req, int valid)
934	{
935	/* swap new dummy for old, link; fill and maybe activate */
936	swap(ep->dummy, req->td);
937	swap(ep->td_dma, req->td_dma);
938
939	req->td->dmadesc = cpu_to_le32 (ep->td_dma);
940
941	fill_dma_desc(ep, req, valid);
942	}
943
944	static void
945	done(struct net2280_ep *ep, struct net2280_request *req, int status)
946	{
947	struct net2280 *dev;
948	unsigned stopped = ep->stopped;
949
950	list_del_init(entry: &req->queue);
951
952	if (req->req.status == -EINPROGRESS)
953	req->req.status = status;
954	else
955	status = req->req.status;
956
957	dev = ep->dev;
958	if (ep->dma)
959	usb_gadget_unmap_request(gadget: &dev->gadget, req: &req->req, is_in: ep->is_in);
960
961	if (status && status != -ESHUTDOWN)
962	ep_vdbg(dev, "complete %s req %p stat %d len %u/%u\n",
963	ep->ep.name, &req->req, status,
964	req->req.actual, req->req.length);
965
966	/* don't modify queue heads during completion callback */
967	ep->stopped = 1;
968	spin_unlock(lock: &dev->lock);
969	usb_gadget_giveback_request(ep: &ep->ep, req: &req->req);
970	spin_lock(lock: &dev->lock);
971	ep->stopped = stopped;
972	}
973
974	/*-------------------------------------------------------------------------*/
975
976	static int
977	net2280_queue(struct usb_ep *_ep, struct usb_request *_req, gfp_t gfp_flags)
978	{
979	struct net2280_request *req;
980	struct net2280_ep *ep;
981	struct net2280 *dev;
982	unsigned long flags;
983	int ret = 0;
984
985	/* we always require a cpu-view buffer, so that we can
986	* always use pio (as fallback or whatever).
987	*/
988	ep = container_of(_ep, struct net2280_ep, ep);
989	if (!_ep || (!ep->desc && ep->num != 0)) {
990	pr_err("%s: Invalid ep=%p or ep->desc\n", __func__, _ep);
991	return -EINVAL;
992	}
993	req = container_of(_req, struct net2280_request, req);
994	if (!_req || !_req->complete || !_req->buf ||
995	!list_empty(head: &req->queue)) {
996	ret = -EINVAL;
997	goto print_err;
998	}
999	if (_req->length > (~0 & DMA_BYTE_COUNT_MASK)) {
1000	ret = -EDOM;
1001	goto print_err;
1002	}
1003	dev = ep->dev;
1004	if (!dev->driver || dev->gadget.speed == USB_SPEED_UNKNOWN) {
1005	ret = -ESHUTDOWN;
1006	goto print_err;
1007	}
1008
1009	/* FIXME implement PIO fallback for ZLPs with DMA */
1010	if (ep->dma && _req->length == 0) {
1011	ret = -EOPNOTSUPP;
1012	goto print_err;
1013	}
1014
1015	/* set up dma mapping in case the caller didn't */
1016	if (ep->dma) {
1017	ret = usb_gadget_map_request(gadget: &dev->gadget, req: _req,
1018	is_in: ep->is_in);
1019	if (ret)
1020	goto print_err;
1021	}
1022
1023	ep_vdbg(dev, "%s queue req %p, len %d buf %p\n",
1024	_ep->name, _req, _req->length, _req->buf);
1025
1026	spin_lock_irqsave(&dev->lock, flags);
1027
1028	_req->status = -EINPROGRESS;
1029	_req->actual = 0;
1030
1031	/* kickstart this i/o queue? */
1032	if (list_empty(head: &ep->queue) && !ep->stopped &&
1033	!((dev->quirks & PLX_PCIE) && ep->dma &&
1034	(readl(addr: &ep->regs->ep_rsp) & BIT(CLEAR_ENDPOINT_HALT)))) {
1035
1036	/* use DMA if the endpoint supports it, else pio */
1037	if (ep->dma)
1038	start_dma(ep, req);
1039	else {
1040	/* maybe there's no control data, just status ack */
1041	if (ep->num == 0 && _req->length == 0) {
1042	allow_status(ep);
1043	done(ep, req, status: 0);
1044	ep_vdbg(dev, "%s status ack\n", ep->ep.name);
1045	goto done;
1046	}
1047
1048	/* PIO ... stuff the fifo, or unblock it. */
1049	if (ep->is_in)
1050	write_fifo(ep, req: _req);
1051	else {
1052	u32 s;
1053
1054	/* OUT FIFO might have packet(s) buffered */
1055	s = readl(addr: &ep->regs->ep_stat);
1056	if ((s & BIT(FIFO_EMPTY)) == 0) {
1057	/* note: _req->short_not_ok is
1058	* ignored here since PIO _always_
1059	* stops queue advance here, and
1060	* _req->status doesn't change for
1061	* short reads (only _req->actual)
1062	*/
1063	if (read_fifo(ep, req) &&
1064	ep->num == 0) {
1065	done(ep, req, status: 0);
1066	allow_status(ep);
1067	/* don't queue it */
1068	req = NULL;
1069	} else if (read_fifo(ep, req) &&
1070	ep->num != 0) {
1071	done(ep, req, status: 0);
1072	req = NULL;
1073	} else
1074	s = readl(addr: &ep->regs->ep_stat);
1075	}
1076
1077	/* don't NAK, let the fifo fill */
1078	if (req && (s & BIT(NAK_OUT_PACKETS)))
1079	writel(BIT(CLEAR_NAK_OUT_PACKETS),
1080	addr: &ep->regs->ep_rsp);
1081	}
1082	}
1083
1084	} else if (ep->dma) {
1085	int valid = 1;
1086
1087	if (ep->is_in) {
1088	int expect;
1089
1090	/* preventing magic zlps is per-engine state, not
1091	* per-transfer; irq logic must recover hiccups.
1092	*/
1093	expect = likely(req->req.zero ||
1094	(req->req.length % ep->ep.maxpacket));
1095	if (expect != ep->in_fifo_validate)
1096	valid = 0;
1097	}
1098	queue_dma(ep, req, valid);
1099
1100	} /* else the irq handler advances the queue. */
1101
1102	ep->responded = 1;
1103	if (req)
1104	list_add_tail(new: &req->queue, head: &ep->queue);
1105	done:
1106	spin_unlock_irqrestore(lock: &dev->lock, flags);
1107
1108	/* pci writes may still be posted */
1109	return ret;
1110
1111	print_err:
1112	dev_err(&ep->dev->pdev->dev, "%s: error=%d\n", __func__, ret);
1113	return ret;
1114	}
1115
1116	static inline void
1117	dma_done(struct net2280_ep *ep, struct net2280_request *req, u32 dmacount,
1118	int status)
1119	{
1120	req->req.actual = req->req.length - (DMA_BYTE_COUNT_MASK & dmacount);
1121	done(ep, req, status);
1122	}
1123
1124	static int scan_dma_completions(struct net2280_ep *ep)
1125	{
1126	int num_completed = 0;
1127
1128	/* only look at descriptors that were "naturally" retired,
1129	* so fifo and list head state won't matter
1130	*/
1131	while (!list_empty(head: &ep->queue)) {
1132	struct net2280_request *req;
1133	u32 req_dma_count;
1134
1135	req = list_entry(ep->queue.next,
1136	struct net2280_request, queue);
1137	if (!req->valid)
1138	break;
1139	rmb();
1140	req_dma_count = le32_to_cpup(p: &req->td->dmacount);
1141	if ((req_dma_count & BIT(VALID_BIT)) != 0)
1142	break;
1143
1144	/* SHORT_PACKET_TRANSFERRED_INTERRUPT handles "usb-short"
1145	* cases where DMA must be aborted; this code handles
1146	* all non-abort DMA completions.
1147	*/
1148	if (unlikely(req->td->dmadesc == 0)) {
1149	/* paranoia */
1150	u32 const ep_dmacount = readl(addr: &ep->dma->dmacount);
1151
1152	if (ep_dmacount & DMA_BYTE_COUNT_MASK)
1153	break;
1154	/* single transfer mode */
1155	dma_done(ep, req, dmacount: req_dma_count, status: 0);
1156	num_completed++;
1157	break;
1158	} else if (!ep->is_in &&
1159	(req->req.length % ep->ep.maxpacket) &&
1160	!(ep->dev->quirks & PLX_PCIE)) {
1161
1162	u32 const ep_stat = readl(addr: &ep->regs->ep_stat);
1163	/* AVOID TROUBLE HERE by not issuing short reads from
1164	* your gadget driver. That helps avoids errata 0121,
1165	* 0122, and 0124; not all cases trigger the warning.
1166	*/
1167	if ((ep_stat & BIT(NAK_OUT_PACKETS)) == 0) {
1168	ep_warn(ep->dev, "%s lost packet sync!\n",
1169	ep->ep.name);
1170	req->req.status = -EOVERFLOW;
1171	} else {
1172	u32 const ep_avail = readl(addr: &ep->regs->ep_avail);
1173	if (ep_avail) {
1174	/* fifo gets flushed later */
1175	ep->out_overflow = 1;
1176	ep_dbg(ep->dev,
1177	"%s dma, discard %d len %d\n",
1178	ep->ep.name, ep_avail,
1179	req->req.length);
1180	req->req.status = -EOVERFLOW;
1181	}
1182	}
1183	}
1184	dma_done(ep, req, dmacount: req_dma_count, status: 0);
1185	num_completed++;
1186	}
1187
1188	return num_completed;
1189	}
1190
1191	static void restart_dma(struct net2280_ep *ep)
1192	{
1193	struct net2280_request *req;
1194
1195	if (ep->stopped)
1196	return;
1197	req = list_entry(ep->queue.next, struct net2280_request, queue);
1198
1199	start_dma(ep, req);
1200	}
1201
1202	static void abort_dma(struct net2280_ep *ep)
1203	{
1204	/* abort the current transfer */
1205	if (likely(!list_empty(&ep->queue))) {
1206	/* FIXME work around errata 0121, 0122, 0124 */
1207	writel(BIT(DMA_ABORT), addr: &ep->dma->dmastat);
1208	spin_stop_dma(dma: ep->dma);
1209	} else
1210	stop_dma(dma: ep->dma);
1211	scan_dma_completions(ep);
1212	}
1213
1214	/* dequeue ALL requests */
1215	static void nuke(struct net2280_ep *ep)
1216	{
1217	struct net2280_request *req;
1218
1219	/* called with spinlock held */
1220	ep->stopped = 1;
1221	if (ep->dma)
1222	abort_dma(ep);
1223	while (!list_empty(head: &ep->queue)) {
1224	req = list_entry(ep->queue.next,
1225	struct net2280_request,
1226	queue);
1227	done(ep, req, status: -ESHUTDOWN);
1228	}
1229	}
1230
1231	/* dequeue JUST ONE request */
1232	static int net2280_dequeue(struct usb_ep *_ep, struct usb_request *_req)
1233	{
1234	struct net2280_ep *ep;
1235	struct net2280_request *req = NULL;
1236	struct net2280_request *iter;
1237	unsigned long flags;
1238	u32 dmactl;
1239	int stopped;
1240
1241	ep = container_of(_ep, struct net2280_ep, ep);
1242	if (!_ep || (!ep->desc && ep->num != 0) || !_req) {
1243	pr_err("%s: Invalid ep=%p or ep->desc or req=%p\n",
1244	__func__, _ep, _req);
1245	return -EINVAL;
1246	}
1247
1248	spin_lock_irqsave(&ep->dev->lock, flags);
1249	stopped = ep->stopped;
1250
1251	/* quiesce dma while we patch the queue */
1252	dmactl = 0;
1253	ep->stopped = 1;
1254	if (ep->dma) {
1255	dmactl = readl(addr: &ep->dma->dmactl);
1256	/* WARNING erratum 0127 may kick in ... */
1257	stop_dma(dma: ep->dma);
1258	scan_dma_completions(ep);
1259	}
1260
1261	/* make sure it's still queued on this endpoint */
1262	list_for_each_entry(iter, &ep->queue, queue) {
1263	if (&iter->req != _req)
1264	continue;
1265	req = iter;
1266	break;
1267	}
1268	if (!req) {
1269	ep->stopped = stopped;
1270	spin_unlock_irqrestore(lock: &ep->dev->lock, flags);
1271	ep_dbg(ep->dev, "%s: Request mismatch\n", __func__);
1272	return -EINVAL;
1273	}
1274
1275	/* queue head may be partially complete. */
1276	if (ep->queue.next == &req->queue) {
1277	if (ep->dma) {
1278	ep_dbg(ep->dev, "unlink (%s) dma\n", _ep->name);
1279	_req->status = -ECONNRESET;
1280	abort_dma(ep);
1281	if (likely(ep->queue.next == &req->queue)) {
1282	/* NOTE: misreports single-transfer mode*/
1283	req->td->dmacount = 0; /* invalidate */
1284	dma_done(ep, req,
1285	readl(addr: &ep->dma->dmacount),
1286	status: -ECONNRESET);
1287	}
1288	} else {
1289	ep_dbg(ep->dev, "unlink (%s) pio\n", _ep->name);
1290	done(ep, req, status: -ECONNRESET);
1291	}
1292	req = NULL;
1293	}
1294
1295	if (req)
1296	done(ep, req, status: -ECONNRESET);
1297	ep->stopped = stopped;
1298
1299	if (ep->dma) {
1300	/* turn off dma on inactive queues */
1301	if (list_empty(head: &ep->queue))
1302	stop_dma(dma: ep->dma);
1303	else if (!ep->stopped) {
1304	/* resume current request, or start new one */
1305	if (req)
1306	writel(val: dmactl, addr: &ep->dma->dmactl);
1307	else
1308	start_dma(ep, list_entry(ep->queue.next,
1309	struct net2280_request, queue));
1310	}
1311	}
1312
1313	spin_unlock_irqrestore(lock: &ep->dev->lock, flags);
1314	return 0;
1315	}
1316
1317	/*-------------------------------------------------------------------------*/
1318
1319	static int net2280_fifo_status(struct usb_ep *_ep);
1320
1321	static int
1322	net2280_set_halt_and_wedge(struct usb_ep *_ep, int value, int wedged)
1323	{
1324	struct net2280_ep *ep;
1325	unsigned long flags;
1326	int retval = 0;
1327
1328	ep = container_of(_ep, struct net2280_ep, ep);
1329	if (!_ep || (!ep->desc && ep->num != 0)) {
1330	pr_err("%s: Invalid ep=%p or ep->desc\n", __func__, _ep);
1331	return -EINVAL;
1332	}
1333	if (!ep->dev->driver || ep->dev->gadget.speed == USB_SPEED_UNKNOWN) {
1334	retval = -ESHUTDOWN;
1335	goto print_err;
1336	}
1337	if (ep->desc /* not ep0 */ && (ep->desc->bmAttributes & 0x03)
1338	== USB_ENDPOINT_XFER_ISOC) {
1339	retval = -EINVAL;
1340	goto print_err;
1341	}
1342
1343	spin_lock_irqsave(&ep->dev->lock, flags);
1344	if (!list_empty(head: &ep->queue)) {
1345	retval = -EAGAIN;
1346	goto print_unlock;
1347	} else if (ep->is_in && value && net2280_fifo_status(_ep) != 0) {
1348	retval = -EAGAIN;
1349	goto print_unlock;
1350	} else {
1351	ep_vdbg(ep->dev, "%s %s %s\n", _ep->name,
1352	value ? "set" : "clear",
1353	wedged ? "wedge" : "halt");
1354	/* set/clear, then synch memory views with the device */
1355	if (value) {
1356	if (ep->num == 0)
1357	ep->dev->protocol_stall = 1;
1358	else
1359	set_halt(ep);
1360	if (wedged)
1361	ep->wedged = 1;
1362	} else {
1363	clear_halt(ep);
1364	if (ep->dev->quirks & PLX_PCIE &&
1365	!list_empty(head: &ep->queue) && ep->td_dma)
1366	restart_dma(ep);
1367	ep->wedged = 0;
1368	}
1369	(void) readl(addr: &ep->regs->ep_rsp);
1370	}
1371	spin_unlock_irqrestore(lock: &ep->dev->lock, flags);
1372
1373	return retval;
1374
1375	print_unlock:
1376	spin_unlock_irqrestore(lock: &ep->dev->lock, flags);
1377	print_err:
1378	dev_err(&ep->dev->pdev->dev, "%s: error=%d\n", __func__, retval);
1379	return retval;
1380	}
1381
1382	static int net2280_set_halt(struct usb_ep *_ep, int value)
1383	{
1384	return net2280_set_halt_and_wedge(_ep, value, wedged: 0);
1385	}
1386
1387	static int net2280_set_wedge(struct usb_ep *_ep)
1388	{
1389	if (!_ep || _ep->name == ep0name) {
1390	pr_err("%s: Invalid ep=%p or ep0\n", __func__, _ep);
1391	return -EINVAL;
1392	}
1393	return net2280_set_halt_and_wedge(_ep, value: 1, wedged: 1);
1394	}
1395
1396	static int net2280_fifo_status(struct usb_ep *_ep)
1397	{
1398	struct net2280_ep *ep;
1399	u32 avail;
1400
1401	ep = container_of(_ep, struct net2280_ep, ep);
1402	if (!_ep || (!ep->desc && ep->num != 0)) {
1403	pr_err("%s: Invalid ep=%p or ep->desc\n", __func__, _ep);
1404	return -ENODEV;
1405	}
1406	if (!ep->dev->driver || ep->dev->gadget.speed == USB_SPEED_UNKNOWN) {
1407	dev_err(&ep->dev->pdev->dev,
1408	"%s: Invalid driver=%p or speed=%d\n",
1409	__func__, ep->dev->driver, ep->dev->gadget.speed);
1410	return -ESHUTDOWN;
1411	}
1412
1413	avail = readl(addr: &ep->regs->ep_avail) & (BIT(12) - 1);
1414	if (avail > ep->fifo_size) {
1415	dev_err(&ep->dev->pdev->dev, "%s: Fifo overflow\n", __func__);
1416	return -EOVERFLOW;
1417	}
1418	if (ep->is_in)
1419	avail = ep->fifo_size - avail;
1420	return avail;
1421	}
1422
1423	static void net2280_fifo_flush(struct usb_ep *_ep)
1424	{
1425	struct net2280_ep *ep;
1426
1427	ep = container_of(_ep, struct net2280_ep, ep);
1428	if (!_ep || (!ep->desc && ep->num != 0)) {
1429	pr_err("%s: Invalid ep=%p or ep->desc\n", __func__, _ep);
1430	return;
1431	}
1432	if (!ep->dev->driver || ep->dev->gadget.speed == USB_SPEED_UNKNOWN) {
1433	dev_err(&ep->dev->pdev->dev,
1434	"%s: Invalid driver=%p or speed=%d\n",
1435	__func__, ep->dev->driver, ep->dev->gadget.speed);
1436	return;
1437	}
1438
1439	writel(BIT(FIFO_FLUSH), addr: &ep->regs->ep_stat);
1440	(void) readl(addr: &ep->regs->ep_rsp);
1441	}
1442
1443	static const struct usb_ep_ops net2280_ep_ops = {
1444	.enable = net2280_enable,
1445	.disable = net2280_disable,
1446
1447	.alloc_request = net2280_alloc_request,
1448	.free_request = net2280_free_request,
1449
1450	.queue = net2280_queue,
1451	.dequeue = net2280_dequeue,
1452
1453	.set_halt = net2280_set_halt,
1454	.set_wedge = net2280_set_wedge,
1455	.fifo_status = net2280_fifo_status,
1456	.fifo_flush = net2280_fifo_flush,
1457	};
1458
1459	/*-------------------------------------------------------------------------*/
1460
1461	static int net2280_get_frame(struct usb_gadget *_gadget)
1462	{
1463	struct net2280 *dev;
1464	unsigned long flags;
1465	u16 retval;
1466
1467	if (!_gadget)
1468	return -ENODEV;
1469	dev = container_of(_gadget, struct net2280, gadget);
1470	spin_lock_irqsave(&dev->lock, flags);
1471	retval = get_idx_reg(regs: dev->regs, REG_FRAME) & 0x03ff;
1472	spin_unlock_irqrestore(lock: &dev->lock, flags);
1473	return retval;
1474	}
1475
1476	static int net2280_wakeup(struct usb_gadget *_gadget)
1477	{
1478	struct net2280 *dev;
1479	u32 tmp;
1480	unsigned long flags;
1481
1482	if (!_gadget)
1483	return 0;
1484	dev = container_of(_gadget, struct net2280, gadget);
1485
1486	spin_lock_irqsave(&dev->lock, flags);
1487	tmp = readl(addr: &dev->usb->usbctl);
1488	if (tmp & BIT(DEVICE_REMOTE_WAKEUP_ENABLE))
1489	writel(BIT(GENERATE_RESUME), addr: &dev->usb->usbstat);
1490	spin_unlock_irqrestore(lock: &dev->lock, flags);
1491
1492	/* pci writes may still be posted */
1493	return 0;
1494	}
1495
1496	static int net2280_set_selfpowered(struct usb_gadget *_gadget, int value)
1497	{
1498	struct net2280 *dev;
1499	u32 tmp;
1500	unsigned long flags;
1501
1502	if (!_gadget)
1503	return 0;
1504	dev = container_of(_gadget, struct net2280, gadget);
1505
1506	spin_lock_irqsave(&dev->lock, flags);
1507	tmp = readl(addr: &dev->usb->usbctl);
1508	if (value) {
1509	tmp |= BIT(SELF_POWERED_STATUS);
1510	_gadget->is_selfpowered = 1;
1511	} else {
1512	tmp &= ~BIT(SELF_POWERED_STATUS);
1513	_gadget->is_selfpowered = 0;
1514	}
1515	writel(val: tmp, addr: &dev->usb->usbctl);
1516	spin_unlock_irqrestore(lock: &dev->lock, flags);
1517
1518	return 0;
1519	}
1520
1521	static int net2280_pullup(struct usb_gadget *_gadget, int is_on)
1522	{
1523	struct net2280 *dev;
1524	u32 tmp;
1525	unsigned long flags;
1526
1527	if (!_gadget)
1528	return -ENODEV;
1529	dev = container_of(_gadget, struct net2280, gadget);
1530
1531	spin_lock_irqsave(&dev->lock, flags);
1532	tmp = readl(addr: &dev->usb->usbctl);
1533	dev->softconnect = (is_on != 0);
1534	if (is_on) {
1535	ep0_start(dev);
1536	writel(val: tmp | BIT(USB_DETECT_ENABLE), addr: &dev->usb->usbctl);
1537	} else {
1538	writel(val: tmp & ~BIT(USB_DETECT_ENABLE), addr: &dev->usb->usbctl);
1539	stop_activity(dev, NULL);
1540	}
1541
1542	spin_unlock_irqrestore(lock: &dev->lock, flags);
1543
1544	return 0;
1545	}
1546
1547	static struct usb_ep *net2280_match_ep(struct usb_gadget *_gadget,
1548	struct usb_endpoint_descriptor *desc,
1549	struct usb_ss_ep_comp_descriptor *ep_comp)
1550	{
1551	char name[8];
1552	struct usb_ep *ep;
1553
1554	if (usb_endpoint_type(epd: desc) == USB_ENDPOINT_XFER_INT) {
1555	/* ep-e, ep-f are PIO with only 64 byte fifos */
1556	ep = gadget_find_ep_by_name(g: _gadget, name: "ep-e");
1557	if (ep && usb_gadget_ep_match_desc(gadget: _gadget, ep, desc, ep_comp))
1558	return ep;
1559	ep = gadget_find_ep_by_name(g: _gadget, name: "ep-f");
1560	if (ep && usb_gadget_ep_match_desc(gadget: _gadget, ep, desc, ep_comp))
1561	return ep;
1562	}
1563
1564	/* USB3380: Only first four endpoints have DMA channels. Allocate
1565	* slower interrupt endpoints from PIO hw endpoints, to allow bulk/isoc
1566	* endpoints use DMA hw endpoints.
1567	*/
1568	if (usb_endpoint_type(epd: desc) == USB_ENDPOINT_XFER_INT &&
1569	usb_endpoint_dir_in(epd: desc)) {
1570	ep = gadget_find_ep_by_name(g: _gadget, name: "ep2in");
1571	if (ep && usb_gadget_ep_match_desc(gadget: _gadget, ep, desc, ep_comp))
1572	return ep;
1573	ep = gadget_find_ep_by_name(g: _gadget, name: "ep4in");
1574	if (ep && usb_gadget_ep_match_desc(gadget: _gadget, ep, desc, ep_comp))
1575	return ep;
1576	} else if (usb_endpoint_type(epd: desc) == USB_ENDPOINT_XFER_INT &&
1577	!usb_endpoint_dir_in(epd: desc)) {
1578	ep = gadget_find_ep_by_name(g: _gadget, name: "ep1out");
1579	if (ep && usb_gadget_ep_match_desc(gadget: _gadget, ep, desc, ep_comp))
1580	return ep;
1581	ep = gadget_find_ep_by_name(g: _gadget, name: "ep3out");
1582	if (ep && usb_gadget_ep_match_desc(gadget: _gadget, ep, desc, ep_comp))
1583	return ep;
1584	} else if (usb_endpoint_type(epd: desc) != USB_ENDPOINT_XFER_BULK &&
1585	usb_endpoint_dir_in(epd: desc)) {
1586	ep = gadget_find_ep_by_name(g: _gadget, name: "ep1in");
1587	if (ep && usb_gadget_ep_match_desc(gadget: _gadget, ep, desc, ep_comp))
1588	return ep;
1589	ep = gadget_find_ep_by_name(g: _gadget, name: "ep3in");
1590	if (ep && usb_gadget_ep_match_desc(gadget: _gadget, ep, desc, ep_comp))
1591	return ep;
1592	} else if (usb_endpoint_type(epd: desc) != USB_ENDPOINT_XFER_BULK &&
1593	!usb_endpoint_dir_in(epd: desc)) {
1594	ep = gadget_find_ep_by_name(g: _gadget, name: "ep2out");
1595	if (ep && usb_gadget_ep_match_desc(gadget: _gadget, ep, desc, ep_comp))
1596	return ep;
1597	ep = gadget_find_ep_by_name(g: _gadget, name: "ep4out");
1598	if (ep && usb_gadget_ep_match_desc(gadget: _gadget, ep, desc, ep_comp))
1599	return ep;
1600	}
1601
1602	/* USB3380: use same address for usb and hardware endpoints */
1603	snprintf(buf: name, size: sizeof(name), fmt: "ep%d%s", usb_endpoint_num(epd: desc),
1604	usb_endpoint_dir_in(epd: desc) ? "in" : "out");
1605	ep = gadget_find_ep_by_name(g: _gadget, name);
1606	if (ep && usb_gadget_ep_match_desc(gadget: _gadget, ep, desc, ep_comp))
1607	return ep;
1608
1609	return NULL;
1610	}
1611
1612	static int net2280_start(struct usb_gadget *_gadget,
1613	struct usb_gadget_driver *driver);
1614	static int net2280_stop(struct usb_gadget *_gadget);
1615	static void net2280_async_callbacks(struct usb_gadget *_gadget, bool enable);
1616
1617	static const struct usb_gadget_ops net2280_ops = {
1618	.get_frame = net2280_get_frame,
1619	.wakeup = net2280_wakeup,
1620	.set_selfpowered = net2280_set_selfpowered,
1621	.pullup = net2280_pullup,
1622	.udc_start = net2280_start,
1623	.udc_stop = net2280_stop,
1624	.udc_async_callbacks = net2280_async_callbacks,
1625	.match_ep = net2280_match_ep,
1626	};
1627
1628	/*-------------------------------------------------------------------------*/
1629
1630	#ifdef CONFIG_USB_GADGET_DEBUG_FILES
1631
1632	/* FIXME move these into procfs, and use seq_file.
1633	* Sysfs _still_ doesn't behave for arbitrarily sized files,
1634	* and also doesn't help products using this with 2.4 kernels.
1635	*/
1636
1637	/* "function" sysfs attribute */
1638	static ssize_t function_show(struct device *_dev, struct device_attribute *attr,
1639	char *buf)
1640	{
1641	struct net2280 *dev = dev_get_drvdata(dev: _dev);
1642
1643	if (!dev->driver || !dev->driver->function ||
1644	strlen(dev->driver->function) > PAGE_SIZE)
1645	return 0;
1646	return scnprintf(buf, PAGE_SIZE, fmt: "%s\n", dev->driver->function);
1647	}
1648	static DEVICE_ATTR_RO(function);
1649
1650	static ssize_t registers_show(struct device *_dev,
1651	struct device_attribute *attr, char *buf)
1652	{
1653	struct net2280 *dev;
1654	char *next;
1655	unsigned size, t;
1656	unsigned long flags;
1657	int i;
1658	u32 t1, t2;
1659	const char *s;
1660
1661	dev = dev_get_drvdata(dev: _dev);
1662	next = buf;
1663	size = PAGE_SIZE;
1664	spin_lock_irqsave(&dev->lock, flags);
1665
1666	if (dev->driver)
1667	s = dev->driver->driver.name;
1668	else
1669	s = "(none)";
1670
1671	/* Main Control Registers */
1672	t = scnprintf(buf: next, size, fmt: "%s version " DRIVER_VERSION
1673	", chiprev %04x\n\n"
1674	"devinit %03x fifoctl %08x gadget '%s'\n"
1675	"pci irqenb0 %02x irqenb1 %08x "
1676	"irqstat0 %04x irqstat1 %08x\n",
1677	driver_name, dev->chiprev,
1678	readl(addr: &dev->regs->devinit),
1679	readl(addr: &dev->regs->fifoctl),
1680	s,
1681	readl(addr: &dev->regs->pciirqenb0),
1682	readl(addr: &dev->regs->pciirqenb1),
1683	readl(addr: &dev->regs->irqstat0),
1684	readl(addr: &dev->regs->irqstat1));
1685	size -= t;
1686	next += t;
1687
1688	/* USB Control Registers */
1689	t1 = readl(addr: &dev->usb->usbctl);
1690	t2 = readl(addr: &dev->usb->usbstat);
1691	if (t1 & BIT(VBUS_PIN)) {
1692	if (t2 & BIT(HIGH_SPEED))
1693	s = "high speed";
1694	else if (dev->gadget.speed == USB_SPEED_UNKNOWN)
1695	s = "powered";
1696	else
1697	s = "full speed";
1698	/* full speed bit (6) not working?? */
1699	} else
1700	s = "not attached";
1701	t = scnprintf(buf: next, size,
1702	fmt: "stdrsp %08x usbctl %08x usbstat %08x "
1703	"addr 0x%02x (%s)\n",
1704	readl(addr: &dev->usb->stdrsp), t1, t2,
1705	readl(addr: &dev->usb->ouraddr), s);
1706	size -= t;
1707	next += t;
1708
1709	/* PCI Master Control Registers */
1710
1711	/* DMA Control Registers */
1712
1713	/* Configurable EP Control Registers */
1714	for (i = 0; i < dev->n_ep; i++) {
1715	struct net2280_ep *ep;
1716
1717	ep = &dev->ep[i];
1718	if (i && !ep->desc)
1719	continue;
1720
1721	t1 = readl(addr: &ep->cfg->ep_cfg);
1722	t2 = readl(addr: &ep->regs->ep_rsp) & 0xff;
1723	t = scnprintf(buf: next, size,
1724	fmt: "\n%s\tcfg %05x rsp (%02x) %s%s%s%s%s%s%s%s"
1725	"irqenb %02x\n",
1726	ep->ep.name, t1, t2,
1727	(t2 & BIT(CLEAR_NAK_OUT_PACKETS))
1728	? "NAK " : "",
1729	(t2 & BIT(CLEAR_EP_HIDE_STATUS_PHASE))
1730	? "hide " : "",
1731	(t2 & BIT(CLEAR_EP_FORCE_CRC_ERROR))
1732	? "CRC " : "",
1733	(t2 & BIT(CLEAR_INTERRUPT_MODE))
1734	? "interrupt " : "",
1735	(t2 & BIT(CLEAR_CONTROL_STATUS_PHASE_HANDSHAKE))
1736	? "status " : "",
1737	(t2 & BIT(CLEAR_NAK_OUT_PACKETS_MODE))
1738	? "NAKmode " : "",
1739	(t2 & BIT(CLEAR_ENDPOINT_TOGGLE))
1740	? "DATA1 " : "DATA0 ",
1741	(t2 & BIT(CLEAR_ENDPOINT_HALT))
1742	? "HALT " : "",
1743	readl(addr: &ep->regs->ep_irqenb));
1744	size -= t;
1745	next += t;
1746
1747	t = scnprintf(buf: next, size,
1748	fmt: "\tstat %08x avail %04x "
1749	"(ep%d%s-%s)%s\n",
1750	readl(addr: &ep->regs->ep_stat),
1751	readl(addr: &ep->regs->ep_avail),
1752	t1 & 0x0f, DIR_STRING(t1),
1753	type_string(bmAttributes: t1 >> 8),
1754	ep->stopped ? "*" : "");
1755	size -= t;
1756	next += t;
1757
1758	if (!ep->dma)
1759	continue;
1760
1761	t = scnprintf(buf: next, size,
1762	fmt: " dma\tctl %08x stat %08x count %08x\n"
1763	"\taddr %08x desc %08x\n",
1764	readl(addr: &ep->dma->dmactl),
1765	readl(addr: &ep->dma->dmastat),
1766	readl(addr: &ep->dma->dmacount),
1767	readl(addr: &ep->dma->dmaaddr),
1768	readl(addr: &ep->dma->dmadesc));
1769	size -= t;
1770	next += t;
1771
1772	}
1773
1774	/* Indexed Registers (none yet) */
1775
1776	/* Statistics */
1777	t = scnprintf(buf: next, size, fmt: "\nirqs: ");
1778	size -= t;
1779	next += t;
1780	for (i = 0; i < dev->n_ep; i++) {
1781	struct net2280_ep *ep;
1782
1783	ep = &dev->ep[i];
1784	if (i && !ep->irqs)
1785	continue;
1786	t = scnprintf(buf: next, size, fmt: " %s/%lu", ep->ep.name, ep->irqs);
1787	size -= t;
1788	next += t;
1789
1790	}
1791	t = scnprintf(buf: next, size, fmt: "\n");
1792	size -= t;
1793	next += t;
1794
1795	spin_unlock_irqrestore(lock: &dev->lock, flags);
1796
1797	return PAGE_SIZE - size;
1798	}
1799	static DEVICE_ATTR_RO(registers);
1800
1801	static ssize_t queues_show(struct device *_dev, struct device_attribute *attr,
1802	char *buf)
1803	{
1804	struct net2280 *dev;
1805	char *next;
1806	unsigned size;
1807	unsigned long flags;
1808	int i;
1809
1810	dev = dev_get_drvdata(dev: _dev);
1811	next = buf;
1812	size = PAGE_SIZE;
1813	spin_lock_irqsave(&dev->lock, flags);
1814
1815	for (i = 0; i < dev->n_ep; i++) {
1816	struct net2280_ep *ep = &dev->ep[i];
1817	struct net2280_request *req;
1818	int t;
1819
1820	if (i != 0) {
1821	const struct usb_endpoint_descriptor *d;
1822
1823	d = ep->desc;
1824	if (!d)
1825	continue;
1826	t = d->bEndpointAddress;
1827	t = scnprintf(buf: next, size,
1828	fmt: "\n%s (ep%d%s-%s) max %04x %s fifo %d\n",
1829	ep->ep.name, t & USB_ENDPOINT_NUMBER_MASK,
1830	(t & USB_DIR_IN) ? "in" : "out",
1831	type_string(bmAttributes: d->bmAttributes),
1832	usb_endpoint_maxp(epd: d),
1833	ep->dma ? "dma" : "pio", ep->fifo_size
1834	);
1835	} else /* ep0 should only have one transfer queued */
1836	t = scnprintf(buf: next, size, fmt: "ep0 max 64 pio %s\n",
1837	ep->is_in ? "in" : "out");
1838	if (t <= 0 || t > size)
1839	goto done;
1840	size -= t;
1841	next += t;
1842
1843	if (list_empty(head: &ep->queue)) {
1844	t = scnprintf(buf: next, size, fmt: "\t(nothing queued)\n");
1845	if (t <= 0 || t > size)
1846	goto done;
1847	size -= t;
1848	next += t;
1849	continue;
1850	}
1851	list_for_each_entry(req, &ep->queue, queue) {
1852	if (ep->dma && req->td_dma == readl(addr: &ep->dma->dmadesc))
1853	t = scnprintf(buf: next, size,
1854	fmt: "\treq %p len %d/%d "
1855	"buf %p (dmacount %08x)\n",
1856	&req->req, req->req.actual,
1857	req->req.length, req->req.buf,
1858	readl(addr: &ep->dma->dmacount));
1859	else
1860	t = scnprintf(buf: next, size,
1861	fmt: "\treq %p len %d/%d buf %p\n",
1862	&req->req, req->req.actual,
1863	req->req.length, req->req.buf);
1864	if (t <= 0 || t > size)
1865	goto done;
1866	size -= t;
1867	next += t;
1868
1869	if (ep->dma) {
1870	struct net2280_dma *td;
1871
1872	td = req->td;
1873	t = scnprintf(buf: next, size, fmt: "\t td %08x "
1874	" count %08x buf %08x desc %08x\n",
1875	(u32) req->td_dma,
1876	le32_to_cpu(td->dmacount),
1877	le32_to_cpu(td->dmaaddr),
1878	le32_to_cpu(td->dmadesc));
1879	if (t <= 0 || t > size)
1880	goto done;
1881	size -= t;
1882	next += t;
1883	}
1884	}
1885	}
1886
1887	done:
1888	spin_unlock_irqrestore(lock: &dev->lock, flags);
1889	return PAGE_SIZE - size;
1890	}
1891	static DEVICE_ATTR_RO(queues);
1892
1893
1894	#else
1895
1896	#define device_create_file(a, b) (0)
1897	#define device_remove_file(a, b) do { } while (0)
1898
1899	#endif
1900
1901	/*-------------------------------------------------------------------------*/
1902
1903	/* another driver-specific mode might be a request type doing dma
1904	* to/from another device fifo instead of to/from memory.
1905	*/
1906
1907	static void set_fifo_mode(struct net2280 *dev, int mode)
1908	{
1909	/* keeping high bits preserves BAR2 */
1910	writel(val: (0xffff << PCI_BASE2_RANGE) | mode, addr: &dev->regs->fifoctl);
1911
1912	/* always ep-{a,b,e,f} ... maybe not ep-c or ep-d */
1913	INIT_LIST_HEAD(list: &dev->gadget.ep_list);
1914	list_add_tail(new: &dev->ep[1].ep.ep_list, head: &dev->gadget.ep_list);
1915	list_add_tail(new: &dev->ep[2].ep.ep_list, head: &dev->gadget.ep_list);
1916	switch (mode) {
1917	case 0:
1918	list_add_tail(new: &dev->ep[3].ep.ep_list, head: &dev->gadget.ep_list);
1919	list_add_tail(new: &dev->ep[4].ep.ep_list, head: &dev->gadget.ep_list);
1920	dev->ep[1].fifo_size = dev->ep[2].fifo_size = 1024;
1921	break;
1922	case 1:
1923	dev->ep[1].fifo_size = dev->ep[2].fifo_size = 2048;
1924	break;
1925	case 2:
1926	list_add_tail(new: &dev->ep[3].ep.ep_list, head: &dev->gadget.ep_list);
1927	dev->ep[1].fifo_size = 2048;
1928	dev->ep[2].fifo_size = 1024;
1929	break;
1930	}
1931	/* fifo sizes for ep0, ep-c, ep-d, ep-e, and ep-f never change */
1932	list_add_tail(new: &dev->ep[5].ep.ep_list, head: &dev->gadget.ep_list);
1933	list_add_tail(new: &dev->ep[6].ep.ep_list, head: &dev->gadget.ep_list);
1934	}
1935
1936	static void defect7374_disable_data_eps(struct net2280 *dev)
1937	{
1938	/*
1939	* For Defect 7374, disable data EPs (and more):
1940	* - This phase undoes the earlier phase of the Defect 7374 workaround,
1941	* returing ep regs back to normal.
1942	*/
1943	struct net2280_ep *ep;
1944	int i;
1945	unsigned char ep_sel;
1946	u32 tmp_reg;
1947
1948	for (i = 1; i < 5; i++) {
1949	ep = &dev->ep[i];
1950	writel(val: i, addr: &ep->cfg->ep_cfg);
1951	}
1952
1953	/* CSROUT, CSRIN, PCIOUT, PCIIN, STATIN, RCIN */
1954	for (i = 0; i < 6; i++)
1955	writel(val: 0, addr: &dev->dep[i].dep_cfg);
1956
1957	for (ep_sel = 0; ep_sel <= 21; ep_sel++) {
1958	/* Select an endpoint for subsequent operations: */
1959	tmp_reg = readl(addr: &dev->plregs->pl_ep_ctrl);
1960	writel(val: ((tmp_reg & ~0x1f) | ep_sel), addr: &dev->plregs->pl_ep_ctrl);
1961
1962	if (ep_sel < 2 || (ep_sel > 9 && ep_sel < 14) ||
1963	ep_sel == 18 || ep_sel == 20)
1964	continue;
1965
1966	/* Change settings on some selected endpoints */
1967	tmp_reg = readl(addr: &dev->plregs->pl_ep_cfg_4);
1968	tmp_reg &= ~BIT(NON_CTRL_IN_TOLERATE_BAD_DIR);
1969	writel(val: tmp_reg, addr: &dev->plregs->pl_ep_cfg_4);
1970	tmp_reg = readl(addr: &dev->plregs->pl_ep_ctrl);
1971	tmp_reg |= BIT(EP_INITIALIZED);
1972	writel(val: tmp_reg, addr: &dev->plregs->pl_ep_ctrl);
1973	}
1974	}
1975
1976	static void defect7374_enable_data_eps_zero(struct net2280 *dev)
1977	{
1978	u32 tmp = 0, tmp_reg;
1979	u32 scratch;
1980	int i;
1981	unsigned char ep_sel;
1982
1983	scratch = get_idx_reg(regs: dev->regs, SCRATCH);
1984
1985	WARN_ON((scratch & (0xf << DEFECT7374_FSM_FIELD))
1986	== DEFECT7374_FSM_SS_CONTROL_READ);
1987
1988	scratch &= ~(0xf << DEFECT7374_FSM_FIELD);
1989
1990	ep_warn(dev, "Operate Defect 7374 workaround soft this time");
1991	ep_warn(dev, "It will operate on cold-reboot and SS connect");
1992
1993	/*GPEPs:*/
1994	tmp = ((0 << ENDPOINT_NUMBER) | BIT(ENDPOINT_DIRECTION) |
1995	(2 << OUT_ENDPOINT_TYPE) | (2 << IN_ENDPOINT_TYPE) |
1996	((dev->enhanced_mode) ?
1997	BIT(OUT_ENDPOINT_ENABLE) | BIT(IN_ENDPOINT_ENABLE) :
1998	BIT(ENDPOINT_ENABLE)));
1999
2000	for (i = 1; i < 5; i++)
2001	writel(val: tmp, addr: &dev->ep[i].cfg->ep_cfg);
2002
2003	/* CSRIN, PCIIN, STATIN, RCIN*/
2004	tmp = ((0 << ENDPOINT_NUMBER) | BIT(ENDPOINT_ENABLE));
2005	writel(val: tmp, addr: &dev->dep[1].dep_cfg);
2006	writel(val: tmp, addr: &dev->dep[3].dep_cfg);
2007	writel(val: tmp, addr: &dev->dep[4].dep_cfg);
2008	writel(val: tmp, addr: &dev->dep[5].dep_cfg);
2009
2010	/*Implemented for development and debug.
2011	* Can be refined/tuned later.*/
2012	for (ep_sel = 0; ep_sel <= 21; ep_sel++) {
2013	/* Select an endpoint for subsequent operations: */
2014	tmp_reg = readl(addr: &dev->plregs->pl_ep_ctrl);
2015	writel(val: ((tmp_reg & ~0x1f) | ep_sel),
2016	addr: &dev->plregs->pl_ep_ctrl);
2017
2018	if (ep_sel == 1) {
2019	tmp =
2020	(readl(addr: &dev->plregs->pl_ep_ctrl) |
2021	BIT(CLEAR_ACK_ERROR_CODE) | 0);
2022	writel(val: tmp, addr: &dev->plregs->pl_ep_ctrl);
2023	continue;
2024	}
2025
2026	if (ep_sel == 0 || (ep_sel > 9 && ep_sel < 14) ||
2027	ep_sel == 18 || ep_sel == 20)
2028	continue;
2029
2030	tmp = (readl(addr: &dev->plregs->pl_ep_cfg_4) |
2031	BIT(NON_CTRL_IN_TOLERATE_BAD_DIR) | 0);
2032	writel(val: tmp, addr: &dev->plregs->pl_ep_cfg_4);
2033
2034	tmp = readl(addr: &dev->plregs->pl_ep_ctrl) &
2035	~BIT(EP_INITIALIZED);
2036	writel(val: tmp, addr: &dev->plregs->pl_ep_ctrl);
2037
2038	}
2039
2040	/* Set FSM to focus on the first Control Read:
2041	* - Tip: Connection speed is known upon the first
2042	* setup request.*/
2043	scratch |= DEFECT7374_FSM_WAITING_FOR_CONTROL_READ;
2044	set_idx_reg(regs: dev->regs, SCRATCH, value: scratch);
2045
2046	}
2047
2048	/* keeping it simple:
2049	* - one bus driver, initted first;
2050	* - one function driver, initted second
2051	*
2052	* most of the work to support multiple net2280 controllers would
2053	* be to associate this gadget driver (yes?) with all of them, or
2054	* perhaps to bind specific drivers to specific devices.
2055	*/
2056
2057	static void usb_reset_228x(struct net2280 *dev)
2058	{
2059	u32 tmp;
2060
2061	dev->gadget.speed = USB_SPEED_UNKNOWN;
2062	(void) readl(addr: &dev->usb->usbctl);
2063
2064	net2280_led_init(dev);
2065
2066	/* disable automatic responses, and irqs */
2067	writel(val: 0, addr: &dev->usb->stdrsp);
2068	writel(val: 0, addr: &dev->regs->pciirqenb0);
2069	writel(val: 0, addr: &dev->regs->pciirqenb1);
2070
2071	/* clear old dma and irq state */
2072	for (tmp = 0; tmp < 4; tmp++) {
2073	struct net2280_ep *ep = &dev->ep[tmp + 1];
2074	if (ep->dma)
2075	abort_dma(ep);
2076	}
2077
2078	writel(val: ~0, addr: &dev->regs->irqstat0),
2079	writel(val: ~(u32)BIT(SUSPEND_REQUEST_INTERRUPT), addr: &dev->regs->irqstat1),
2080
2081	/* reset, and enable pci */
2082	tmp = readl(addr: &dev->regs->devinit) |
2083	BIT(PCI_ENABLE) |
2084	BIT(FIFO_SOFT_RESET) |
2085	BIT(USB_SOFT_RESET) |
2086	BIT(M8051_RESET);
2087	writel(val: tmp, addr: &dev->regs->devinit);
2088
2089	/* standard fifo and endpoint allocations */
2090	set_fifo_mode(dev, mode: (fifo_mode <= 2) ? fifo_mode : 0);
2091	}
2092
2093	static void usb_reset_338x(struct net2280 *dev)
2094	{
2095	u32 tmp;
2096
2097	dev->gadget.speed = USB_SPEED_UNKNOWN;
2098	(void)readl(addr: &dev->usb->usbctl);
2099
2100	net2280_led_init(dev);
2101
2102	if (dev->bug7734_patched) {
2103	/* disable automatic responses, and irqs */
2104	writel(val: 0, addr: &dev->usb->stdrsp);
2105	writel(val: 0, addr: &dev->regs->pciirqenb0);
2106	writel(val: 0, addr: &dev->regs->pciirqenb1);
2107	}
2108
2109	/* clear old dma and irq state */
2110	for (tmp = 0; tmp < 4; tmp++) {
2111	struct net2280_ep *ep = &dev->ep[tmp + 1];
2112	struct net2280_dma_regs __iomem *dma;
2113
2114	if (ep->dma) {
2115	abort_dma(ep);
2116	} else {
2117	dma = &dev->dma[tmp];
2118	writel(BIT(DMA_ABORT), addr: &dma->dmastat);
2119	writel(val: 0, addr: &dma->dmactl);
2120	}
2121	}
2122
2123	writel(val: ~0, addr: &dev->regs->irqstat0), writel(val: ~0, addr: &dev->regs->irqstat1);
2124
2125	if (dev->bug7734_patched) {
2126	/* reset, and enable pci */
2127	tmp = readl(addr: &dev->regs->devinit) |
2128	BIT(PCI_ENABLE) |
2129	BIT(FIFO_SOFT_RESET) |
2130	BIT(USB_SOFT_RESET) |
2131	BIT(M8051_RESET);
2132
2133	writel(val: tmp, addr: &dev->regs->devinit);
2134	}
2135
2136	/* always ep-{1,2,3,4} ... maybe not ep-3 or ep-4 */
2137	INIT_LIST_HEAD(list: &dev->gadget.ep_list);
2138
2139	for (tmp = 1; tmp < dev->n_ep; tmp++)
2140	list_add_tail(new: &dev->ep[tmp].ep.ep_list, head: &dev->gadget.ep_list);
2141
2142	}
2143
2144	static void usb_reset(struct net2280 *dev)
2145	{
2146	if (dev->quirks & PLX_LEGACY)
2147	return usb_reset_228x(dev);
2148	return usb_reset_338x(dev);
2149	}
2150
2151	static void usb_reinit_228x(struct net2280 *dev)
2152	{
2153	u32 tmp;
2154
2155	/* basic endpoint init */
2156	for (tmp = 0; tmp < 7; tmp++) {
2157	struct net2280_ep *ep = &dev->ep[tmp];
2158
2159	ep->ep.name = ep_info_dft[tmp].name;
2160	ep->ep.caps = ep_info_dft[tmp].caps;
2161	ep->dev = dev;
2162	ep->num = tmp;
2163
2164	if (tmp > 0 && tmp <= 4) {
2165	ep->fifo_size = 1024;
2166	ep->dma = &dev->dma[tmp - 1];
2167	} else
2168	ep->fifo_size = 64;
2169	ep->regs = &dev->epregs[tmp];
2170	ep->cfg = &dev->epregs[tmp];
2171	ep_reset_228x(regs: dev->regs, ep);
2172	}
2173	usb_ep_set_maxpacket_limit(ep: &dev->ep[0].ep, maxpacket_limit: 64);
2174	usb_ep_set_maxpacket_limit(ep: &dev->ep[5].ep, maxpacket_limit: 64);
2175	usb_ep_set_maxpacket_limit(ep: &dev->ep[6].ep, maxpacket_limit: 64);
2176
2177	dev->gadget.ep0 = &dev->ep[0].ep;
2178	dev->ep[0].stopped = 0;
2179	INIT_LIST_HEAD(list: &dev->gadget.ep0->ep_list);
2180
2181	/* we want to prevent lowlevel/insecure access from the USB host,
2182	* but erratum 0119 means this enable bit is ignored
2183	*/
2184	for (tmp = 0; tmp < 5; tmp++)
2185	writel(EP_DONTUSE, addr: &dev->dep[tmp].dep_cfg);
2186	}
2187
2188	static void usb_reinit_338x(struct net2280 *dev)
2189	{
2190	int i;
2191	u32 tmp, val;
2192	static const u32 ne[9] = { 0, 1, 2, 3, 4, 1, 2, 3, 4 };
2193	static const u32 ep_reg_addr[9] = { 0x00, 0xC0, 0x00, 0xC0, 0x00,
2194	0x00, 0xC0, 0x00, 0xC0 };
2195
2196	/* basic endpoint init */
2197	for (i = 0; i < dev->n_ep; i++) {
2198	struct net2280_ep *ep = &dev->ep[i];
2199
2200	ep->ep.name = dev->enhanced_mode ? ep_info_adv[i].name :
2201	ep_info_dft[i].name;
2202	ep->ep.caps = dev->enhanced_mode ? ep_info_adv[i].caps :
2203	ep_info_dft[i].caps;
2204	ep->dev = dev;
2205	ep->num = i;
2206
2207	if (i > 0 && i <= 4)
2208	ep->dma = &dev->dma[i - 1];
2209
2210	if (dev->enhanced_mode) {
2211	ep->cfg = &dev->epregs[ne[i]];
2212	/*
2213	* Set USB endpoint number, hardware allows same number
2214	* in both directions.
2215	*/
2216	if (i > 0 && i < 5)
2217	writel(val: ne[i], addr: &ep->cfg->ep_cfg);
2218	ep->regs = (struct net2280_ep_regs __iomem *)
2219	(((void __iomem *)&dev->epregs[ne[i]]) +
2220	ep_reg_addr[i]);
2221	} else {
2222	ep->cfg = &dev->epregs[i];
2223	ep->regs = &dev->epregs[i];
2224	}
2225
2226	ep->fifo_size = (i != 0) ? 2048 : 512;
2227
2228	ep_reset_338x(regs: dev->regs, ep);
2229	}
2230	usb_ep_set_maxpacket_limit(ep: &dev->ep[0].ep, maxpacket_limit: 512);
2231
2232	dev->gadget.ep0 = &dev->ep[0].ep;
2233	dev->ep[0].stopped = 0;
2234
2235	/* Link layer set up */
2236	if (dev->bug7734_patched) {
2237	tmp = readl(addr: &dev->usb_ext->usbctl2) &
2238	~(BIT(U1_ENABLE) | BIT(U2_ENABLE) | BIT(LTM_ENABLE));
2239	writel(val: tmp, addr: &dev->usb_ext->usbctl2);
2240	}
2241
2242	/* Hardware Defect and Workaround */
2243	val = readl(addr: &dev->llregs->ll_lfps_5);
2244	val &= ~(0xf << TIMER_LFPS_6US);
2245	val |= 0x5 << TIMER_LFPS_6US;
2246	writel(val, addr: &dev->llregs->ll_lfps_5);
2247
2248	val = readl(addr: &dev->llregs->ll_lfps_6);
2249	val &= ~(0xffff << TIMER_LFPS_80US);
2250	val |= 0x0100 << TIMER_LFPS_80US;
2251	writel(val, addr: &dev->llregs->ll_lfps_6);
2252
2253	/*
2254	* AA_AB Errata. Issue 4. Workaround for SuperSpeed USB
2255	* Hot Reset Exit Handshake may Fail in Specific Case using
2256	* Default Register Settings. Workaround for Enumeration test.
2257	*/
2258	val = readl(addr: &dev->llregs->ll_tsn_counters_2);
2259	val &= ~(0x1f << HOT_TX_NORESET_TS2);
2260	val |= 0x10 << HOT_TX_NORESET_TS2;
2261	writel(val, addr: &dev->llregs->ll_tsn_counters_2);
2262
2263	val = readl(addr: &dev->llregs->ll_tsn_counters_3);
2264	val &= ~(0x1f << HOT_RX_RESET_TS2);
2265	val |= 0x3 << HOT_RX_RESET_TS2;
2266	writel(val, addr: &dev->llregs->ll_tsn_counters_3);
2267
2268	/*
2269	* AB errata. Errata 11. Workaround for Default Duration of LFPS
2270	* Handshake Signaling for Device-Initiated U1 Exit is too short.
2271	* Without this, various enumeration failures observed with
2272	* modern superspeed hosts.
2273	*/
2274	val = readl(addr: &dev->llregs->ll_lfps_timers_2);
2275	writel(val: (val & 0xffff0000) | LFPS_TIMERS_2_WORKAROUND_VALUE,
2276	addr: &dev->llregs->ll_lfps_timers_2);
2277
2278	/*
2279	* Set Recovery Idle to Recover bit:
2280	* - On SS connections, setting Recovery Idle to Recover Fmw improves
2281	* link robustness with various hosts and hubs.
2282	* - It is safe to set for all connection speeds; all chip revisions.
2283	* - R-M-W to leave other bits undisturbed.
2284	* - Reference PLX TT-7372
2285	*/
2286	val = readl(addr: &dev->llregs->ll_tsn_chicken_bit);
2287	val |= BIT(RECOVERY_IDLE_TO_RECOVER_FMW);
2288	writel(val, addr: &dev->llregs->ll_tsn_chicken_bit);
2289
2290	INIT_LIST_HEAD(list: &dev->gadget.ep0->ep_list);
2291
2292	/* disable dedicated endpoints */
2293	writel(val: 0x0D, addr: &dev->dep[0].dep_cfg);
2294	writel(val: 0x0D, addr: &dev->dep[1].dep_cfg);
2295	writel(val: 0x0E, addr: &dev->dep[2].dep_cfg);
2296	writel(val: 0x0E, addr: &dev->dep[3].dep_cfg);
2297	writel(val: 0x0F, addr: &dev->dep[4].dep_cfg);
2298	writel(val: 0x0C, addr: &dev->dep[5].dep_cfg);
2299	}
2300
2301	static void usb_reinit(struct net2280 *dev)
2302	{
2303	if (dev->quirks & PLX_LEGACY)
2304	return usb_reinit_228x(dev);
2305	return usb_reinit_338x(dev);
2306	}
2307
2308	static void ep0_start_228x(struct net2280 *dev)
2309	{
2310	writel(BIT(CLEAR_EP_HIDE_STATUS_PHASE) |
2311	BIT(CLEAR_NAK_OUT_PACKETS) |
2312	BIT(CLEAR_CONTROL_STATUS_PHASE_HANDSHAKE),
2313	addr: &dev->epregs[0].ep_rsp);
2314
2315	/*
2316	* hardware optionally handles a bunch of standard requests
2317	* that the API hides from drivers anyway. have it do so.
2318	* endpoint status/features are handled in software, to
2319	* help pass tests for some dubious behavior.
2320	*/
2321	writel(BIT(SET_TEST_MODE) |
2322	BIT(SET_ADDRESS) |
2323	BIT(DEVICE_SET_CLEAR_DEVICE_REMOTE_WAKEUP) |
2324	BIT(GET_DEVICE_STATUS) |
2325	BIT(GET_INTERFACE_STATUS),
2326	addr: &dev->usb->stdrsp);
2327	writel(BIT(USB_ROOT_PORT_WAKEUP_ENABLE) |
2328	BIT(SELF_POWERED_USB_DEVICE) |
2329	BIT(REMOTE_WAKEUP_SUPPORT) |
2330	(dev->softconnect << USB_DETECT_ENABLE) |
2331	BIT(SELF_POWERED_STATUS),
2332	addr: &dev->usb->usbctl);
2333
2334	/* enable irqs so we can see ep0 and general operation */
2335	writel(BIT(SETUP_PACKET_INTERRUPT_ENABLE) |
2336	BIT(ENDPOINT_0_INTERRUPT_ENABLE),
2337	addr: &dev->regs->pciirqenb0);
2338	writel(BIT(PCI_INTERRUPT_ENABLE) |
2339	BIT(PCI_MASTER_ABORT_RECEIVED_INTERRUPT_ENABLE) |
2340	BIT(PCI_TARGET_ABORT_RECEIVED_INTERRUPT_ENABLE) |
2341	BIT(PCI_RETRY_ABORT_INTERRUPT_ENABLE) |
2342	BIT(VBUS_INTERRUPT_ENABLE) |
2343	BIT(ROOT_PORT_RESET_INTERRUPT_ENABLE) |
2344	BIT(SUSPEND_REQUEST_CHANGE_INTERRUPT_ENABLE),
2345	addr: &dev->regs->pciirqenb1);
2346
2347	/* don't leave any writes posted */
2348	(void) readl(addr: &dev->usb->usbctl);
2349	}
2350
2351	static void ep0_start_338x(struct net2280 *dev)
2352	{
2353
2354	if (dev->bug7734_patched)
2355	writel(BIT(CLEAR_NAK_OUT_PACKETS_MODE) |
2356	BIT(SET_EP_HIDE_STATUS_PHASE),
2357	addr: &dev->epregs[0].ep_rsp);
2358
2359	/*
2360	* hardware optionally handles a bunch of standard requests
2361	* that the API hides from drivers anyway. have it do so.
2362	* endpoint status/features are handled in software, to
2363	* help pass tests for some dubious behavior.
2364	*/
2365	writel(BIT(SET_ISOCHRONOUS_DELAY) |
2366	BIT(SET_SEL) |
2367	BIT(SET_TEST_MODE) |
2368	BIT(SET_ADDRESS) |
2369	BIT(GET_INTERFACE_STATUS) |
2370	BIT(GET_DEVICE_STATUS),
2371	addr: &dev->usb->stdrsp);
2372	dev->wakeup_enable = 1;
2373	writel(BIT(USB_ROOT_PORT_WAKEUP_ENABLE) |
2374	(dev->softconnect << USB_DETECT_ENABLE) |
2375	BIT(DEVICE_REMOTE_WAKEUP_ENABLE),
2376	addr: &dev->usb->usbctl);
2377
2378	/* enable irqs so we can see ep0 and general operation */
2379	writel(BIT(SETUP_PACKET_INTERRUPT_ENABLE) |
2380	BIT(ENDPOINT_0_INTERRUPT_ENABLE),
2381	addr: &dev->regs->pciirqenb0);
2382	writel(BIT(PCI_INTERRUPT_ENABLE) |
2383	BIT(ROOT_PORT_RESET_INTERRUPT_ENABLE) |
2384	BIT(SUSPEND_REQUEST_CHANGE_INTERRUPT_ENABLE) |
2385	BIT(VBUS_INTERRUPT_ENABLE),
2386	addr: &dev->regs->pciirqenb1);
2387
2388	/* don't leave any writes posted */
2389	(void)readl(addr: &dev->usb->usbctl);
2390	}
2391
2392	static void ep0_start(struct net2280 *dev)
2393	{
2394	if (dev->quirks & PLX_LEGACY)
2395	return ep0_start_228x(dev);
2396	return ep0_start_338x(dev);
2397	}
2398
2399	/* when a driver is successfully registered, it will receive
2400	* control requests including set_configuration(), which enables
2401	* non-control requests. then usb traffic follows until a
2402	* disconnect is reported. then a host may connect again, or
2403	* the driver might get unbound.
2404	*/
2405	static int net2280_start(struct usb_gadget *_gadget,
2406	struct usb_gadget_driver *driver)
2407	{
2408	struct net2280 *dev;
2409	int retval;
2410	unsigned i;
2411
2412	/* insist on high speed support from the driver, since
2413	* (dev->usb->xcvrdiag & FORCE_FULL_SPEED_MODE)
2414	* "must not be used in normal operation"
2415	*/
2416	if (!driver || driver->max_speed < USB_SPEED_HIGH ||
2417	!driver->setup)
2418	return -EINVAL;
2419
2420	dev = container_of(_gadget, struct net2280, gadget);
2421
2422	for (i = 0; i < dev->n_ep; i++)
2423	dev->ep[i].irqs = 0;
2424
2425	/* hook up the driver ... */
2426	dev->driver = driver;
2427
2428	retval = device_create_file(device: &dev->pdev->dev, entry: &dev_attr_function);
2429	if (retval)
2430	goto err_unbind;
2431	retval = device_create_file(device: &dev->pdev->dev, entry: &dev_attr_queues);
2432	if (retval)
2433	goto err_func;
2434
2435	/* enable host detection and ep0; and we're ready
2436	* for set_configuration as well as eventual disconnect.
2437	*/
2438	net2280_led_active(dev, is_active: 1);
2439
2440	if ((dev->quirks & PLX_PCIE) && !dev->bug7734_patched)
2441	defect7374_enable_data_eps_zero(dev);
2442
2443	ep0_start(dev);
2444
2445	/* pci writes may still be posted */
2446	return 0;
2447
2448	err_func:
2449	device_remove_file(dev: &dev->pdev->dev, attr: &dev_attr_function);
2450	err_unbind:
2451	dev->driver = NULL;
2452	return retval;
2453	}
2454
2455	static void stop_activity(struct net2280 *dev, struct usb_gadget_driver *driver)
2456	{
2457	int i;
2458
2459	/* don't disconnect if it's not connected */
2460	if (dev->gadget.speed == USB_SPEED_UNKNOWN)
2461	driver = NULL;
2462
2463	/* stop hardware; prevent new request submissions;
2464	* and kill any outstanding requests.
2465	*/
2466	usb_reset(dev);
2467	for (i = 0; i < dev->n_ep; i++)
2468	nuke(ep: &dev->ep[i]);
2469
2470	/* report disconnect; the driver is already quiesced */
2471	if (dev->async_callbacks && driver) {
2472	spin_unlock(lock: &dev->lock);
2473	driver->disconnect(&dev->gadget);
2474	spin_lock(lock: &dev->lock);
2475	}
2476
2477	usb_reinit(dev);
2478	}
2479
2480	static int net2280_stop(struct usb_gadget *_gadget)
2481	{
2482	struct net2280 *dev;
2483	unsigned long flags;
2484
2485	dev = container_of(_gadget, struct net2280, gadget);
2486
2487	spin_lock_irqsave(&dev->lock, flags);
2488	stop_activity(dev, NULL);
2489	spin_unlock_irqrestore(lock: &dev->lock, flags);
2490
2491	net2280_led_active(dev, is_active: 0);
2492
2493	device_remove_file(dev: &dev->pdev->dev, attr: &dev_attr_function);
2494	device_remove_file(dev: &dev->pdev->dev, attr: &dev_attr_queues);
2495
2496	dev->driver = NULL;
2497
2498	return 0;
2499	}
2500
2501	static void net2280_async_callbacks(struct usb_gadget *_gadget, bool enable)
2502	{
2503	struct net2280 *dev = container_of(_gadget, struct net2280, gadget);
2504
2505	spin_lock_irq(lock: &dev->lock);
2506	dev->async_callbacks = enable;
2507	spin_unlock_irq(lock: &dev->lock);
2508	}
2509
2510	/*-------------------------------------------------------------------------*/
2511
2512	/* handle ep0, ep-e, ep-f with 64 byte packets: packet per irq.
2513	* also works for dma-capable endpoints, in pio mode or just
2514	* to manually advance the queue after short OUT transfers.
2515	*/
2516	static void handle_ep_small(struct net2280_ep *ep)
2517	{
2518	struct net2280_request *req;
2519	u32 t;
2520	/* 0 error, 1 mid-data, 2 done */
2521	int mode = 1;
2522
2523	if (!list_empty(head: &ep->queue))
2524	req = list_entry(ep->queue.next,
2525	struct net2280_request, queue);
2526	else
2527	req = NULL;
2528
2529	/* ack all, and handle what we care about */
2530	t = readl(addr: &ep->regs->ep_stat);
2531	ep->irqs++;
2532
2533	ep_vdbg(ep->dev, "%s ack ep_stat %08x, req %p\n",
2534	ep->ep.name, t, req ? &req->req : NULL);
2535
2536	if (!ep->is_in || (ep->dev->quirks & PLX_2280))
2537	writel(val: t & ~BIT(NAK_OUT_PACKETS), addr: &ep->regs->ep_stat);
2538	else
2539	/* Added for 2282 */
2540	writel(val: t, addr: &ep->regs->ep_stat);
2541
2542	/* for ep0, monitor token irqs to catch data stage length errors
2543	* and to synchronize on status.
2544	*
2545	* also, to defer reporting of protocol stalls ... here's where
2546	* data or status first appears, handling stalls here should never
2547	* cause trouble on the host side..
2548	*
2549	* control requests could be slightly faster without token synch for
2550	* status, but status can jam up that way.
2551	*/
2552	if (unlikely(ep->num == 0)) {
2553	if (ep->is_in) {
2554	/* status; stop NAKing */
2555	if (t & BIT(DATA_OUT_PING_TOKEN_INTERRUPT)) {
2556	if (ep->dev->protocol_stall) {
2557	ep->stopped = 1;
2558	set_halt(ep);
2559	}
2560	if (!req)
2561	allow_status(ep);
2562	mode = 2;
2563	/* reply to extra IN data tokens with a zlp */
2564	} else if (t & BIT(DATA_IN_TOKEN_INTERRUPT)) {
2565	if (ep->dev->protocol_stall) {
2566	ep->stopped = 1;
2567	set_halt(ep);
2568	mode = 2;
2569	} else if (ep->responded &&
2570	!req && !ep->stopped)
2571	write_fifo(ep, NULL);
2572	}
2573	} else {
2574	/* status; stop NAKing */
2575	if (t & BIT(DATA_IN_TOKEN_INTERRUPT)) {
2576	if (ep->dev->protocol_stall) {
2577	ep->stopped = 1;
2578	set_halt(ep);
2579	}
2580	mode = 2;
2581	/* an extra OUT token is an error */
2582	} else if (((t & BIT(DATA_OUT_PING_TOKEN_INTERRUPT)) &&
2583	req &&
2584	req->req.actual == req->req.length) ||
2585	(ep->responded && !req)) {
2586	ep->dev->protocol_stall = 1;
2587	set_halt(ep);
2588	ep->stopped = 1;
2589	if (req)
2590	done(ep, req, status: -EOVERFLOW);
2591	req = NULL;
2592	}
2593	}
2594	}
2595
2596	if (unlikely(!req))
2597	return;
2598
2599	/* manual DMA queue advance after short OUT */
2600	if (likely(ep->dma)) {
2601	if (t & BIT(SHORT_PACKET_TRANSFERRED_INTERRUPT)) {
2602	struct net2280_request *stuck_req = NULL;
2603	int stopped = ep->stopped;
2604	int num_completed;
2605	int stuck = 0;
2606	u32 count;
2607
2608	/* TRANSFERRED works around OUT_DONE erratum 0112.
2609	* we expect (N <= maxpacket) bytes; host wrote M.
2610	* iff (M < N) we won't ever see a DMA interrupt.
2611	*/
2612	ep->stopped = 1;
2613	for (count = 0; ; t = readl(addr: &ep->regs->ep_stat)) {
2614
2615	/* any preceding dma transfers must finish.
2616	* dma handles (M >= N), may empty the queue
2617	*/
2618	num_completed = scan_dma_completions(ep);
2619	if (unlikely(list_empty(&ep->queue) ||
2620	ep->out_overflow)) {
2621	req = NULL;
2622	break;
2623	}
2624	req = list_entry(ep->queue.next,
2625	struct net2280_request, queue);
2626
2627	/* here either (M < N), a "real" short rx;
2628	* or (M == N) and the queue didn't empty
2629	*/
2630	if (likely(t & BIT(FIFO_EMPTY))) {
2631	count = readl(addr: &ep->dma->dmacount);
2632	count &= DMA_BYTE_COUNT_MASK;
2633	if (readl(addr: &ep->dma->dmadesc)
2634	!= req->td_dma)
2635	req = NULL;
2636	break;
2637	}
2638
2639	/* Escape loop if no dma transfers completed
2640	* after few retries.
2641	*/
2642	if (num_completed == 0) {
2643	if (stuck_req == req &&
2644	readl(addr: &ep->dma->dmadesc) !=
2645	req->td_dma && stuck++ > 5) {
2646	count = readl(
2647	addr: &ep->dma->dmacount);
2648	count &= DMA_BYTE_COUNT_MASK;
2649	req = NULL;
2650	ep_dbg(ep->dev, "%s escape stuck %d, count %u\n",
2651	ep->ep.name, stuck,
2652	count);
2653	break;
2654	} else if (stuck_req != req) {
2655	stuck_req = req;
2656	stuck = 0;
2657	}
2658	} else {
2659	stuck_req = NULL;
2660	stuck = 0;
2661	}
2662
2663	udelay(1);
2664	}
2665
2666	/* stop DMA, leave ep NAKing */
2667	writel(BIT(DMA_ABORT), addr: &ep->dma->dmastat);
2668	spin_stop_dma(dma: ep->dma);
2669
2670	if (likely(req)) {
2671	req->td->dmacount = 0;
2672	t = readl(addr: &ep->regs->ep_avail);
2673	dma_done(ep, req, dmacount: count,
2674	status: (ep->out_overflow || t)
2675	? -EOVERFLOW : 0);
2676	}
2677
2678	/* also flush to prevent erratum 0106 trouble */
2679	if (unlikely(ep->out_overflow ||
2680	(ep->dev->chiprev == 0x0100 &&
2681	ep->dev->gadget.speed
2682	== USB_SPEED_FULL))) {
2683	out_flush(ep);
2684	ep->out_overflow = 0;
2685	}
2686
2687	/* (re)start dma if needed, stop NAKing */
2688	ep->stopped = stopped;
2689	if (!list_empty(head: &ep->queue))
2690	restart_dma(ep);
2691	} else
2692	ep_dbg(ep->dev, "%s dma ep_stat %08x ??\n",
2693	ep->ep.name, t);
2694	return;
2695
2696	/* data packet(s) received (in the fifo, OUT) */
2697	} else if (t & BIT(DATA_PACKET_RECEIVED_INTERRUPT)) {
2698	if (read_fifo(ep, req) && ep->num != 0)
2699	mode = 2;
2700
2701	/* data packet(s) transmitted (IN) */
2702	} else if (t & BIT(DATA_PACKET_TRANSMITTED_INTERRUPT)) {
2703	unsigned len;
2704
2705	len = req->req.length - req->req.actual;
2706	if (len > ep->ep.maxpacket)
2707	len = ep->ep.maxpacket;
2708	req->req.actual += len;
2709
2710	/* if we wrote it all, we're usually done */
2711	/* send zlps until the status stage */
2712	if ((req->req.actual == req->req.length) &&
2713	(!req->req.zero || len != ep->ep.maxpacket) && ep->num)
2714	mode = 2;
2715
2716	/* there was nothing to do ... */
2717	} else if (mode == 1)
2718	return;
2719
2720	/* done */
2721	if (mode == 2) {
2722	/* stream endpoints often resubmit/unlink in completion */
2723	done(ep, req, status: 0);
2724
2725	/* maybe advance queue to next request */
2726	if (ep->num == 0) {
2727	/* NOTE: net2280 could let gadget driver start the
2728	* status stage later. since not all controllers let
2729	* them control that, the api doesn't (yet) allow it.
2730	*/
2731	if (!ep->stopped)
2732	allow_status(ep);
2733	req = NULL;
2734	} else {
2735	if (!list_empty(head: &ep->queue) && !ep->stopped)
2736	req = list_entry(ep->queue.next,
2737	struct net2280_request, queue);
2738	else
2739	req = NULL;
2740	if (req && !ep->is_in)
2741	stop_out_naking(ep);
2742	}
2743	}
2744
2745	/* is there a buffer for the next packet?
2746	* for best streaming performance, make sure there is one.
2747	*/
2748	if (req && !ep->stopped) {
2749
2750	/* load IN fifo with next packet (may be zlp) */
2751	if (t & BIT(DATA_PACKET_TRANSMITTED_INTERRUPT))
2752	write_fifo(ep, req: &req->req);
2753	}
2754	}
2755
2756	static struct net2280_ep *get_ep_by_addr(struct net2280 *dev, u16 wIndex)
2757	{
2758	struct net2280_ep *ep;
2759
2760	if ((wIndex & USB_ENDPOINT_NUMBER_MASK) == 0)
2761	return &dev->ep[0];
2762	list_for_each_entry(ep, &dev->gadget.ep_list, ep.ep_list) {
2763	u8 bEndpointAddress;
2764
2765	if (!ep->desc)
2766	continue;
2767	bEndpointAddress = ep->desc->bEndpointAddress;
2768	if ((wIndex ^ bEndpointAddress) & USB_DIR_IN)
2769	continue;
2770	if ((wIndex & 0x0f) == (bEndpointAddress & 0x0f))
2771	return ep;
2772	}
2773	return NULL;
2774	}
2775
2776	static void defect7374_workaround(struct net2280 *dev, struct usb_ctrlrequest r)
2777	{
2778	u32 scratch, fsmvalue;
2779	u32 ack_wait_timeout, state;
2780
2781	/* Workaround for Defect 7374 (U1/U2 erroneously rejected): */
2782	scratch = get_idx_reg(regs: dev->regs, SCRATCH);
2783	fsmvalue = scratch & (0xf << DEFECT7374_FSM_FIELD);
2784	scratch &= ~(0xf << DEFECT7374_FSM_FIELD);
2785
2786	if (!((fsmvalue == DEFECT7374_FSM_WAITING_FOR_CONTROL_READ) &&
2787	(r.bRequestType & USB_DIR_IN)))
2788	return;
2789
2790	/* This is the first Control Read for this connection: */
2791	if (!(readl(addr: &dev->usb->usbstat) & BIT(SUPER_SPEED_MODE))) {
2792	/*
2793	* Connection is NOT SS:
2794	* - Connection must be FS or HS.
2795	* - This FSM state should allow workaround software to
2796	* run after the next USB connection.
2797	*/
2798	scratch |= DEFECT7374_FSM_NON_SS_CONTROL_READ;
2799	dev->bug7734_patched = 1;
2800	goto restore_data_eps;
2801	}
2802
2803	/* Connection is SS: */
2804	for (ack_wait_timeout = 0;
2805	ack_wait_timeout < DEFECT_7374_NUMBEROF_MAX_WAIT_LOOPS;
2806	ack_wait_timeout++) {
2807
2808	state = readl(addr: &dev->plregs->pl_ep_status_1)
2809	& (0xff << STATE);
2810	if ((state >= (ACK_GOOD_NORMAL << STATE)) &&
2811	(state <= (ACK_GOOD_MORE_ACKS_TO_COME << STATE))) {
2812	scratch |= DEFECT7374_FSM_SS_CONTROL_READ;
2813	dev->bug7734_patched = 1;
2814	break;
2815	}
2816
2817	/*
2818	* We have not yet received host's Data Phase ACK
2819	* - Wait and try again.
2820	*/
2821	udelay(DEFECT_7374_PROCESSOR_WAIT_TIME);
2822	}
2823
2824
2825	if (ack_wait_timeout >= DEFECT_7374_NUMBEROF_MAX_WAIT_LOOPS) {
2826	ep_err(dev, "FAIL: Defect 7374 workaround waited but failed "
2827	"to detect SS host's data phase ACK.");
2828	ep_err(dev, "PL_EP_STATUS_1(23:16):.Expected from 0x11 to 0x16"
2829	"got 0x%2.2x.\n", state >> STATE);
2830	} else {
2831	ep_warn(dev, "INFO: Defect 7374 workaround waited about\n"
2832	"%duSec for Control Read Data Phase ACK\n",
2833	DEFECT_7374_PROCESSOR_WAIT_TIME * ack_wait_timeout);
2834	}
2835
2836	restore_data_eps:
2837	/*
2838	* Restore data EPs to their pre-workaround settings (disabled,
2839	* initialized, and other details).
2840	*/
2841	defect7374_disable_data_eps(dev);
2842
2843	set_idx_reg(regs: dev->regs, SCRATCH, value: scratch);
2844
2845	return;
2846	}
2847
2848	static void ep_clear_seqnum(struct net2280_ep *ep)
2849	{
2850	struct net2280 *dev = ep->dev;
2851	u32 val;
2852	static const u32 ep_pl[9] = { 0, 3, 4, 7, 8, 2, 5, 6, 9 };
2853
2854	val = readl(addr: &dev->plregs->pl_ep_ctrl) & ~0x1f;
2855	val |= ep_pl[ep->num];
2856	writel(val, addr: &dev->plregs->pl_ep_ctrl);
2857	val |= BIT(SEQUENCE_NUMBER_RESET);
2858	writel(val, addr: &dev->plregs->pl_ep_ctrl);
2859
2860	return;
2861	}
2862
2863	static void handle_stat0_irqs_superspeed(struct net2280 *dev,
2864	struct net2280_ep *ep, struct usb_ctrlrequest r)
2865	{
2866	struct net2280_ep *e;
2867	u16 status;
2868	int tmp = 0;
2869
2870	#define w_value le16_to_cpu(r.wValue)
2871	#define w_index le16_to_cpu(r.wIndex)
2872	#define w_length le16_to_cpu(r.wLength)
2873
2874	switch (r.bRequest) {
2875	case USB_REQ_SET_CONFIGURATION:
2876	dev->addressed_state = !w_value;
2877	goto usb3_delegate;
2878
2879	case USB_REQ_GET_STATUS:
2880	switch (r.bRequestType) {
2881	case (USB_DIR_IN | USB_TYPE_STANDARD | USB_RECIP_DEVICE):
2882	status = dev->wakeup_enable ? 0x02 : 0x00;
2883	if (dev->gadget.is_selfpowered)
2884	status |= BIT(0);
2885	status |= (dev->u1_enable << 2 | dev->u2_enable << 3 |
2886	dev->ltm_enable << 4);
2887	writel(val: 0, addr: &dev->epregs[0].ep_irqenb);
2888	set_fifo_bytecount(ep, count: sizeof(status));
2889	writel(val: (__force u32) status, addr: &dev->epregs[0].ep_data);
2890	allow_status_338x(ep);
2891	break;
2892
2893	case (USB_DIR_IN | USB_TYPE_STANDARD | USB_RECIP_ENDPOINT):
2894	e = get_ep_by_addr(dev, w_index);
2895	if (!e)
2896	goto do_stall3;
2897	status = readl(addr: &e->regs->ep_rsp) &
2898	BIT(CLEAR_ENDPOINT_HALT);
2899	writel(val: 0, addr: &dev->epregs[0].ep_irqenb);
2900	set_fifo_bytecount(ep, count: sizeof(status));
2901	writel(val: (__force u32) status, addr: &dev->epregs[0].ep_data);
2902	allow_status_338x(ep);
2903	break;
2904
2905	default:
2906	goto usb3_delegate;
2907	}
2908	break;
2909
2910	case USB_REQ_CLEAR_FEATURE:
2911	switch (r.bRequestType) {
2912	case (USB_DIR_OUT | USB_TYPE_STANDARD | USB_RECIP_DEVICE):
2913	if (!dev->addressed_state) {
2914	switch (w_value) {
2915	case USB_DEVICE_U1_ENABLE:
2916	dev->u1_enable = 0;
2917	writel(readl(addr: &dev->usb_ext->usbctl2) &
2918	~BIT(U1_ENABLE),
2919	addr: &dev->usb_ext->usbctl2);
2920	allow_status_338x(ep);
2921	goto next_endpoints3;
2922
2923	case USB_DEVICE_U2_ENABLE:
2924	dev->u2_enable = 0;
2925	writel(readl(addr: &dev->usb_ext->usbctl2) &
2926	~BIT(U2_ENABLE),
2927	addr: &dev->usb_ext->usbctl2);
2928	allow_status_338x(ep);
2929	goto next_endpoints3;
2930
2931	case USB_DEVICE_LTM_ENABLE:
2932	dev->ltm_enable = 0;
2933	writel(readl(addr: &dev->usb_ext->usbctl2) &
2934	~BIT(LTM_ENABLE),
2935	addr: &dev->usb_ext->usbctl2);
2936	allow_status_338x(ep);
2937	goto next_endpoints3;
2938
2939	default:
2940	break;
2941	}
2942	}
2943	if (w_value == USB_DEVICE_REMOTE_WAKEUP) {
2944	dev->wakeup_enable = 0;
2945	writel(readl(addr: &dev->usb->usbctl) &
2946	~BIT(DEVICE_REMOTE_WAKEUP_ENABLE),
2947	addr: &dev->usb->usbctl);
2948	allow_status_338x(ep);
2949	break;
2950	}
2951	goto usb3_delegate;
2952
2953	case (USB_DIR_OUT | USB_TYPE_STANDARD | USB_RECIP_ENDPOINT):
2954	e = get_ep_by_addr(dev, w_index);
2955	if (!e)
2956	goto do_stall3;
2957	if (w_value != USB_ENDPOINT_HALT)
2958	goto do_stall3;
2959	ep_vdbg(dev, "%s clear halt\n", e->ep.name);
2960	/*
2961	* Workaround for SS SeqNum not cleared via
2962	* Endpoint Halt (Clear) bit. select endpoint
2963	*/
2964	ep_clear_seqnum(ep: e);
2965	clear_halt(ep: e);
2966	if (!list_empty(head: &e->queue) && e->td_dma)
2967	restart_dma(ep: e);
2968	allow_status(ep);
2969	ep->stopped = 1;
2970	break;
2971
2972	default:
2973	goto usb3_delegate;
2974	}
2975	break;
2976	case USB_REQ_SET_FEATURE:
2977	switch (r.bRequestType) {
2978	case (USB_DIR_OUT | USB_TYPE_STANDARD | USB_RECIP_DEVICE):
2979	if (!dev->addressed_state) {
2980	switch (w_value) {
2981	case USB_DEVICE_U1_ENABLE:
2982	dev->u1_enable = 1;
2983	writel(readl(addr: &dev->usb_ext->usbctl2) |
2984	BIT(U1_ENABLE),
2985	addr: &dev->usb_ext->usbctl2);
2986	allow_status_338x(ep);
2987	goto next_endpoints3;
2988
2989	case USB_DEVICE_U2_ENABLE:
2990	dev->u2_enable = 1;
2991	writel(readl(addr: &dev->usb_ext->usbctl2) |
2992	BIT(U2_ENABLE),
2993	addr: &dev->usb_ext->usbctl2);
2994	allow_status_338x(ep);
2995	goto next_endpoints3;
2996
2997	case USB_DEVICE_LTM_ENABLE:
2998	dev->ltm_enable = 1;
2999	writel(readl(addr: &dev->usb_ext->usbctl2) |
3000	BIT(LTM_ENABLE),
3001	addr: &dev->usb_ext->usbctl2);
3002	allow_status_338x(ep);
3003	goto next_endpoints3;
3004	default:
3005	break;
3006	}
3007	}
3008
3009	if (w_value == USB_DEVICE_REMOTE_WAKEUP) {
3010	dev->wakeup_enable = 1;
3011	writel(readl(addr: &dev->usb->usbctl) |
3012	BIT(DEVICE_REMOTE_WAKEUP_ENABLE),
3013	addr: &dev->usb->usbctl);
3014	allow_status_338x(ep);
3015	break;
3016	}
3017	goto usb3_delegate;
3018
3019	case (USB_DIR_OUT | USB_TYPE_STANDARD | USB_RECIP_ENDPOINT):
3020	e = get_ep_by_addr(dev, w_index);
3021	if (!e || (w_value != USB_ENDPOINT_HALT))
3022	goto do_stall3;
3023	ep->stopped = 1;
3024	if (ep->num == 0)
3025	ep->dev->protocol_stall = 1;
3026	else {
3027	if (ep->dma)
3028	abort_dma(ep);
3029	set_halt(ep);
3030	}
3031	allow_status_338x(ep);
3032	break;
3033
3034	default:
3035	goto usb3_delegate;
3036	}
3037
3038	break;
3039	default:
3040
3041	usb3_delegate:
3042	ep_vdbg(dev, "setup %02x.%02x v%04x i%04x l%04x ep_cfg %08x\n",
3043	r.bRequestType, r.bRequest,
3044	w_value, w_index, w_length,
3045	readl(&ep->cfg->ep_cfg));
3046
3047	ep->responded = 0;
3048	if (dev->async_callbacks) {
3049	spin_unlock(lock: &dev->lock);
3050	tmp = dev->driver->setup(&dev->gadget, &r);
3051	spin_lock(lock: &dev->lock);
3052	}
3053	}
3054	do_stall3:
3055	if (tmp < 0) {
3056	ep_vdbg(dev, "req %02x.%02x protocol STALL; stat %d\n",
3057	r.bRequestType, r.bRequest, tmp);
3058	dev->protocol_stall = 1;
3059	/* TD 9.9 Halt Endpoint test. TD 9.22 Set feature test */
3060	set_halt(ep);
3061	}
3062
3063	next_endpoints3:
3064
3065	#undef w_value
3066	#undef w_index
3067	#undef w_length
3068
3069	return;
3070	}
3071
3072	static void usb338x_handle_ep_intr(struct net2280 *dev, u32 stat0)
3073	{
3074	u32 index;
3075	u32 bit;
3076
3077	for (index = 0; index < ARRAY_SIZE(ep_bit); index++) {
3078	bit = BIT(ep_bit[index]);
3079
3080	if (!stat0)
3081	break;
3082
3083	if (!(stat0 & bit))
3084	continue;
3085
3086	stat0 &= ~bit;
3087
3088	handle_ep_small(ep: &dev->ep[index]);
3089	}
3090	}
3091
3092	static void handle_stat0_irqs(struct net2280 *dev, u32 stat)
3093	{
3094	struct net2280_ep *ep;
3095	u32 num, scratch;
3096
3097	/* most of these don't need individual acks */
3098	stat &= ~BIT(INTA_ASSERTED);
3099	if (!stat)
3100	return;
3101	/* ep_dbg(dev, "irqstat0 %04x\n", stat); */
3102
3103	/* starting a control request? */
3104	if (unlikely(stat & BIT(SETUP_PACKET_INTERRUPT))) {
3105	union {
3106	u32 raw[2];
3107	struct usb_ctrlrequest r;
3108	} u;
3109	int tmp;
3110	struct net2280_request *req;
3111
3112	if (dev->gadget.speed == USB_SPEED_UNKNOWN) {
3113	u32 val = readl(addr: &dev->usb->usbstat);
3114	if (val & BIT(SUPER_SPEED)) {
3115	dev->gadget.speed = USB_SPEED_SUPER;
3116	usb_ep_set_maxpacket_limit(ep: &dev->ep[0].ep,
3117	EP0_SS_MAX_PACKET_SIZE);
3118	} else if (val & BIT(HIGH_SPEED)) {
3119	dev->gadget.speed = USB_SPEED_HIGH;
3120	usb_ep_set_maxpacket_limit(ep: &dev->ep[0].ep,
3121	EP0_HS_MAX_PACKET_SIZE);
3122	} else {
3123	dev->gadget.speed = USB_SPEED_FULL;
3124	usb_ep_set_maxpacket_limit(ep: &dev->ep[0].ep,
3125	EP0_HS_MAX_PACKET_SIZE);
3126	}
3127	net2280_led_speed(dev, speed: dev->gadget.speed);
3128	ep_dbg(dev, "%s\n",
3129	usb_speed_string(dev->gadget.speed));
3130	}
3131
3132	ep = &dev->ep[0];
3133	ep->irqs++;
3134
3135	/* make sure any leftover request state is cleared */
3136	stat &= ~BIT(ENDPOINT_0_INTERRUPT);
3137	while (!list_empty(head: &ep->queue)) {
3138	req = list_entry(ep->queue.next,
3139	struct net2280_request, queue);
3140	done(ep, req, status: (req->req.actual == req->req.length)
3141	? 0 : -EPROTO);
3142	}
3143	ep->stopped = 0;
3144	dev->protocol_stall = 0;
3145	if (!(dev->quirks & PLX_PCIE)) {
3146	if (ep->dev->quirks & PLX_2280)
3147	tmp = BIT(FIFO_OVERFLOW) |
3148	BIT(FIFO_UNDERFLOW);
3149	else
3150	tmp = 0;
3151
3152	writel(val: tmp | BIT(TIMEOUT) |
3153	BIT(USB_STALL_SENT) |
3154	BIT(USB_IN_NAK_SENT) |
3155	BIT(USB_IN_ACK_RCVD) |
3156	BIT(USB_OUT_PING_NAK_SENT) |
3157	BIT(USB_OUT_ACK_SENT) |
3158	BIT(SHORT_PACKET_OUT_DONE_INTERRUPT) |
3159	BIT(SHORT_PACKET_TRANSFERRED_INTERRUPT) |
3160	BIT(DATA_PACKET_RECEIVED_INTERRUPT) |
3161	BIT(DATA_PACKET_TRANSMITTED_INTERRUPT) |
3162	BIT(DATA_OUT_PING_TOKEN_INTERRUPT) |
3163	BIT(DATA_IN_TOKEN_INTERRUPT),
3164	addr: &ep->regs->ep_stat);
3165	}
3166	u.raw[0] = readl(addr: &dev->usb->setup0123);
3167	u.raw[1] = readl(addr: &dev->usb->setup4567);
3168
3169	cpu_to_le32s(&u.raw[0]);
3170	cpu_to_le32s(&u.raw[1]);
3171
3172	if ((dev->quirks & PLX_PCIE) && !dev->bug7734_patched)
3173	defect7374_workaround(dev, r: u.r);
3174
3175	tmp = 0;
3176
3177	#define w_value le16_to_cpu(u.r.wValue)
3178	#define w_index le16_to_cpu(u.r.wIndex)
3179	#define w_length le16_to_cpu(u.r.wLength)
3180
3181	/* ack the irq */
3182	writel(BIT(SETUP_PACKET_INTERRUPT), addr: &dev->regs->irqstat0);
3183	stat ^= BIT(SETUP_PACKET_INTERRUPT);
3184
3185	/* watch control traffic at the token level, and force
3186	* synchronization before letting the status stage happen.
3187	* FIXME ignore tokens we'll NAK, until driver responds.
3188	* that'll mean a lot less irqs for some drivers.
3189	*/
3190	ep->is_in = (u.r.bRequestType & USB_DIR_IN) != 0;
3191	if (ep->is_in) {
3192	scratch = BIT(DATA_PACKET_TRANSMITTED_INTERRUPT) |
3193	BIT(DATA_OUT_PING_TOKEN_INTERRUPT) |
3194	BIT(DATA_IN_TOKEN_INTERRUPT);
3195	stop_out_naking(ep);
3196	} else
3197	scratch = BIT(DATA_PACKET_RECEIVED_INTERRUPT) |
3198	BIT(DATA_OUT_PING_TOKEN_INTERRUPT) |
3199	BIT(DATA_IN_TOKEN_INTERRUPT);
3200	writel(val: scratch, addr: &dev->epregs[0].ep_irqenb);
3201
3202	/* we made the hardware handle most lowlevel requests;
3203	* everything else goes uplevel to the gadget code.
3204	*/
3205	ep->responded = 1;
3206
3207	if (dev->gadget.speed == USB_SPEED_SUPER) {
3208	handle_stat0_irqs_superspeed(dev, ep, r: u.r);
3209	goto next_endpoints;
3210	}
3211
3212	switch (u.r.bRequest) {
3213	case USB_REQ_GET_STATUS: {
3214	struct net2280_ep *e;
3215	__le32 status;
3216
3217	/* hw handles device and interface status */
3218	if (u.r.bRequestType != (USB_DIR_IN|USB_RECIP_ENDPOINT))
3219	goto delegate;
3220	e = get_ep_by_addr(dev, w_index);
3221	if (!e || w_length > 2)
3222	goto do_stall;
3223
3224	if (readl(addr: &e->regs->ep_rsp) & BIT(SET_ENDPOINT_HALT))
3225	status = cpu_to_le32(1);
3226	else
3227	status = cpu_to_le32(0);
3228
3229	/* don't bother with a request object! */
3230	writel(val: 0, addr: &dev->epregs[0].ep_irqenb);
3231	set_fifo_bytecount(ep, w_length);
3232	writel(val: (__force u32)status, addr: &dev->epregs[0].ep_data);
3233	allow_status(ep);
3234	ep_vdbg(dev, "%s stat %02x\n", ep->ep.name, status);
3235	goto next_endpoints;
3236	}
3237	break;
3238	case USB_REQ_CLEAR_FEATURE: {
3239	struct net2280_ep *e;
3240
3241	/* hw handles device features */
3242	if (u.r.bRequestType != USB_RECIP_ENDPOINT)
3243	goto delegate;
3244	if (w_value != USB_ENDPOINT_HALT || w_length != 0)
3245	goto do_stall;
3246	e = get_ep_by_addr(dev, w_index);
3247	if (!e)
3248	goto do_stall;
3249	if (e->wedged) {
3250	ep_vdbg(dev, "%s wedged, halt not cleared\n",
3251	ep->ep.name);
3252	} else {
3253	ep_vdbg(dev, "%s clear halt\n", e->ep.name);
3254	clear_halt(ep: e);
3255	if ((ep->dev->quirks & PLX_PCIE) &&
3256	!list_empty(head: &e->queue) && e->td_dma)
3257	restart_dma(ep: e);
3258	}
3259	allow_status(ep);
3260	goto next_endpoints;
3261	}
3262	break;
3263	case USB_REQ_SET_FEATURE: {
3264	struct net2280_ep *e;
3265
3266	/* hw handles device features */
3267	if (u.r.bRequestType != USB_RECIP_ENDPOINT)
3268	goto delegate;
3269	if (w_value != USB_ENDPOINT_HALT || w_length != 0)
3270	goto do_stall;
3271	e = get_ep_by_addr(dev, w_index);
3272	if (!e)
3273	goto do_stall;
3274	if (e->ep.name == ep0name)
3275	goto do_stall;
3276	set_halt(e);
3277	if ((dev->quirks & PLX_PCIE) && e->dma)
3278	abort_dma(ep: e);
3279	allow_status(ep);
3280	ep_vdbg(dev, "%s set halt\n", ep->ep.name);
3281	goto next_endpoints;
3282	}
3283	break;
3284	default:
3285	delegate:
3286	ep_vdbg(dev, "setup %02x.%02x v%04x i%04x l%04x "
3287	"ep_cfg %08x\n",
3288	u.r.bRequestType, u.r.bRequest,
3289	w_value, w_index, w_length,
3290	readl(&ep->cfg->ep_cfg));
3291	ep->responded = 0;
3292	if (dev->async_callbacks) {
3293	spin_unlock(lock: &dev->lock);
3294	tmp = dev->driver->setup(&dev->gadget, &u.r);
3295	spin_lock(lock: &dev->lock);
3296	}
3297	}
3298
3299	/* stall ep0 on error */
3300	if (tmp < 0) {
3301	do_stall:
3302	ep_vdbg(dev, "req %02x.%02x protocol STALL; stat %d\n",
3303	u.r.bRequestType, u.r.bRequest, tmp);
3304	dev->protocol_stall = 1;
3305	}
3306
3307	/* some in/out token irq should follow; maybe stall then.
3308	* driver must queue a request (even zlp) or halt ep0
3309	* before the host times out.
3310	*/
3311	}
3312
3313	#undef w_value
3314	#undef w_index
3315	#undef w_length
3316
3317	next_endpoints:
3318	if ((dev->quirks & PLX_PCIE) && dev->enhanced_mode) {
3319	u32 mask = (BIT(ENDPOINT_0_INTERRUPT) |
3320	USB3380_IRQSTAT0_EP_INTR_MASK_IN |
3321	USB3380_IRQSTAT0_EP_INTR_MASK_OUT);
3322
3323	if (stat & mask) {
3324	usb338x_handle_ep_intr(dev, stat0: stat & mask);
3325	stat &= ~mask;
3326	}
3327	} else {
3328	/* endpoint data irq ? */
3329	scratch = stat & 0x7f;
3330	stat &= ~0x7f;
3331	for (num = 0; scratch; num++) {
3332	u32 t;
3333
3334	/* do this endpoint's FIFO and queue need tending? */
3335	t = BIT(num);
3336	if ((scratch & t) == 0)
3337	continue;
3338	scratch ^= t;
3339
3340	ep = &dev->ep[num];
3341	handle_ep_small(ep);
3342	}
3343	}
3344
3345	if (stat)
3346	ep_dbg(dev, "unhandled irqstat0 %08x\n", stat);
3347	}
3348
3349	#define DMA_INTERRUPTS (BIT(DMA_D_INTERRUPT) | \
3350	BIT(DMA_C_INTERRUPT) | \
3351	BIT(DMA_B_INTERRUPT) | \
3352	BIT(DMA_A_INTERRUPT))
3353	#define PCI_ERROR_INTERRUPTS ( \
3354	BIT(PCI_MASTER_ABORT_RECEIVED_INTERRUPT) | \
3355	BIT(PCI_TARGET_ABORT_RECEIVED_INTERRUPT) | \
3356	BIT(PCI_RETRY_ABORT_INTERRUPT))
3357
3358	static void handle_stat1_irqs(struct net2280 *dev, u32 stat)
3359	__releases(dev->lock)
3360	__acquires(dev->lock)
3361	{
3362	struct net2280_ep *ep;
3363	u32 tmp, num, mask, scratch;
3364
3365	/* after disconnect there's nothing else to do! */
3366	tmp = BIT(VBUS_INTERRUPT) | BIT(ROOT_PORT_RESET_INTERRUPT);
3367	mask = BIT(SUPER_SPEED) | BIT(HIGH_SPEED) | BIT(FULL_SPEED);
3368
3369	/* VBUS disconnect is indicated by VBUS_PIN and VBUS_INTERRUPT set.
3370	* Root Port Reset is indicated by ROOT_PORT_RESET_INTERRUPT set and
3371	* both HIGH_SPEED and FULL_SPEED clear (as ROOT_PORT_RESET_INTERRUPT
3372	* only indicates a change in the reset state).
3373	*/
3374	if (stat & tmp) {
3375	bool reset = false;
3376	bool disconnect = false;
3377
3378	/*
3379	* Ignore disconnects and resets if the speed hasn't been set.
3380	* VBUS can bounce and there's always an initial reset.
3381	*/
3382	writel(val: tmp, addr: &dev->regs->irqstat1);
3383	if (dev->gadget.speed != USB_SPEED_UNKNOWN) {
3384	if ((stat & BIT(VBUS_INTERRUPT)) &&
3385	(readl(addr: &dev->usb->usbctl) &
3386	BIT(VBUS_PIN)) == 0) {
3387	disconnect = true;
3388	ep_dbg(dev, "disconnect %s\n",
3389	dev->driver->driver.name);
3390	} else if ((stat & BIT(ROOT_PORT_RESET_INTERRUPT)) &&
3391	(readl(addr: &dev->usb->usbstat) & mask)
3392	== 0) {
3393	reset = true;
3394	ep_dbg(dev, "reset %s\n",
3395	dev->driver->driver.name);
3396	}
3397
3398	if (disconnect || reset) {
3399	stop_activity(dev, driver: dev->driver);
3400	ep0_start(dev);
3401	if (dev->async_callbacks) {
3402	spin_unlock(lock: &dev->lock);
3403	if (reset)
3404	usb_gadget_udc_reset(gadget: &dev->gadget, driver: dev->driver);
3405	else
3406	(dev->driver->disconnect)(&dev->gadget);
3407	spin_lock(lock: &dev->lock);
3408	}
3409	return;
3410	}
3411	}
3412	stat &= ~tmp;
3413
3414	/* vBUS can bounce ... one of many reasons to ignore the
3415	* notion of hotplug events on bus connect/disconnect!
3416	*/
3417	if (!stat)
3418	return;
3419	}
3420
3421	/* NOTE: chip stays in PCI D0 state for now, but it could
3422	* enter D1 to save more power
3423	*/
3424	tmp = BIT(SUSPEND_REQUEST_CHANGE_INTERRUPT);
3425	if (stat & tmp) {
3426	writel(val: tmp, addr: &dev->regs->irqstat1);
3427	spin_unlock(lock: &dev->lock);
3428	if (stat & BIT(SUSPEND_REQUEST_INTERRUPT)) {
3429	if (dev->async_callbacks && dev->driver->suspend)
3430	dev->driver->suspend(&dev->gadget);
3431	if (!enable_suspend)
3432	stat &= ~BIT(SUSPEND_REQUEST_INTERRUPT);
3433	} else {
3434	if (dev->async_callbacks && dev->driver->resume)
3435	dev->driver->resume(&dev->gadget);
3436	/* at high speed, note erratum 0133 */
3437	}
3438	spin_lock(lock: &dev->lock);
3439	stat &= ~tmp;
3440	}
3441
3442	/* clear any other status/irqs */
3443	if (stat)
3444	writel(val: stat, addr: &dev->regs->irqstat1);
3445
3446	/* some status we can just ignore */
3447	if (dev->quirks & PLX_2280)
3448	stat &= ~(BIT(CONTROL_STATUS_INTERRUPT) |
3449	BIT(SUSPEND_REQUEST_INTERRUPT) |
3450	BIT(RESUME_INTERRUPT) |
3451	BIT(SOF_INTERRUPT));
3452	else
3453	stat &= ~(BIT(CONTROL_STATUS_INTERRUPT) |
3454	BIT(RESUME_INTERRUPT) |
3455	BIT(SOF_DOWN_INTERRUPT) |
3456	BIT(SOF_INTERRUPT));
3457
3458	if (!stat)
3459	return;
3460	/* ep_dbg(dev, "irqstat1 %08x\n", stat);*/
3461
3462	/* DMA status, for ep-{a,b,c,d} */
3463	scratch = stat & DMA_INTERRUPTS;
3464	stat &= ~DMA_INTERRUPTS;
3465	scratch >>= 9;
3466	for (num = 0; scratch; num++) {
3467	struct net2280_dma_regs __iomem *dma;
3468
3469	tmp = BIT(num);
3470	if ((tmp & scratch) == 0)
3471	continue;
3472	scratch ^= tmp;
3473
3474	ep = &dev->ep[num + 1];
3475	dma = ep->dma;
3476
3477	if (!dma)
3478	continue;
3479
3480	/* clear ep's dma status */
3481	tmp = readl(addr: &dma->dmastat);
3482	writel(val: tmp, addr: &dma->dmastat);
3483
3484	/* dma sync*/
3485	if (dev->quirks & PLX_PCIE) {
3486	u32 r_dmacount = readl(addr: &dma->dmacount);
3487	if (!ep->is_in && (r_dmacount & 0x00FFFFFF) &&
3488	(tmp & BIT(DMA_TRANSACTION_DONE_INTERRUPT)))
3489	continue;
3490	}
3491
3492	if (!(tmp & BIT(DMA_TRANSACTION_DONE_INTERRUPT))) {
3493	ep_dbg(ep->dev, "%s no xact done? %08x\n",
3494	ep->ep.name, tmp);
3495	continue;
3496	}
3497	stop_dma(dma: ep->dma);
3498
3499	/* OUT transfers terminate when the data from the
3500	* host is in our memory. Process whatever's done.
3501	* On this path, we know transfer's last packet wasn't
3502	* less than req->length. NAK_OUT_PACKETS may be set,
3503	* or the FIFO may already be holding new packets.
3504	*
3505	* IN transfers can linger in the FIFO for a very
3506	* long time ... we ignore that for now, accounting
3507	* precisely (like PIO does) needs per-packet irqs
3508	*/
3509	scan_dma_completions(ep);
3510
3511	/* disable dma on inactive queues; else maybe restart */
3512	if (!list_empty(head: &ep->queue)) {
3513	tmp = readl(addr: &dma->dmactl);
3514	restart_dma(ep);
3515	}
3516	ep->irqs++;
3517	}
3518
3519	/* NOTE: there are other PCI errors we might usefully notice.
3520	* if they appear very often, here's where to try recovering.
3521	*/
3522	if (stat & PCI_ERROR_INTERRUPTS) {
3523	ep_err(dev, "pci dma error; stat %08x\n", stat);
3524	stat &= ~PCI_ERROR_INTERRUPTS;
3525	/* these are fatal errors, but "maybe" they won't
3526	* happen again ...
3527	*/
3528	stop_activity(dev, driver: dev->driver);
3529	ep0_start(dev);
3530	stat = 0;
3531	}
3532
3533	if (stat)
3534	ep_dbg(dev, "unhandled irqstat1 %08x\n", stat);
3535	}
3536
3537	static irqreturn_t net2280_irq(int irq, void *_dev)
3538	{
3539	struct net2280 *dev = _dev;
3540
3541	/* shared interrupt, not ours */
3542	if ((dev->quirks & PLX_LEGACY) &&
3543	(!(readl(addr: &dev->regs->irqstat0) & BIT(INTA_ASSERTED))))
3544	return IRQ_NONE;
3545
3546	spin_lock(lock: &dev->lock);
3547
3548	/* handle disconnect, dma, and more */
3549	handle_stat1_irqs(dev, readl(addr: &dev->regs->irqstat1));
3550
3551	/* control requests and PIO */
3552	handle_stat0_irqs(dev, readl(addr: &dev->regs->irqstat0));
3553
3554	if (dev->quirks & PLX_PCIE) {
3555	/* re-enable interrupt to trigger any possible new interrupt */
3556	u32 pciirqenb1 = readl(addr: &dev->regs->pciirqenb1);
3557	writel(val: pciirqenb1 & 0x7FFFFFFF, addr: &dev->regs->pciirqenb1);
3558	writel(val: pciirqenb1, addr: &dev->regs->pciirqenb1);
3559	}
3560
3561	spin_unlock(lock: &dev->lock);
3562
3563	return IRQ_HANDLED;
3564	}
3565
3566	/*-------------------------------------------------------------------------*/
3567
3568	static void gadget_release(struct device *_dev)
3569	{
3570	struct net2280 *dev = container_of(_dev, struct net2280, gadget.dev);
3571
3572	kfree(objp: dev);
3573	}
3574
3575	/* tear down the binding between this driver and the pci device */
3576
3577	static void net2280_remove(struct pci_dev *pdev)
3578	{
3579	struct net2280 *dev = pci_get_drvdata(pdev);
3580
3581	if (dev->added)
3582	usb_del_gadget(gadget: &dev->gadget);
3583
3584	BUG_ON(dev->driver);
3585
3586	/* then clean up the resources we allocated during probe() */
3587	if (dev->requests) {
3588	int i;
3589	for (i = 1; i < 5; i++) {
3590	if (!dev->ep[i].dummy)
3591	continue;
3592	dma_pool_free(pool: dev->requests, vaddr: dev->ep[i].dummy,
3593	addr: dev->ep[i].td_dma);
3594	}
3595	dma_pool_destroy(pool: dev->requests);
3596	}
3597	if (dev->got_irq)
3598	free_irq(pdev->irq, dev);
3599	if (dev->quirks & PLX_PCIE)
3600	pci_disable_msi(dev: pdev);
3601	if (dev->regs) {
3602	net2280_led_shutdown(dev);
3603	iounmap(addr: dev->regs);
3604	}
3605	if (dev->region)
3606	release_mem_region(pci_resource_start(pdev, 0),
3607	pci_resource_len(pdev, 0));
3608	if (dev->enabled)
3609	pci_disable_device(dev: pdev);
3610	device_remove_file(dev: &pdev->dev, attr: &dev_attr_registers);
3611
3612	ep_info(dev, "unbind\n");
3613	usb_put_gadget(gadget: &dev->gadget);
3614	}
3615
3616	/* wrap this driver around the specified device, but
3617	* don't respond over USB until a gadget driver binds to us.
3618	*/
3619
3620	static int net2280_probe(struct pci_dev *pdev, const struct pci_device_id *id)
3621	{
3622	struct net2280 *dev;
3623	unsigned long resource, len;
3624	void __iomem *base = NULL;
3625	int retval, i;
3626
3627	/* alloc, and start init */
3628	dev = kzalloc(size: sizeof(*dev), GFP_KERNEL);
3629	if (dev == NULL) {
3630	retval = -ENOMEM;
3631	goto done;
3632	}
3633
3634	pci_set_drvdata(pdev, data: dev);
3635	usb_initialize_gadget(parent: &pdev->dev, gadget: &dev->gadget, release: gadget_release);
3636	spin_lock_init(&dev->lock);
3637	dev->quirks = id->driver_data;
3638	dev->pdev = pdev;
3639	dev->gadget.ops = &net2280_ops;
3640	dev->gadget.max_speed = (dev->quirks & PLX_SUPERSPEED) ?
3641	USB_SPEED_SUPER : USB_SPEED_HIGH;
3642
3643	/* the "gadget" abstracts/virtualizes the controller */
3644	dev->gadget.name = driver_name;
3645
3646	/* now all the pci goodies ... */
3647	if (pci_enable_device(dev: pdev) < 0) {
3648	retval = -ENODEV;
3649	goto done;
3650	}
3651	dev->enabled = 1;
3652
3653	/* BAR 0 holds all the registers
3654	* BAR 1 is 8051 memory; unused here (note erratum 0103)
3655	* BAR 2 is fifo memory; unused here
3656	*/
3657	resource = pci_resource_start(pdev, 0);
3658	len = pci_resource_len(pdev, 0);
3659	if (!request_mem_region(resource, len, driver_name)) {
3660	ep_dbg(dev, "controller already in use\n");
3661	retval = -EBUSY;
3662	goto done;
3663	}
3664	dev->region = 1;
3665
3666	/* FIXME provide firmware download interface to put
3667	* 8051 code into the chip, e.g. to turn on PCI PM.
3668	*/
3669
3670	base = ioremap(offset: resource, size: len);
3671	if (base == NULL) {
3672	ep_dbg(dev, "can't map memory\n");
3673	retval = -EFAULT;
3674	goto done;
3675	}
3676	dev->regs = (struct net2280_regs __iomem *) base;
3677	dev->usb = (struct net2280_usb_regs __iomem *) (base + 0x0080);
3678	dev->pci = (struct net2280_pci_regs __iomem *) (base + 0x0100);
3679	dev->dma = (struct net2280_dma_regs __iomem *) (base + 0x0180);
3680	dev->dep = (struct net2280_dep_regs __iomem *) (base + 0x0200);
3681	dev->epregs = (struct net2280_ep_regs __iomem *) (base + 0x0300);
3682
3683	if (dev->quirks & PLX_PCIE) {
3684	u32 fsmvalue;
3685	u32 usbstat;
3686	dev->usb_ext = (struct usb338x_usb_ext_regs __iomem *)
3687	(base + 0x00b4);
3688	dev->llregs = (struct usb338x_ll_regs __iomem *)
3689	(base + 0x0700);
3690	dev->plregs = (struct usb338x_pl_regs __iomem *)
3691	(base + 0x0800);
3692	usbstat = readl(addr: &dev->usb->usbstat);
3693	dev->enhanced_mode = !!(usbstat & BIT(11));
3694	dev->n_ep = (dev->enhanced_mode) ? 9 : 5;
3695	/* put into initial config, link up all endpoints */
3696	fsmvalue = get_idx_reg(regs: dev->regs, SCRATCH) &
3697	(0xf << DEFECT7374_FSM_FIELD);
3698	/* See if firmware needs to set up for workaround: */
3699	if (fsmvalue == DEFECT7374_FSM_SS_CONTROL_READ) {
3700	dev->bug7734_patched = 1;
3701	writel(val: 0, addr: &dev->usb->usbctl);
3702	} else
3703	dev->bug7734_patched = 0;
3704	} else {
3705	dev->enhanced_mode = 0;
3706	dev->n_ep = 7;
3707	/* put into initial config, link up all endpoints */
3708	writel(val: 0, addr: &dev->usb->usbctl);
3709	}
3710
3711	usb_reset(dev);
3712	usb_reinit(dev);
3713
3714	/* irq setup after old hardware is cleaned up */
3715	if (!pdev->irq) {
3716	ep_err(dev, "No IRQ. Check PCI setup!\n");
3717	retval = -ENODEV;
3718	goto done;
3719	}
3720
3721	if (dev->quirks & PLX_PCIE)
3722	if (pci_enable_msi(dev: pdev))
3723	ep_err(dev, "Failed to enable MSI mode\n");
3724
3725	if (request_irq(irq: pdev->irq, handler: net2280_irq, IRQF_SHARED,
3726	name: driver_name, dev)) {
3727	ep_err(dev, "request interrupt %d failed\n", pdev->irq);
3728	retval = -EBUSY;
3729	goto done;
3730	}
3731	dev->got_irq = 1;
3732
3733	/* DMA setup */
3734	/* NOTE: we know only the 32 LSBs of dma addresses may be nonzero */
3735	dev->requests = dma_pool_create(name: "requests", dev: &pdev->dev,
3736	size: sizeof(struct net2280_dma),
3737	align: 0 /* no alignment requirements */,
3738	allocation: 0 /* or page-crossing issues */);
3739	if (!dev->requests) {
3740	ep_dbg(dev, "can't get request pool\n");
3741	retval = -ENOMEM;
3742	goto done;
3743	}
3744	for (i = 1; i < 5; i++) {
3745	struct net2280_dma *td;
3746
3747	td = dma_pool_alloc(pool: dev->requests, GFP_KERNEL,
3748	handle: &dev->ep[i].td_dma);
3749	if (!td) {
3750	ep_dbg(dev, "can't get dummy %d\n", i);
3751	retval = -ENOMEM;
3752	goto done;
3753	}
3754	td->dmacount = 0; /* not VALID */
3755	td->dmadesc = td->dmaaddr;
3756	dev->ep[i].dummy = td;
3757	}
3758
3759	/* enable lower-overhead pci memory bursts during DMA */
3760	if (dev->quirks & PLX_LEGACY)
3761	writel(BIT(DMA_MEMORY_WRITE_AND_INVALIDATE_ENABLE) |
3762	/*
3763	* 256 write retries may not be enough...
3764	BIT(PCI_RETRY_ABORT_ENABLE) |
3765	*/
3766	BIT(DMA_READ_MULTIPLE_ENABLE) |
3767	BIT(DMA_READ_LINE_ENABLE),
3768	addr: &dev->pci->pcimstctl);
3769	/* erratum 0115 shouldn't appear: Linux inits PCI_LATENCY_TIMER */
3770	pci_set_master(dev: pdev);
3771	pci_try_set_mwi(dev: pdev);
3772
3773	/* ... also flushes any posted pci writes */
3774	dev->chiprev = get_idx_reg(regs: dev->regs, REG_CHIPREV) & 0xffff;
3775
3776	/* done */
3777	ep_info(dev, "%s\n", driver_desc);
3778	ep_info(dev, "irq %d, pci mem %p, chip rev %04x\n",
3779	pdev->irq, base, dev->chiprev);
3780	ep_info(dev, "version: " DRIVER_VERSION "; %s\n",
3781	dev->enhanced_mode ? "enhanced mode" : "legacy mode");
3782	retval = device_create_file(device: &pdev->dev, entry: &dev_attr_registers);
3783	if (retval)
3784	goto done;
3785
3786	retval = usb_add_gadget(gadget: &dev->gadget);
3787	if (retval)
3788	goto done;
3789	dev->added = 1;
3790	return 0;
3791
3792	done:
3793	if (dev) {
3794	net2280_remove(pdev);
3795	kfree(objp: dev);
3796	}
3797	return retval;
3798	}
3799
3800	/* make sure the board is quiescent; otherwise it will continue
3801	* generating IRQs across the upcoming reboot.
3802	*/
3803
3804	static void net2280_shutdown(struct pci_dev *pdev)
3805	{
3806	struct net2280 *dev = pci_get_drvdata(pdev);
3807
3808	/* disable IRQs */
3809	writel(val: 0, addr: &dev->regs->pciirqenb0);
3810	writel(val: 0, addr: &dev->regs->pciirqenb1);
3811
3812	/* disable the pullup so the host will think we're gone */
3813	writel(val: 0, addr: &dev->usb->usbctl);
3814
3815	}
3816
3817
3818	/*-------------------------------------------------------------------------*/
3819
3820	static const struct pci_device_id pci_ids[] = { {
3821	.class = PCI_CLASS_SERIAL_USB_DEVICE,
3822	.class_mask = ~0,
3823	.vendor = PCI_VENDOR_ID_PLX_LEGACY,
3824	.device = 0x2280,
3825	.subvendor = PCI_ANY_ID,
3826	.subdevice = PCI_ANY_ID,
3827	.driver_data = PLX_LEGACY | PLX_2280,
3828	}, {
3829	.class = PCI_CLASS_SERIAL_USB_DEVICE,
3830	.class_mask = ~0,
3831	.vendor = PCI_VENDOR_ID_PLX_LEGACY,
3832	.device = 0x2282,
3833	.subvendor = PCI_ANY_ID,
3834	.subdevice = PCI_ANY_ID,
3835	.driver_data = PLX_LEGACY,
3836	},
3837	{
3838	.class = PCI_CLASS_SERIAL_USB_DEVICE,
3839	.class_mask = ~0,
3840	.vendor = PCI_VENDOR_ID_PLX,
3841	.device = 0x2380,
3842	.subvendor = PCI_ANY_ID,
3843	.subdevice = PCI_ANY_ID,
3844	.driver_data = PLX_PCIE,
3845	},
3846	{
3847	.class = ((PCI_CLASS_SERIAL_USB << 8) | 0xfe),
3848	.class_mask = ~0,
3849	.vendor = PCI_VENDOR_ID_PLX,
3850	.device = 0x3380,
3851	.subvendor = PCI_ANY_ID,
3852	.subdevice = PCI_ANY_ID,
3853	.driver_data = PLX_PCIE | PLX_SUPERSPEED,
3854	},
3855	{
3856	.class = PCI_CLASS_SERIAL_USB_DEVICE,
3857	.class_mask = ~0,
3858	.vendor = PCI_VENDOR_ID_PLX,
3859	.device = 0x3382,
3860	.subvendor = PCI_ANY_ID,
3861	.subdevice = PCI_ANY_ID,
3862	.driver_data = PLX_PCIE | PLX_SUPERSPEED,
3863	},
3864	{ /* end: all zeroes */ }
3865	};
3866	MODULE_DEVICE_TABLE(pci, pci_ids);
3867
3868	/* pci driver glue; this is a "new style" PCI driver module */
3869	static struct pci_driver net2280_pci_driver = {
3870	.name = driver_name,
3871	.id_table = pci_ids,
3872
3873	.probe = net2280_probe,
3874	.remove = net2280_remove,
3875	.shutdown = net2280_shutdown,
3876
3877	/* FIXME add power management support */
3878	};
3879
3880	module_pci_driver(net2280_pci_driver);
3881
3882	MODULE_DESCRIPTION(DRIVER_DESC);
3883	MODULE_AUTHOR("David Brownell");
3884	MODULE_LICENSE("GPL");
3885