1	// SPDX-License-Identifier: GPL-2.0+
2	/*
3	* Copyright (c) 2021 Aspeed Technology Inc.
4	*/
5
6	#include <linux/clk.h>
7	#include <linux/delay.h>
8	#include <linux/dma-mapping.h>
9	#include <linux/interrupt.h>
10	#include <linux/kernel.h>
11	#include <linux/module.h>
12	#include <linux/of.h>
13	#include <linux/platform_device.h>
14	#include <linux/prefetch.h>
15	#include <linux/usb/ch9.h>
16	#include <linux/usb/gadget.h>
17	#include <linux/slab.h>
18
19	#define AST_UDC_NUM_ENDPOINTS (1 + 4)
20	#define AST_UDC_EP0_MAX_PACKET 64 /* EP0's max packet size */
21	#define AST_UDC_EPn_MAX_PACKET 1024 /* Generic EPs max packet size */
22	#define AST_UDC_DESCS_COUNT 256 /* Use 256 stages descriptor mode (32/256) */
23	#define AST_UDC_DESC_MODE 1 /* Single/Multiple Stage(s) Descriptor Mode */
24
25	#define AST_UDC_EP_DMA_SIZE (AST_UDC_EPn_MAX_PACKET + 8 * AST_UDC_DESCS_COUNT)
26
27	/*****************************
28	* *
29	* UDC register definitions *
30	* *
31	*****************************/
32
33	#define AST_UDC_FUNC_CTRL 0x00 /* Root Function Control & Status Register */
34	#define AST_UDC_CONFIG 0x04 /* Root Configuration Setting Register */
35	#define AST_UDC_IER 0x08 /* Interrupt Control Register */
36	#define AST_UDC_ISR 0x0C /* Interrupt Status Register */
37	#define AST_UDC_EP_ACK_IER 0x10 /* Programmable ep Pool ACK Interrupt Enable Reg */
38	#define AST_UDC_EP_NAK_IER 0x14 /* Programmable ep Pool NAK Interrupt Enable Reg */
39	#define AST_UDC_EP_ACK_ISR 0x18 /* Programmable ep Pool ACK Interrupt Status Reg */
40	#define AST_UDC_EP_NAK_ISR 0x1C /* Programmable ep Pool NAK Interrupt Status Reg */
41	#define AST_UDC_DEV_RESET 0x20 /* Device Controller Soft Reset Enable Register */
42	#define AST_UDC_STS 0x24 /* USB Status Register */
43	#define AST_VHUB_EP_DATA 0x28 /* Programmable ep Pool Data Toggle Value Set */
44	#define AST_VHUB_ISO_TX_FAIL 0x2C /* Isochronous Transaction Fail Accumulator */
45	#define AST_UDC_EP0_CTRL 0x30 /* Endpoint 0 Control/Status Register */
46	#define AST_UDC_EP0_DATA_BUFF 0x34 /* Base Address of ep0 IN/OUT Data Buffer Reg */
47	#define AST_UDC_SETUP0 0x80 /* Root Device Setup Data Buffer0 */
48	#define AST_UDC_SETUP1 0x84 /* Root Device Setup Data Buffer1 */
49
50
51	/* Main control reg */
52	#define USB_PHY_CLK_EN BIT(31)
53	#define USB_FIFO_DYN_PWRD_EN BIT(19)
54	#define USB_EP_LONG_DESC BIT(18)
55	#define USB_BIST_TEST_PASS BIT(13)
56	#define USB_BIST_TURN_ON BIT(12)
57	#define USB_PHY_RESET_DIS BIT(11)
58	#define USB_TEST_MODE(x) ((x) << 8)
59	#define USB_FORCE_TIMER_HS BIT(7)
60	#define USB_FORCE_HS BIT(6)
61	#define USB_REMOTE_WAKEUP_12MS BIT(5)
62	#define USB_REMOTE_WAKEUP_EN BIT(4)
63	#define USB_AUTO_REMOTE_WAKEUP_EN BIT(3)
64	#define USB_STOP_CLK_IN_SUPEND BIT(2)
65	#define USB_UPSTREAM_FS BIT(1)
66	#define USB_UPSTREAM_EN BIT(0)
67
68	/* Main config reg */
69	#define UDC_CFG_SET_ADDR(x) ((x) & UDC_CFG_ADDR_MASK)
70	#define UDC_CFG_ADDR_MASK GENMASK(6, 0)
71
72	/* Interrupt ctrl & status reg */
73	#define UDC_IRQ_EP_POOL_NAK BIT(17)
74	#define UDC_IRQ_EP_POOL_ACK_STALL BIT(16)
75	#define UDC_IRQ_BUS_RESUME BIT(8)
76	#define UDC_IRQ_BUS_SUSPEND BIT(7)
77	#define UDC_IRQ_BUS_RESET BIT(6)
78	#define UDC_IRQ_EP0_IN_DATA_NAK BIT(4)
79	#define UDC_IRQ_EP0_IN_ACK_STALL BIT(3)
80	#define UDC_IRQ_EP0_OUT_NAK BIT(2)
81	#define UDC_IRQ_EP0_OUT_ACK_STALL BIT(1)
82	#define UDC_IRQ_EP0_SETUP BIT(0)
83	#define UDC_IRQ_ACK_ALL (0x1ff)
84
85	/* EP isr reg */
86	#define USB_EP3_ISR BIT(3)
87	#define USB_EP2_ISR BIT(2)
88	#define USB_EP1_ISR BIT(1)
89	#define USB_EP0_ISR BIT(0)
90	#define UDC_IRQ_EP_ACK_ALL (0xf)
91
92	/*Soft reset reg */
93	#define ROOT_UDC_SOFT_RESET BIT(0)
94
95	/* USB status reg */
96	#define UDC_STS_HIGHSPEED BIT(27)
97
98	/* Programmable EP data toggle */
99	#define EP_TOGGLE_SET_EPNUM(x) ((x) & 0x3)
100
101	/* EP0 ctrl reg */
102	#define EP0_GET_RX_LEN(x) ((x >> 16) & 0x7f)
103	#define EP0_TX_LEN(x) ((x & 0x7f) << 8)
104	#define EP0_RX_BUFF_RDY BIT(2)
105	#define EP0_TX_BUFF_RDY BIT(1)
106	#define EP0_STALL BIT(0)
107
108	/*************************************
109	* *
110	* per-endpoint register definitions *
111	* *
112	*************************************/
113
114	#define AST_UDC_EP_CONFIG 0x00 /* Endpoint Configuration Register */
115	#define AST_UDC_EP_DMA_CTRL 0x04 /* DMA Descriptor List Control/Status Register */
116	#define AST_UDC_EP_DMA_BUFF 0x08 /* DMA Descriptor/Buffer Base Address */
117	#define AST_UDC_EP_DMA_STS 0x0C /* DMA Descriptor List R/W Pointer and Status */
118
119	#define AST_UDC_EP_BASE 0x200
120	#define AST_UDC_EP_OFFSET 0x10
121
122	/* EP config reg */
123	#define EP_SET_MAX_PKT(x) ((x & 0x3ff) << 16)
124	#define EP_DATA_FETCH_CTRL(x) ((x & 0x3) << 14)
125	#define EP_AUTO_DATA_DISABLE (0x1 << 13)
126	#define EP_SET_EP_STALL (0x1 << 12)
127	#define EP_SET_EP_NUM(x) ((x & 0xf) << 8)
128	#define EP_SET_TYPE_MASK(x) ((x) << 5)
129	#define EP_TYPE_BULK (0x1)
130	#define EP_TYPE_INT (0x2)
131	#define EP_TYPE_ISO (0x3)
132	#define EP_DIR_OUT (0x1 << 4)
133	#define EP_ALLOCATED_MASK (0x7 << 1)
134	#define EP_ENABLE BIT(0)
135
136	/* EP DMA ctrl reg */
137	#define EP_DMA_CTRL_GET_PROC_STS(x) ((x >> 4) & 0xf)
138	#define EP_DMA_CTRL_STS_RX_IDLE 0x0
139	#define EP_DMA_CTRL_STS_TX_IDLE 0x8
140	#define EP_DMA_CTRL_IN_LONG_MODE (0x1 << 3)
141	#define EP_DMA_CTRL_RESET (0x1 << 2)
142	#define EP_DMA_SINGLE_STAGE (0x1 << 1)
143	#define EP_DMA_DESC_MODE (0x1 << 0)
144
145	/* EP DMA status reg */
146	#define EP_DMA_SET_TX_SIZE(x) ((x & 0x7ff) << 16)
147	#define EP_DMA_GET_TX_SIZE(x) (((x) >> 16) & 0x7ff)
148	#define EP_DMA_GET_RPTR(x) (((x) >> 8) & 0xff)
149	#define EP_DMA_GET_WPTR(x) ((x) & 0xff)
150	#define EP_DMA_SINGLE_KICK (1 << 0) /* WPTR = 1 for single mode */
151
152	/* EP desc reg */
153	#define AST_EP_DMA_DESC_INTR_ENABLE BIT(31)
154	#define AST_EP_DMA_DESC_PID_DATA0 (0 << 14)
155	#define AST_EP_DMA_DESC_PID_DATA2 BIT(14)
156	#define AST_EP_DMA_DESC_PID_DATA1 (2 << 14)
157	#define AST_EP_DMA_DESC_PID_MDATA (3 << 14)
158	#define EP_DESC1_IN_LEN(x) ((x) & 0x1fff)
159	#define AST_EP_DMA_DESC_MAX_LEN (7680) /* Max packet length for transmit in 1 desc */
160
161	struct ast_udc_request {
162	struct usb_request req;
163	struct list_head queue;
164	unsigned mapped:1;
165	unsigned int actual_dma_length;
166	u32 saved_dma_wptr;
167	};
168
169	#define to_ast_req(__req) container_of(__req, struct ast_udc_request, req)
170
171	struct ast_dma_desc {
172	u32 des_0;
173	u32 des_1;
174	};
175
176	struct ast_udc_ep {
177	struct usb_ep ep;
178
179	/* Request queue */
180	struct list_head queue;
181
182	struct ast_udc_dev *udc;
183	void __iomem *ep_reg;
184	void *epn_buf;
185	dma_addr_t epn_buf_dma;
186	const struct usb_endpoint_descriptor *desc;
187
188	/* DMA Descriptors */
189	struct ast_dma_desc *descs;
190	dma_addr_t descs_dma;
191	u32 descs_wptr;
192	u32 chunk_max;
193
194	bool dir_in:1;
195	unsigned stopped:1;
196	bool desc_mode:1;
197	};
198
199	#define to_ast_ep(__ep) container_of(__ep, struct ast_udc_ep, ep)
200
201	struct ast_udc_dev {
202	struct platform_device *pdev;
203	void __iomem *reg;
204	int irq;
205	spinlock_t lock;
206	struct clk *clk;
207	struct work_struct wake_work;
208
209	/* EP0 DMA buffers allocated in one chunk */
210	void *ep0_buf;
211	dma_addr_t ep0_buf_dma;
212	struct ast_udc_ep ep[AST_UDC_NUM_ENDPOINTS];
213
214	struct usb_gadget gadget;
215	struct usb_gadget_driver *driver;
216	void __iomem *creq;
217	enum usb_device_state suspended_from;
218	int desc_mode;
219
220	/* Force full speed only */
221	bool force_usb1:1;
222	unsigned is_control_tx:1;
223	bool wakeup_en:1;
224	};
225
226	#define to_ast_dev(__g) container_of(__g, struct ast_udc_dev, gadget)
227
228	static const char * const ast_ep_name[] = {
229	"ep0", "ep1", "ep2", "ep3", "ep4"
230	};
231
232	#ifdef AST_UDC_DEBUG_ALL
233	#define AST_UDC_DEBUG
234	#define AST_SETUP_DEBUG
235	#define AST_EP_DEBUG
236	#define AST_ISR_DEBUG
237	#endif
238
239	#ifdef AST_SETUP_DEBUG
240	#define SETUP_DBG(u, fmt, ...) \
241	dev_dbg(&(u)->pdev->dev, "%s() " fmt, __func__, ##__VA_ARGS__)
242	#else
243	#define SETUP_DBG(u, fmt, ...)
244	#endif
245
246	#ifdef AST_EP_DEBUG
247	#define EP_DBG(e, fmt, ...) \
248	dev_dbg(&(e)->udc->pdev->dev, "%s():%s " fmt, __func__, \
249	(e)->ep.name, ##__VA_ARGS__)
250	#else
251	#define EP_DBG(ep, fmt, ...) ((void)(ep))
252	#endif
253
254	#ifdef AST_UDC_DEBUG
255	#define UDC_DBG(u, fmt, ...) \
256	dev_dbg(&(u)->pdev->dev, "%s() " fmt, __func__, ##__VA_ARGS__)
257	#else
258	#define UDC_DBG(u, fmt, ...)
259	#endif
260
261	#ifdef AST_ISR_DEBUG
262	#define ISR_DBG(u, fmt, ...) \
263	dev_dbg(&(u)->pdev->dev, "%s() " fmt, __func__, ##__VA_ARGS__)
264	#else
265	#define ISR_DBG(u, fmt, ...)
266	#endif
267
268	/*-------------------------------------------------------------------------*/
269	#define ast_udc_read(udc, offset) \
270	readl((udc)->reg + (offset))
271	#define ast_udc_write(udc, val, offset) \
272	writel((val), (udc)->reg + (offset))
273
274	#define ast_ep_read(ep, reg) \
275	readl((ep)->ep_reg + (reg))
276	#define ast_ep_write(ep, val, reg) \
277	writel((val), (ep)->ep_reg + (reg))
278
279	/*-------------------------------------------------------------------------*/
280
281	static void ast_udc_done(struct ast_udc_ep *ep, struct ast_udc_request *req,
282	int status)
283	{
284	struct ast_udc_dev *udc = ep->udc;
285
286	EP_DBG(ep, "req @%p, len (%d/%d), buf:0x%x, dir:0x%x\n",
287	req, req->req.actual, req->req.length,
288	(u32)req->req.buf, ep->dir_in);
289
290	list_del(entry: &req->queue);
291
292	if (req->req.status == -EINPROGRESS)
293	req->req.status = status;
294	else
295	status = req->req.status;
296
297	if (status && status != -ESHUTDOWN)
298	EP_DBG(ep, "done req:%p, status:%d\n", req, status);
299
300	spin_unlock(lock: &udc->lock);
301	usb_gadget_giveback_request(ep: &ep->ep, req: &req->req);
302	spin_lock(lock: &udc->lock);
303	}
304
305	static void ast_udc_nuke(struct ast_udc_ep *ep, int status)
306	{
307	int count = 0;
308
309	while (!list_empty(head: &ep->queue)) {
310	struct ast_udc_request *req;
311
312	req = list_entry(ep->queue.next, struct ast_udc_request,
313	queue);
314	ast_udc_done(ep, req, status);
315	count++;
316	}
317
318	if (count)
319	EP_DBG(ep, "Nuked %d request(s)\n", count);
320	}
321
322	/*
323	* Stop activity on all endpoints.
324	* Device controller for which EP activity is to be stopped.
325	*
326	* All the endpoints are stopped and any pending transfer requests if any on
327	* the endpoint are terminated.
328	*/
329	static void ast_udc_stop_activity(struct ast_udc_dev *udc)
330	{
331	struct ast_udc_ep *ep;
332	int i;
333
334	for (i = 0; i < AST_UDC_NUM_ENDPOINTS; i++) {
335	ep = &udc->ep[i];
336	ep->stopped = 1;
337	ast_udc_nuke(ep, status: -ESHUTDOWN);
338	}
339	}
340
341	static int ast_udc_ep_enable(struct usb_ep *_ep,
342	const struct usb_endpoint_descriptor *desc)
343	{
344	u16 maxpacket = usb_endpoint_maxp(epd: desc);
345	struct ast_udc_ep *ep = to_ast_ep(_ep);
346	struct ast_udc_dev *udc = ep->udc;
347	u8 epnum = usb_endpoint_num(epd: desc);
348	unsigned long flags;
349	u32 ep_conf = 0;
350	u8 dir_in;
351	u8 type;
352
353	if (!_ep || !ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT ||
354	maxpacket == 0 || maxpacket > ep->ep.maxpacket) {
355	EP_DBG(ep, "Failed, invalid EP enable param\n");
356	return -EINVAL;
357	}
358
359	if (!udc->driver) {
360	EP_DBG(ep, "bogus device state\n");
361	return -ESHUTDOWN;
362	}
363
364	EP_DBG(ep, "maxpacket:0x%x\n", maxpacket);
365
366	spin_lock_irqsave(&udc->lock, flags);
367
368	ep->desc = desc;
369	ep->stopped = 0;
370	ep->ep.maxpacket = maxpacket;
371	ep->chunk_max = AST_EP_DMA_DESC_MAX_LEN;
372
373	if (maxpacket < AST_UDC_EPn_MAX_PACKET)
374	ep_conf = EP_SET_MAX_PKT(maxpacket);
375
376	ep_conf |= EP_SET_EP_NUM(epnum);
377
378	type = usb_endpoint_type(epd: desc);
379	dir_in = usb_endpoint_dir_in(epd: desc);
380	ep->dir_in = dir_in;
381	if (!ep->dir_in)
382	ep_conf |= EP_DIR_OUT;
383
384	EP_DBG(ep, "type %d, dir_in %d\n", type, dir_in);
385	switch (type) {
386	case USB_ENDPOINT_XFER_ISOC:
387	ep_conf |= EP_SET_TYPE_MASK(EP_TYPE_ISO);
388	break;
389
390	case USB_ENDPOINT_XFER_BULK:
391	ep_conf |= EP_SET_TYPE_MASK(EP_TYPE_BULK);
392	break;
393
394	case USB_ENDPOINT_XFER_INT:
395	ep_conf |= EP_SET_TYPE_MASK(EP_TYPE_INT);
396	break;
397	}
398
399	ep->desc_mode = udc->desc_mode && ep->descs_dma && ep->dir_in;
400	if (ep->desc_mode) {
401	ast_ep_write(ep, EP_DMA_CTRL_RESET, AST_UDC_EP_DMA_CTRL);
402	ast_ep_write(ep, 0, AST_UDC_EP_DMA_STS);
403	ast_ep_write(ep, ep->descs_dma, AST_UDC_EP_DMA_BUFF);
404
405	/* Enable Long Descriptor Mode */
406	ast_ep_write(ep, EP_DMA_CTRL_IN_LONG_MODE | EP_DMA_DESC_MODE,
407	AST_UDC_EP_DMA_CTRL);
408
409	ep->descs_wptr = 0;
410
411	} else {
412	ast_ep_write(ep, EP_DMA_CTRL_RESET, AST_UDC_EP_DMA_CTRL);
413	ast_ep_write(ep, EP_DMA_SINGLE_STAGE, AST_UDC_EP_DMA_CTRL);
414	ast_ep_write(ep, 0, AST_UDC_EP_DMA_STS);
415	}
416
417	/* Cleanup data toggle just in case */
418	ast_udc_write(udc, EP_TOGGLE_SET_EPNUM(epnum), AST_VHUB_EP_DATA);
419
420	/* Enable EP */
421	ast_ep_write(ep, ep_conf | EP_ENABLE, AST_UDC_EP_CONFIG);
422
423	EP_DBG(ep, "ep_config: 0x%x\n", ast_ep_read(ep, AST_UDC_EP_CONFIG));
424
425	spin_unlock_irqrestore(lock: &udc->lock, flags);
426
427	return 0;
428	}
429
430	static int ast_udc_ep_disable(struct usb_ep *_ep)
431	{
432	struct ast_udc_ep *ep = to_ast_ep(_ep);
433	struct ast_udc_dev *udc = ep->udc;
434	unsigned long flags;
435
436	spin_lock_irqsave(&udc->lock, flags);
437
438	ep->ep.desc = NULL;
439	ep->stopped = 1;
440
441	ast_udc_nuke(ep, status: -ESHUTDOWN);
442	ast_ep_write(ep, 0, AST_UDC_EP_CONFIG);
443
444	spin_unlock_irqrestore(lock: &udc->lock, flags);
445
446	return 0;
447	}
448
449	static struct usb_request *ast_udc_ep_alloc_request(struct usb_ep *_ep,
450	gfp_t gfp_flags)
451	{
452	struct ast_udc_ep *ep = to_ast_ep(_ep);
453	struct ast_udc_request *req;
454
455	req = kzalloc(sizeof(struct ast_udc_request), gfp_flags);
456	if (!req) {
457	EP_DBG(ep, "request allocation failed\n");
458	return NULL;
459	}
460
461	INIT_LIST_HEAD(list: &req->queue);
462
463	return &req->req;
464	}
465
466	static void ast_udc_ep_free_request(struct usb_ep *_ep,
467	struct usb_request *_req)
468	{
469	struct ast_udc_request *req = to_ast_req(_req);
470
471	kfree(objp: req);
472	}
473
474	static int ast_dma_descriptor_setup(struct ast_udc_ep *ep, u32 dma_buf,
475	u16 tx_len, struct ast_udc_request *req)
476	{
477	struct ast_udc_dev *udc = ep->udc;
478	struct device *dev = &udc->pdev->dev;
479	bool last = false;
480	int chunk, count;
481	u32 offset;
482
483	if (!ep->descs) {
484	dev_warn(dev, "%s: Empty DMA descs list failure\n",
485	ep->ep.name);
486	return -EINVAL;
487	}
488
489	chunk = tx_len;
490	offset = count = 0;
491
492	EP_DBG(ep, "req @%p, %s:%d, %s:0x%x, %s:0x%x\n", req,
493	"wptr", ep->descs_wptr, "dma_buf", dma_buf,
494	"tx_len", tx_len);
495
496	/* Create Descriptor Lists */
497	while (chunk >= 0 && !last && count < AST_UDC_DESCS_COUNT) {
498
499	ep->descs[ep->descs_wptr].des_0 = dma_buf + offset;
500
501	if (chunk > ep->chunk_max) {
502	ep->descs[ep->descs_wptr].des_1 = ep->chunk_max;
503	} else {
504	ep->descs[ep->descs_wptr].des_1 = chunk;
505	last = true;
506	}
507
508	chunk -= ep->chunk_max;
509
510	EP_DBG(ep, "descs[%d]: 0x%x 0x%x\n",
511	ep->descs_wptr,
512	ep->descs[ep->descs_wptr].des_0,
513	ep->descs[ep->descs_wptr].des_1);
514
515	if (count == 0)
516	req->saved_dma_wptr = ep->descs_wptr;
517
518	ep->descs_wptr++;
519	count++;
520
521	if (ep->descs_wptr >= AST_UDC_DESCS_COUNT)
522	ep->descs_wptr = 0;
523
524	offset = ep->chunk_max * count;
525	}
526
527	return 0;
528	}
529
530	static void ast_udc_epn_kick(struct ast_udc_ep *ep, struct ast_udc_request *req)
531	{
532	u32 tx_len;
533	u32 last;
534
535	last = req->req.length - req->req.actual;
536	tx_len = last > ep->ep.maxpacket ? ep->ep.maxpacket : last;
537
538	EP_DBG(ep, "kick req @%p, len:%d, dir:%d\n",
539	req, tx_len, ep->dir_in);
540
541	ast_ep_write(ep, req->req.dma + req->req.actual, AST_UDC_EP_DMA_BUFF);
542
543	/* Start DMA */
544	ast_ep_write(ep, EP_DMA_SET_TX_SIZE(tx_len), AST_UDC_EP_DMA_STS);
545	ast_ep_write(ep, EP_DMA_SET_TX_SIZE(tx_len) | EP_DMA_SINGLE_KICK,
546	AST_UDC_EP_DMA_STS);
547	}
548
549	static void ast_udc_epn_kick_desc(struct ast_udc_ep *ep,
550	struct ast_udc_request *req)
551	{
552	u32 descs_max_size;
553	u32 tx_len;
554	u32 last;
555
556	descs_max_size = AST_EP_DMA_DESC_MAX_LEN * AST_UDC_DESCS_COUNT;
557
558	last = req->req.length - req->req.actual;
559	tx_len = last > descs_max_size ? descs_max_size : last;
560
561	EP_DBG(ep, "kick req @%p, %s:%d, %s:0x%x, %s:0x%x (%d/%d), %s:0x%x\n",
562	req, "tx_len", tx_len, "dir_in", ep->dir_in,
563	"dma", req->req.dma + req->req.actual,
564	req->req.actual, req->req.length,
565	"descs_max_size", descs_max_size);
566
567	if (!ast_dma_descriptor_setup(ep, dma_buf: req->req.dma + req->req.actual,
568	tx_len, req))
569	req->actual_dma_length += tx_len;
570
571	/* make sure CPU done everything before triggering DMA */
572	mb();
573
574	ast_ep_write(ep, ep->descs_wptr, AST_UDC_EP_DMA_STS);
575
576	EP_DBG(ep, "descs_wptr:%d, dstat:0x%x, dctrl:0x%x\n",
577	ep->descs_wptr,
578	ast_ep_read(ep, AST_UDC_EP_DMA_STS),
579	ast_ep_read(ep, AST_UDC_EP_DMA_CTRL));
580	}
581
582	static void ast_udc_ep0_queue(struct ast_udc_ep *ep,
583	struct ast_udc_request *req)
584	{
585	struct ast_udc_dev *udc = ep->udc;
586	u32 tx_len;
587	u32 last;
588
589	last = req->req.length - req->req.actual;
590	tx_len = last > ep->ep.maxpacket ? ep->ep.maxpacket : last;
591
592	ast_udc_write(udc, req->req.dma + req->req.actual,
593	AST_UDC_EP0_DATA_BUFF);
594
595	if (ep->dir_in) {
596	/* IN requests, send data */
597	SETUP_DBG(udc, "IN: %s:0x%x, %s:0x%x, %s:%d (%d/%d), %s:%d\n",
598	"buf", (u32)req->req.buf,
599	"dma", req->req.dma + req->req.actual,
600	"tx_len", tx_len,
601	req->req.actual, req->req.length,
602	"dir_in", ep->dir_in);
603
604	req->req.actual += tx_len;
605	ast_udc_write(udc, EP0_TX_LEN(tx_len), AST_UDC_EP0_CTRL);
606	ast_udc_write(udc, EP0_TX_LEN(tx_len) | EP0_TX_BUFF_RDY,
607	AST_UDC_EP0_CTRL);
608
609	} else {
610	/* OUT requests, receive data */
611	SETUP_DBG(udc, "OUT: %s:%x, %s:%x, %s:(%d/%d), %s:%d\n",
612	"buf", (u32)req->req.buf,
613	"dma", req->req.dma + req->req.actual,
614	"len", req->req.actual, req->req.length,
615	"dir_in", ep->dir_in);
616
617	if (!req->req.length) {
618	/* 0 len request, send tx as completion */
619	ast_udc_write(udc, EP0_TX_BUFF_RDY, AST_UDC_EP0_CTRL);
620	ep->dir_in = 0x1;
621	} else
622	ast_udc_write(udc, EP0_RX_BUFF_RDY, AST_UDC_EP0_CTRL);
623	}
624	}
625
626	static int ast_udc_ep_queue(struct usb_ep *_ep, struct usb_request *_req,
627	gfp_t gfp_flags)
628	{
629	struct ast_udc_request *req = to_ast_req(_req);
630	struct ast_udc_ep *ep = to_ast_ep(_ep);
631	struct ast_udc_dev *udc = ep->udc;
632	struct device *dev = &udc->pdev->dev;
633	unsigned long flags;
634	int rc;
635
636	if (unlikely(!_req || !_req->complete || !_req->buf || !_ep)) {
637	dev_warn(dev, "Invalid EP request !\n");
638	return -EINVAL;
639	}
640
641	if (ep->stopped) {
642	dev_warn(dev, "%s is already stopped !\n", _ep->name);
643	return -ESHUTDOWN;
644	}
645
646	spin_lock_irqsave(&udc->lock, flags);
647
648	list_add_tail(new: &req->queue, head: &ep->queue);
649
650	req->req.actual = 0;
651	req->req.status = -EINPROGRESS;
652	req->actual_dma_length = 0;
653
654	rc = usb_gadget_map_request(gadget: &udc->gadget, req: &req->req, is_in: ep->dir_in);
655	if (rc) {
656	EP_DBG(ep, "Request mapping failure %d\n", rc);
657	dev_warn(dev, "Request mapping failure %d\n", rc);
658	goto end;
659	}
660
661	EP_DBG(ep, "enqueue req @%p\n", req);
662	EP_DBG(ep, "l=%d, dma:0x%x, zero:%d, is_in:%d\n",
663	_req->length, _req->dma, _req->zero, ep->dir_in);
664
665	/* EP0 request enqueue */
666	if (ep->ep.desc == NULL) {
667	if ((req->req.dma % 4) != 0) {
668	dev_warn(dev, "EP0 req dma alignment error\n");
669	rc = -ESHUTDOWN;
670	goto end;
671	}
672
673	ast_udc_ep0_queue(ep, req);
674	goto end;
675	}
676
677	/* EPn request enqueue */
678	if (list_is_singular(head: &ep->queue)) {
679	if (ep->desc_mode)
680	ast_udc_epn_kick_desc(ep, req);
681	else
682	ast_udc_epn_kick(ep, req);
683	}
684
685	end:
686	spin_unlock_irqrestore(lock: &udc->lock, flags);
687
688	return rc;
689	}
690
691	static int ast_udc_ep_dequeue(struct usb_ep *_ep, struct usb_request *_req)
692	{
693	struct ast_udc_ep *ep = to_ast_ep(_ep);
694	struct ast_udc_dev *udc = ep->udc;
695	struct ast_udc_request *req;
696	unsigned long flags;
697	int rc = 0;
698
699	spin_lock_irqsave(&udc->lock, flags);
700
701	/* make sure it's actually queued on this endpoint */
702	list_for_each_entry(req, &ep->queue, queue) {
703	if (&req->req == _req) {
704	list_del_init(entry: &req->queue);
705	ast_udc_done(ep, req, status: -ESHUTDOWN);
706	_req->status = -ECONNRESET;
707	break;
708	}
709	}
710
711	/* dequeue request not found */
712	if (&req->req != _req)
713	rc = -EINVAL;
714
715	spin_unlock_irqrestore(lock: &udc->lock, flags);
716
717	return rc;
718	}
719
720	static int ast_udc_ep_set_halt(struct usb_ep *_ep, int value)
721	{
722	struct ast_udc_ep *ep = to_ast_ep(_ep);
723	struct ast_udc_dev *udc = ep->udc;
724	unsigned long flags;
725	int epnum;
726	u32 ctrl;
727
728	EP_DBG(ep, "val:%d\n", value);
729
730	spin_lock_irqsave(&udc->lock, flags);
731
732	epnum = usb_endpoint_num(epd: ep->desc);
733
734	/* EP0 */
735	if (epnum == 0) {
736	ctrl = ast_udc_read(udc, AST_UDC_EP0_CTRL);
737	if (value)
738	ctrl |= EP0_STALL;
739	else
740	ctrl &= ~EP0_STALL;
741
742	ast_udc_write(udc, ctrl, AST_UDC_EP0_CTRL);
743
744	} else {
745	/* EPn */
746	ctrl = ast_udc_read(udc, AST_UDC_EP_CONFIG);
747	if (value)
748	ctrl |= EP_SET_EP_STALL;
749	else
750	ctrl &= ~EP_SET_EP_STALL;
751
752	ast_ep_write(ep, ctrl, AST_UDC_EP_CONFIG);
753
754	/* only epn is stopped and waits for clear */
755	ep->stopped = value ? 1 : 0;
756	}
757
758	spin_unlock_irqrestore(lock: &udc->lock, flags);
759
760	return 0;
761	}
762
763	static const struct usb_ep_ops ast_udc_ep_ops = {
764	.enable = ast_udc_ep_enable,
765	.disable = ast_udc_ep_disable,
766	.alloc_request = ast_udc_ep_alloc_request,
767	.free_request = ast_udc_ep_free_request,
768	.queue = ast_udc_ep_queue,
769	.dequeue = ast_udc_ep_dequeue,
770	.set_halt = ast_udc_ep_set_halt,
771	/* there's only imprecise fifo status reporting */
772	};
773
774	static void ast_udc_ep0_rx(struct ast_udc_dev *udc)
775	{
776	ast_udc_write(udc, udc->ep0_buf_dma, AST_UDC_EP0_DATA_BUFF);
777	ast_udc_write(udc, EP0_RX_BUFF_RDY, AST_UDC_EP0_CTRL);
778	}
779
780	static void ast_udc_ep0_tx(struct ast_udc_dev *udc)
781	{
782	ast_udc_write(udc, udc->ep0_buf_dma, AST_UDC_EP0_DATA_BUFF);
783	ast_udc_write(udc, EP0_TX_BUFF_RDY, AST_UDC_EP0_CTRL);
784	}
785
786	static void ast_udc_ep0_out(struct ast_udc_dev *udc)
787	{
788	struct device *dev = &udc->pdev->dev;
789	struct ast_udc_ep *ep = &udc->ep[0];
790	struct ast_udc_request *req;
791	u16 rx_len;
792
793	if (list_empty(head: &ep->queue))
794	return;
795
796	req = list_entry(ep->queue.next, struct ast_udc_request, queue);
797
798	rx_len = EP0_GET_RX_LEN(ast_udc_read(udc, AST_UDC_EP0_CTRL));
799	req->req.actual += rx_len;
800
801	SETUP_DBG(udc, "req %p (%d/%d)\n", req,
802	req->req.actual, req->req.length);
803
804	if ((rx_len < ep->ep.maxpacket) ||
805	(req->req.actual == req->req.length)) {
806	ast_udc_ep0_tx(udc);
807	if (!ep->dir_in)
808	ast_udc_done(ep, req, status: 0);
809
810	} else {
811	if (rx_len > req->req.length) {
812	// Issue Fix
813	dev_warn(dev, "Something wrong (%d/%d)\n",
814	req->req.actual, req->req.length);
815	ast_udc_ep0_tx(udc);
816	ast_udc_done(ep, req, status: 0);
817	return;
818	}
819
820	ep->dir_in = 0;
821
822	/* More works */
823	ast_udc_ep0_queue(ep, req);
824	}
825	}
826
827	static void ast_udc_ep0_in(struct ast_udc_dev *udc)
828	{
829	struct ast_udc_ep *ep = &udc->ep[0];
830	struct ast_udc_request *req;
831
832	if (list_empty(head: &ep->queue)) {
833	if (udc->is_control_tx) {
834	ast_udc_ep0_rx(udc);
835	udc->is_control_tx = 0;
836	}
837
838	return;
839	}
840
841	req = list_entry(ep->queue.next, struct ast_udc_request, queue);
842
843	SETUP_DBG(udc, "req %p (%d/%d)\n", req,
844	req->req.actual, req->req.length);
845
846	if (req->req.length == req->req.actual) {
847	if (req->req.length)
848	ast_udc_ep0_rx(udc);
849
850	if (ep->dir_in)
851	ast_udc_done(ep, req, status: 0);
852
853	} else {
854	/* More works */
855	ast_udc_ep0_queue(ep, req);
856	}
857	}
858
859	static void ast_udc_epn_handle(struct ast_udc_dev *udc, u16 ep_num)
860	{
861	struct ast_udc_ep *ep = &udc->ep[ep_num];
862	struct ast_udc_request *req;
863	u16 len = 0;
864
865	if (list_empty(head: &ep->queue))
866	return;
867
868	req = list_first_entry(&ep->queue, struct ast_udc_request, queue);
869
870	len = EP_DMA_GET_TX_SIZE(ast_ep_read(ep, AST_UDC_EP_DMA_STS));
871	req->req.actual += len;
872
873	EP_DBG(ep, "req @%p, length:(%d/%d), %s:0x%x\n", req,
874	req->req.actual, req->req.length, "len", len);
875
876	/* Done this request */
877	if (req->req.length == req->req.actual) {
878	ast_udc_done(ep, req, status: 0);
879	req = list_first_entry_or_null(&ep->queue,
880	struct ast_udc_request,
881	queue);
882
883	} else {
884	/* Check for short packet */
885	if (len < ep->ep.maxpacket) {
886	ast_udc_done(ep, req, status: 0);
887	req = list_first_entry_or_null(&ep->queue,
888	struct ast_udc_request,
889	queue);
890	}
891	}
892
893	/* More requests */
894	if (req)
895	ast_udc_epn_kick(ep, req);
896	}
897
898	static void ast_udc_epn_handle_desc(struct ast_udc_dev *udc, u16 ep_num)
899	{
900	struct ast_udc_ep *ep = &udc->ep[ep_num];
901	struct device *dev = &udc->pdev->dev;
902	struct ast_udc_request *req;
903	u32 proc_sts, wr_ptr, rd_ptr;
904	u32 len_in_desc, ctrl;
905	u16 total_len = 0;
906	int i;
907
908	if (list_empty(head: &ep->queue)) {
909	dev_warn(dev, "%s request queue empty!\n", ep->ep.name);
910	return;
911	}
912
913	req = list_first_entry(&ep->queue, struct ast_udc_request, queue);
914
915	ctrl = ast_ep_read(ep, AST_UDC_EP_DMA_CTRL);
916	proc_sts = EP_DMA_CTRL_GET_PROC_STS(ctrl);
917
918	/* Check processing status is idle */
919	if (proc_sts != EP_DMA_CTRL_STS_RX_IDLE &&
920	proc_sts != EP_DMA_CTRL_STS_TX_IDLE) {
921	dev_warn(dev, "EP DMA CTRL: 0x%x, PS:0x%x\n",
922	ast_ep_read(ep, AST_UDC_EP_DMA_CTRL),
923	proc_sts);
924	return;
925	}
926
927	ctrl = ast_ep_read(ep, AST_UDC_EP_DMA_STS);
928	rd_ptr = EP_DMA_GET_RPTR(ctrl);
929	wr_ptr = EP_DMA_GET_WPTR(ctrl);
930
931	if (rd_ptr != wr_ptr) {
932	dev_warn(dev, "desc list is not empty ! %s:%d, %s:%d\n",
933	"rptr", rd_ptr, "wptr", wr_ptr);
934	return;
935	}
936
937	EP_DBG(ep, "rd_ptr:%d, wr_ptr:%d\n", rd_ptr, wr_ptr);
938	i = req->saved_dma_wptr;
939
940	do {
941	len_in_desc = EP_DESC1_IN_LEN(ep->descs[i].des_1);
942	EP_DBG(ep, "desc[%d] len: %d\n", i, len_in_desc);
943	total_len += len_in_desc;
944	i++;
945	if (i >= AST_UDC_DESCS_COUNT)
946	i = 0;
947
948	} while (i != wr_ptr);
949
950	req->req.actual += total_len;
951
952	EP_DBG(ep, "req @%p, length:(%d/%d), %s:0x%x\n", req,
953	req->req.actual, req->req.length, "len", total_len);
954
955	/* Done this request */
956	if (req->req.length == req->req.actual) {
957	ast_udc_done(ep, req, status: 0);
958	req = list_first_entry_or_null(&ep->queue,
959	struct ast_udc_request,
960	queue);
961
962	} else {
963	/* Check for short packet */
964	if (total_len < ep->ep.maxpacket) {
965	ast_udc_done(ep, req, status: 0);
966	req = list_first_entry_or_null(&ep->queue,
967	struct ast_udc_request,
968	queue);
969	}
970	}
971
972	/* More requests & dma descs not setup yet */
973	if (req && (req->actual_dma_length == req->req.actual)) {
974	EP_DBG(ep, "More requests\n");
975	ast_udc_epn_kick_desc(ep, req);
976	}
977	}
978
979	static void ast_udc_ep0_data_tx(struct ast_udc_dev *udc, u8 *tx_data, u32 len)
980	{
981	if (len) {
982	memcpy(udc->ep0_buf, tx_data, len);
983
984	ast_udc_write(udc, udc->ep0_buf_dma, AST_UDC_EP0_DATA_BUFF);
985	ast_udc_write(udc, EP0_TX_LEN(len), AST_UDC_EP0_CTRL);
986	ast_udc_write(udc, EP0_TX_LEN(len) | EP0_TX_BUFF_RDY,
987	AST_UDC_EP0_CTRL);
988	udc->is_control_tx = 1;
989
990	} else
991	ast_udc_write(udc, EP0_TX_BUFF_RDY, AST_UDC_EP0_CTRL);
992	}
993
994	static void ast_udc_getstatus(struct ast_udc_dev *udc)
995	{
996	struct usb_ctrlrequest crq;
997	struct ast_udc_ep *ep;
998	u16 status = 0;
999	u16 epnum = 0;
1000
1001	memcpy_fromio(&crq, udc->creq, sizeof(crq));
1002
1003	switch (crq.bRequestType & USB_RECIP_MASK) {
1004	case USB_RECIP_DEVICE:
1005	/* Get device status */
1006	status = 1 << USB_DEVICE_SELF_POWERED;
1007	break;
1008	case USB_RECIP_INTERFACE:
1009	break;
1010	case USB_RECIP_ENDPOINT:
1011	epnum = crq.wIndex & USB_ENDPOINT_NUMBER_MASK;
1012	if (epnum >= AST_UDC_NUM_ENDPOINTS)
1013	goto stall;
1014	status = udc->ep[epnum].stopped;
1015	break;
1016	default:
1017	goto stall;
1018	}
1019
1020	ep = &udc->ep[epnum];
1021	EP_DBG(ep, "status: 0x%x\n", status);
1022	ast_udc_ep0_data_tx(udc, tx_data: (u8 *)&status, len: sizeof(status));
1023
1024	return;
1025
1026	stall:
1027	EP_DBG(ep, "Can't respond request\n");
1028	ast_udc_write(udc, ast_udc_read(udc, AST_UDC_EP0_CTRL) | EP0_STALL,
1029	AST_UDC_EP0_CTRL);
1030	}
1031
1032	static void ast_udc_ep0_handle_setup(struct ast_udc_dev *udc)
1033	{
1034	struct ast_udc_ep *ep = &udc->ep[0];
1035	struct ast_udc_request *req;
1036	struct usb_ctrlrequest crq;
1037	int req_num = 0;
1038	int rc = 0;
1039	u32 reg;
1040
1041	memcpy_fromio(&crq, udc->creq, sizeof(crq));
1042
1043	SETUP_DBG(udc, "SETUP packet: %02x/%02x/%04x/%04x/%04x\n",
1044	crq.bRequestType, crq.bRequest, le16_to_cpu(crq.wValue),
1045	le16_to_cpu(crq.wIndex), le16_to_cpu(crq.wLength));
1046
1047	/*
1048	* Cleanup ep0 request(s) in queue because
1049	* there is a new control setup comes.
1050	*/
1051	list_for_each_entry(req, &udc->ep[0].queue, queue) {
1052	req_num++;
1053	EP_DBG(ep, "there is req %p in ep0 queue !\n", req);
1054	}
1055
1056	if (req_num)
1057	ast_udc_nuke(ep: &udc->ep[0], status: -ETIMEDOUT);
1058
1059	udc->ep[0].dir_in = crq.bRequestType & USB_DIR_IN;
1060
1061	if ((crq.bRequestType & USB_TYPE_MASK) == USB_TYPE_STANDARD) {
1062	switch (crq.bRequest) {
1063	case USB_REQ_SET_ADDRESS:
1064	if (ast_udc_read(udc, AST_UDC_STS) & UDC_STS_HIGHSPEED)
1065	udc->gadget.speed = USB_SPEED_HIGH;
1066	else
1067	udc->gadget.speed = USB_SPEED_FULL;
1068
1069	SETUP_DBG(udc, "set addr: 0x%x\n", crq.wValue);
1070	reg = ast_udc_read(udc, AST_UDC_CONFIG);
1071	reg &= ~UDC_CFG_ADDR_MASK;
1072	reg |= UDC_CFG_SET_ADDR(crq.wValue);
1073	ast_udc_write(udc, reg, AST_UDC_CONFIG);
1074	goto req_complete;
1075
1076	case USB_REQ_CLEAR_FEATURE:
1077	SETUP_DBG(udc, "ep0: CLEAR FEATURE\n");
1078	goto req_driver;
1079
1080	case USB_REQ_SET_FEATURE:
1081	SETUP_DBG(udc, "ep0: SET FEATURE\n");
1082	goto req_driver;
1083
1084	case USB_REQ_GET_STATUS:
1085	ast_udc_getstatus(udc);
1086	return;
1087
1088	default:
1089	goto req_driver;
1090	}
1091
1092	}
1093
1094	req_driver:
1095	if (udc->driver) {
1096	SETUP_DBG(udc, "Forwarding %s to gadget...\n",
1097	udc->gadget.name);
1098
1099	spin_unlock(lock: &udc->lock);
1100	rc = udc->driver->setup(&udc->gadget, &crq);
1101	spin_lock(lock: &udc->lock);
1102
1103	} else {
1104	SETUP_DBG(udc, "No gadget for request !\n");
1105	}
1106
1107	if (rc >= 0)
1108	return;
1109
1110	/* Stall if gadget failed */
1111	SETUP_DBG(udc, "Stalling, rc:0x%x\n", rc);
1112	ast_udc_write(udc, ast_udc_read(udc, AST_UDC_EP0_CTRL) | EP0_STALL,
1113	AST_UDC_EP0_CTRL);
1114	return;
1115
1116	req_complete:
1117	SETUP_DBG(udc, "ep0: Sending IN status without data\n");
1118	ast_udc_write(udc, EP0_TX_BUFF_RDY, AST_UDC_EP0_CTRL);
1119	}
1120
1121	static irqreturn_t ast_udc_isr(int irq, void *data)
1122	{
1123	struct ast_udc_dev *udc = (struct ast_udc_dev *)data;
1124	struct ast_udc_ep *ep;
1125	u32 isr, ep_isr;
1126	int i;
1127
1128	spin_lock(lock: &udc->lock);
1129
1130	isr = ast_udc_read(udc, AST_UDC_ISR);
1131	if (!isr)
1132	goto done;
1133
1134	/* Ack interrupts */
1135	ast_udc_write(udc, isr, AST_UDC_ISR);
1136
1137	if (isr & UDC_IRQ_BUS_RESET) {
1138	ISR_DBG(udc, "UDC_IRQ_BUS_RESET\n");
1139	udc->gadget.speed = USB_SPEED_UNKNOWN;
1140
1141	ep = &udc->ep[1];
1142	EP_DBG(ep, "dctrl:0x%x\n",
1143	ast_ep_read(ep, AST_UDC_EP_DMA_CTRL));
1144
1145	if (udc->driver && udc->driver->reset) {
1146	spin_unlock(lock: &udc->lock);
1147	udc->driver->reset(&udc->gadget);
1148	spin_lock(lock: &udc->lock);
1149	}
1150	}
1151
1152	if (isr & UDC_IRQ_BUS_SUSPEND) {
1153	ISR_DBG(udc, "UDC_IRQ_BUS_SUSPEND\n");
1154	udc->suspended_from = udc->gadget.state;
1155	usb_gadget_set_state(gadget: &udc->gadget, state: USB_STATE_SUSPENDED);
1156
1157	if (udc->driver && udc->driver->suspend) {
1158	spin_unlock(lock: &udc->lock);
1159	udc->driver->suspend(&udc->gadget);
1160	spin_lock(lock: &udc->lock);
1161	}
1162	}
1163
1164	if (isr & UDC_IRQ_BUS_RESUME) {
1165	ISR_DBG(udc, "UDC_IRQ_BUS_RESUME\n");
1166	usb_gadget_set_state(gadget: &udc->gadget, state: udc->suspended_from);
1167
1168	if (udc->driver && udc->driver->resume) {
1169	spin_unlock(lock: &udc->lock);
1170	udc->driver->resume(&udc->gadget);
1171	spin_lock(lock: &udc->lock);
1172	}
1173	}
1174
1175	if (isr & UDC_IRQ_EP0_IN_ACK_STALL) {
1176	ISR_DBG(udc, "UDC_IRQ_EP0_IN_ACK_STALL\n");
1177	ast_udc_ep0_in(udc);
1178	}
1179
1180	if (isr & UDC_IRQ_EP0_OUT_ACK_STALL) {
1181	ISR_DBG(udc, "UDC_IRQ_EP0_OUT_ACK_STALL\n");
1182	ast_udc_ep0_out(udc);
1183	}
1184
1185	if (isr & UDC_IRQ_EP0_SETUP) {
1186	ISR_DBG(udc, "UDC_IRQ_EP0_SETUP\n");
1187	ast_udc_ep0_handle_setup(udc);
1188	}
1189
1190	if (isr & UDC_IRQ_EP_POOL_ACK_STALL) {
1191	ISR_DBG(udc, "UDC_IRQ_EP_POOL_ACK_STALL\n");
1192	ep_isr = ast_udc_read(udc, AST_UDC_EP_ACK_ISR);
1193
1194	/* Ack EP interrupts */
1195	ast_udc_write(udc, ep_isr, AST_UDC_EP_ACK_ISR);
1196
1197	/* Handle each EP */
1198	for (i = 0; i < AST_UDC_NUM_ENDPOINTS - 1; i++) {
1199	if (ep_isr & (0x1 << i)) {
1200	ep = &udc->ep[i + 1];
1201	if (ep->desc_mode)
1202	ast_udc_epn_handle_desc(udc, ep_num: i + 1);
1203	else
1204	ast_udc_epn_handle(udc, ep_num: i + 1);
1205	}
1206	}
1207	}
1208
1209	done:
1210	spin_unlock(lock: &udc->lock);
1211	return IRQ_HANDLED;
1212	}
1213
1214	static int ast_udc_gadget_getframe(struct usb_gadget *gadget)
1215	{
1216	struct ast_udc_dev *udc = to_ast_dev(gadget);
1217
1218	return (ast_udc_read(udc, AST_UDC_STS) >> 16) & 0x7ff;
1219	}
1220
1221	static void ast_udc_wake_work(struct work_struct *work)
1222	{
1223	struct ast_udc_dev *udc = container_of(work, struct ast_udc_dev,
1224	wake_work);
1225	unsigned long flags;
1226	u32 ctrl;
1227
1228	spin_lock_irqsave(&udc->lock, flags);
1229
1230	UDC_DBG(udc, "Wakeup Host !\n");
1231	ctrl = ast_udc_read(udc, AST_UDC_FUNC_CTRL);
1232	ast_udc_write(udc, ctrl | USB_REMOTE_WAKEUP_EN, AST_UDC_FUNC_CTRL);
1233
1234	spin_unlock_irqrestore(lock: &udc->lock, flags);
1235	}
1236
1237	static void ast_udc_wakeup_all(struct ast_udc_dev *udc)
1238	{
1239	/*
1240	* A device is trying to wake the world, because this
1241	* can recurse into the device, we break the call chain
1242	* using a work queue
1243	*/
1244	schedule_work(work: &udc->wake_work);
1245	}
1246
1247	static int ast_udc_wakeup(struct usb_gadget *gadget)
1248	{
1249	struct ast_udc_dev *udc = to_ast_dev(gadget);
1250	unsigned long flags;
1251	int rc = 0;
1252
1253	spin_lock_irqsave(&udc->lock, flags);
1254
1255	if (!udc->wakeup_en) {
1256	UDC_DBG(udc, "Remote Wakeup is disabled\n");
1257	rc = -EINVAL;
1258	goto err;
1259	}
1260
1261	UDC_DBG(udc, "Device initiated wakeup\n");
1262	ast_udc_wakeup_all(udc);
1263
1264	err:
1265	spin_unlock_irqrestore(lock: &udc->lock, flags);
1266	return rc;
1267	}
1268
1269	/*
1270	* Activate/Deactivate link with host
1271	*/
1272	static int ast_udc_pullup(struct usb_gadget *gadget, int is_on)
1273	{
1274	struct ast_udc_dev *udc = to_ast_dev(gadget);
1275	unsigned long flags;
1276	u32 ctrl;
1277
1278	spin_lock_irqsave(&udc->lock, flags);
1279
1280	UDC_DBG(udc, "is_on: %d\n", is_on);
1281	if (is_on)
1282	ctrl = ast_udc_read(udc, AST_UDC_FUNC_CTRL) | USB_UPSTREAM_EN;
1283	else
1284	ctrl = ast_udc_read(udc, AST_UDC_FUNC_CTRL) & ~USB_UPSTREAM_EN;
1285
1286	ast_udc_write(udc, ctrl, AST_UDC_FUNC_CTRL);
1287
1288	spin_unlock_irqrestore(lock: &udc->lock, flags);
1289
1290	return 0;
1291	}
1292
1293	static int ast_udc_start(struct usb_gadget *gadget,
1294	struct usb_gadget_driver *driver)
1295	{
1296	struct ast_udc_dev *udc = to_ast_dev(gadget);
1297	struct ast_udc_ep *ep;
1298	unsigned long flags;
1299	int i;
1300
1301	spin_lock_irqsave(&udc->lock, flags);
1302
1303	UDC_DBG(udc, "\n");
1304	udc->driver = driver;
1305	udc->gadget.dev.of_node = udc->pdev->dev.of_node;
1306
1307	for (i = 0; i < AST_UDC_NUM_ENDPOINTS; i++) {
1308	ep = &udc->ep[i];
1309	ep->stopped = 0;
1310	}
1311
1312	spin_unlock_irqrestore(lock: &udc->lock, flags);
1313
1314	return 0;
1315	}
1316
1317	static int ast_udc_stop(struct usb_gadget *gadget)
1318	{
1319	struct ast_udc_dev *udc = to_ast_dev(gadget);
1320	unsigned long flags;
1321	u32 ctrl;
1322
1323	spin_lock_irqsave(&udc->lock, flags);
1324
1325	UDC_DBG(udc, "\n");
1326	ctrl = ast_udc_read(udc, AST_UDC_FUNC_CTRL) & ~USB_UPSTREAM_EN;
1327	ast_udc_write(udc, ctrl, AST_UDC_FUNC_CTRL);
1328
1329	udc->gadget.speed = USB_SPEED_UNKNOWN;
1330	udc->driver = NULL;
1331
1332	ast_udc_stop_activity(udc);
1333	usb_gadget_set_state(gadget: &udc->gadget, state: USB_STATE_NOTATTACHED);
1334
1335	spin_unlock_irqrestore(lock: &udc->lock, flags);
1336
1337	return 0;
1338	}
1339
1340	static const struct usb_gadget_ops ast_udc_ops = {
1341	.get_frame = ast_udc_gadget_getframe,
1342	.wakeup = ast_udc_wakeup,
1343	.pullup = ast_udc_pullup,
1344	.udc_start = ast_udc_start,
1345	.udc_stop = ast_udc_stop,
1346	};
1347
1348	/*
1349	* Support 1 Control Endpoint.
1350	* Support multiple programmable endpoints that can be configured to
1351	* Bulk IN/OUT, Interrupt IN/OUT, and Isochronous IN/OUT type endpoint.
1352	*/
1353	static void ast_udc_init_ep(struct ast_udc_dev *udc)
1354	{
1355	struct ast_udc_ep *ep;
1356	int i;
1357
1358	for (i = 0; i < AST_UDC_NUM_ENDPOINTS; i++) {
1359	ep = &udc->ep[i];
1360	ep->ep.name = ast_ep_name[i];
1361	if (i == 0) {
1362	ep->ep.caps.type_control = true;
1363	} else {
1364	ep->ep.caps.type_iso = true;
1365	ep->ep.caps.type_bulk = true;
1366	ep->ep.caps.type_int = true;
1367	}
1368	ep->ep.caps.dir_in = true;
1369	ep->ep.caps.dir_out = true;
1370
1371	ep->ep.ops = &ast_udc_ep_ops;
1372	ep->udc = udc;
1373
1374	INIT_LIST_HEAD(list: &ep->queue);
1375
1376	if (i == 0) {
1377	usb_ep_set_maxpacket_limit(ep: &ep->ep,
1378	AST_UDC_EP0_MAX_PACKET);
1379	continue;
1380	}
1381
1382	ep->ep_reg = udc->reg + AST_UDC_EP_BASE +
1383	(AST_UDC_EP_OFFSET * (i - 1));
1384
1385	ep->epn_buf = udc->ep0_buf + (i * AST_UDC_EP_DMA_SIZE);
1386	ep->epn_buf_dma = udc->ep0_buf_dma + (i * AST_UDC_EP_DMA_SIZE);
1387	usb_ep_set_maxpacket_limit(ep: &ep->ep, AST_UDC_EPn_MAX_PACKET);
1388
1389	ep->descs = ep->epn_buf + AST_UDC_EPn_MAX_PACKET;
1390	ep->descs_dma = ep->epn_buf_dma + AST_UDC_EPn_MAX_PACKET;
1391	ep->descs_wptr = 0;
1392
1393	list_add_tail(new: &ep->ep.ep_list, head: &udc->gadget.ep_list);
1394	}
1395	}
1396
1397	static void ast_udc_init_dev(struct ast_udc_dev *udc)
1398	{
1399	INIT_WORK(&udc->wake_work, ast_udc_wake_work);
1400	}
1401
1402	static void ast_udc_init_hw(struct ast_udc_dev *udc)
1403	{
1404	u32 ctrl;
1405
1406	/* Enable PHY */
1407	ctrl = USB_PHY_CLK_EN | USB_PHY_RESET_DIS;
1408	ast_udc_write(udc, ctrl, AST_UDC_FUNC_CTRL);
1409
1410	udelay(usec: 1);
1411	ast_udc_write(udc, 0, AST_UDC_DEV_RESET);
1412
1413	/* Set descriptor ring size */
1414	if (AST_UDC_DESCS_COUNT == 256) {
1415	ctrl |= USB_EP_LONG_DESC;
1416	ast_udc_write(udc, ctrl, AST_UDC_FUNC_CTRL);
1417	}
1418
1419	/* Mask & ack all interrupts before installing the handler */
1420	ast_udc_write(udc, 0, AST_UDC_IER);
1421	ast_udc_write(udc, UDC_IRQ_ACK_ALL, AST_UDC_ISR);
1422
1423	/* Enable some interrupts */
1424	ctrl = UDC_IRQ_EP_POOL_ACK_STALL | UDC_IRQ_BUS_RESUME |
1425	UDC_IRQ_BUS_SUSPEND | UDC_IRQ_BUS_RESET |
1426	UDC_IRQ_EP0_IN_ACK_STALL | UDC_IRQ_EP0_OUT_ACK_STALL |
1427	UDC_IRQ_EP0_SETUP;
1428	ast_udc_write(udc, ctrl, AST_UDC_IER);
1429
1430	/* Cleanup and enable ep ACK interrupts */
1431	ast_udc_write(udc, UDC_IRQ_EP_ACK_ALL, AST_UDC_EP_ACK_IER);
1432	ast_udc_write(udc, UDC_IRQ_EP_ACK_ALL, AST_UDC_EP_ACK_ISR);
1433
1434	ast_udc_write(udc, 0, AST_UDC_EP0_CTRL);
1435	}
1436
1437	static void ast_udc_remove(struct platform_device *pdev)
1438	{
1439	struct ast_udc_dev *udc = platform_get_drvdata(pdev);
1440	unsigned long flags;
1441	u32 ctrl;
1442
1443	usb_del_gadget_udc(gadget: &udc->gadget);
1444	if (udc->driver) {
1445	/*
1446	* This is broken as only some cleanup is skipped, *udev is
1447	* freed and the register mapping goes away. Any further usage
1448	* probably crashes. Also the device is unbound, so the skipped
1449	* cleanup is never catched up later.
1450	*/
1451	dev_alert(&pdev->dev,
1452	"Driver is busy and still going away. Fasten your seat belts!\n");
1453	return;
1454	}
1455
1456	spin_lock_irqsave(&udc->lock, flags);
1457
1458	/* Disable upstream port connection */
1459	ctrl = ast_udc_read(udc, AST_UDC_FUNC_CTRL) & ~USB_UPSTREAM_EN;
1460	ast_udc_write(udc, ctrl, AST_UDC_FUNC_CTRL);
1461
1462	clk_disable_unprepare(clk: udc->clk);
1463
1464	spin_unlock_irqrestore(lock: &udc->lock, flags);
1465
1466	if (udc->ep0_buf)
1467	dma_free_coherent(dev: &pdev->dev,
1468	AST_UDC_EP_DMA_SIZE * AST_UDC_NUM_ENDPOINTS,
1469	cpu_addr: udc->ep0_buf,
1470	dma_handle: udc->ep0_buf_dma);
1471
1472	udc->ep0_buf = NULL;
1473	}
1474
1475	static int ast_udc_probe(struct platform_device *pdev)
1476	{
1477	enum usb_device_speed max_speed;
1478	struct device *dev = &pdev->dev;
1479	struct ast_udc_dev *udc;
1480	int rc;
1481
1482	udc = devm_kzalloc(dev: &pdev->dev, size: sizeof(struct ast_udc_dev), GFP_KERNEL);
1483	if (!udc)
1484	return -ENOMEM;
1485
1486	udc->gadget.dev.parent = dev;
1487	udc->pdev = pdev;
1488	spin_lock_init(&udc->lock);
1489
1490	udc->gadget.ops = &ast_udc_ops;
1491	udc->gadget.ep0 = &udc->ep[0].ep;
1492	udc->gadget.name = "aspeed-udc";
1493	udc->gadget.dev.init_name = "gadget";
1494
1495	udc->reg = devm_platform_ioremap_resource(pdev, index: 0);
1496	if (IS_ERR(ptr: udc->reg)) {
1497	dev_err(&pdev->dev, "Failed to map resources\n");
1498	return PTR_ERR(ptr: udc->reg);
1499	}
1500
1501	platform_set_drvdata(pdev, data: udc);
1502
1503	udc->clk = devm_clk_get(dev: &pdev->dev, NULL);
1504	if (IS_ERR(ptr: udc->clk)) {
1505	rc = PTR_ERR(ptr: udc->clk);
1506	goto err;
1507	}
1508	rc = clk_prepare_enable(clk: udc->clk);
1509	if (rc) {
1510	dev_err(&pdev->dev, "Failed to enable clock (0x%x)\n", rc);
1511	goto err;
1512	}
1513
1514	/* Check if we need to limit the HW to USB1 */
1515	max_speed = usb_get_maximum_speed(dev: &pdev->dev);
1516	if (max_speed != USB_SPEED_UNKNOWN && max_speed < USB_SPEED_HIGH)
1517	udc->force_usb1 = true;
1518
1519	/*
1520	* Allocate DMA buffers for all EPs in one chunk
1521	*/
1522	udc->ep0_buf = dma_alloc_coherent(dev: &pdev->dev,
1523	AST_UDC_EP_DMA_SIZE *
1524	AST_UDC_NUM_ENDPOINTS,
1525	dma_handle: &udc->ep0_buf_dma, GFP_KERNEL);
1526
1527	udc->gadget.speed = USB_SPEED_UNKNOWN;
1528	udc->gadget.max_speed = USB_SPEED_HIGH;
1529	udc->creq = udc->reg + AST_UDC_SETUP0;
1530
1531	/*
1532	* Support single stage mode or 32/256 stages descriptor mode.
1533	* Set default as Descriptor Mode.
1534	*/
1535	udc->desc_mode = AST_UDC_DESC_MODE;
1536
1537	dev_info(&pdev->dev, "DMA %s\n", udc->desc_mode ?
1538	"descriptor mode" : "single mode");
1539
1540	INIT_LIST_HEAD(list: &udc->gadget.ep_list);
1541	INIT_LIST_HEAD(list: &udc->gadget.ep0->ep_list);
1542
1543	/* Initialized udc ep */
1544	ast_udc_init_ep(udc);
1545
1546	/* Initialized udc device */
1547	ast_udc_init_dev(udc);
1548
1549	/* Initialized udc hardware */
1550	ast_udc_init_hw(udc);
1551
1552	/* Find interrupt and install handler */
1553	udc->irq = platform_get_irq(pdev, 0);
1554	if (udc->irq < 0) {
1555	rc = udc->irq;
1556	goto err;
1557	}
1558
1559	rc = devm_request_irq(dev: &pdev->dev, irq: udc->irq, handler: ast_udc_isr, irqflags: 0,
1560	KBUILD_MODNAME, dev_id: udc);
1561	if (rc) {
1562	dev_err(&pdev->dev, "Failed to request interrupt\n");
1563	goto err;
1564	}
1565
1566	rc = usb_add_gadget_udc(parent: &pdev->dev, gadget: &udc->gadget);
1567	if (rc) {
1568	dev_err(&pdev->dev, "Failed to add gadget udc\n");
1569	goto err;
1570	}
1571
1572	dev_info(&pdev->dev, "Initialized udc in USB%s mode\n",
1573	udc->force_usb1 ? "1" : "2");
1574
1575	return 0;
1576
1577	err:
1578	dev_err(&pdev->dev, "Failed to udc probe, rc:0x%x\n", rc);
1579	ast_udc_remove(pdev);
1580
1581	return rc;
1582	}
1583
1584	static const struct of_device_id ast_udc_of_dt_ids[] = {
1585	{ .compatible = "aspeed,ast2600-udc", },
1586	{}
1587	};
1588
1589	MODULE_DEVICE_TABLE(of, ast_udc_of_dt_ids);
1590
1591	static struct platform_driver ast_udc_driver = {
1592	.probe = ast_udc_probe,
1593	.remove = ast_udc_remove,
1594	.driver = {
1595	.name = KBUILD_MODNAME,
1596	.of_match_table = ast_udc_of_dt_ids,
1597	},
1598	};
1599
1600	module_platform_driver(ast_udc_driver);
1601
1602	MODULE_DESCRIPTION("ASPEED UDC driver");
1603	MODULE_AUTHOR("Neal Liu <neal_liu@aspeedtech.com>");
1604	MODULE_LICENSE("GPL");
1605