1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
4	*
5	* Copyright (C) 2010-2011 Texas Instruments Incorporated - https://www.ti.com
6	*
7	* Authors: Felipe Balbi <balbi@ti.com>,
8	* Sebastian Andrzej Siewior <bigeasy@linutronix.de>
9	*/
10
11	#include <linux/kernel.h>
12	#include <linux/delay.h>
13	#include <linux/slab.h>
14	#include <linux/spinlock.h>
15	#include <linux/platform_device.h>
16	#include <linux/pm_runtime.h>
17	#include <linux/interrupt.h>
18	#include <linux/io.h>
19	#include <linux/list.h>
20	#include <linux/dma-mapping.h>
21
22	#include <linux/usb/ch9.h>
23	#include <linux/usb/gadget.h>
24
25	#include "debug.h"
26	#include "core.h"
27	#include "gadget.h"
28	#include "io.h"
29
30	#define DWC3_ALIGN_FRAME(d, n) (((d)->frame_number + ((d)->interval * (n))) \
31	& ~((d)->interval - 1))
32
33	/**
34	* dwc3_gadget_set_test_mode - enables usb2 test modes
35	* @dwc: pointer to our context structure
36	* @mode: the mode to set (J, K SE0 NAK, Force Enable)
37	*
38	* Caller should take care of locking. This function will return 0 on
39	* success or -EINVAL if wrong Test Selector is passed.
40	*/
41	int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode)
42	{
43	u32 reg;
44
45	reg = dwc3_readl(base: dwc->regs, DWC3_DCTL);
46	reg &= ~DWC3_DCTL_TSTCTRL_MASK;
47
48	switch (mode) {
49	case USB_TEST_J:
50	case USB_TEST_K:
51	case USB_TEST_SE0_NAK:
52	case USB_TEST_PACKET:
53	case USB_TEST_FORCE_ENABLE:
54	reg |= mode << 1;
55	break;
56	default:
57	return -EINVAL;
58	}
59
60	dwc3_gadget_dctl_write_safe(dwc, value: reg);
61
62	return 0;
63	}
64
65	/**
66	* dwc3_gadget_get_link_state - gets current state of usb link
67	* @dwc: pointer to our context structure
68	*
69	* Caller should take care of locking. This function will
70	* return the link state on success (>= 0) or -ETIMEDOUT.
71	*/
72	int dwc3_gadget_get_link_state(struct dwc3 *dwc)
73	{
74	u32 reg;
75
76	reg = dwc3_readl(base: dwc->regs, DWC3_DSTS);
77
78	return DWC3_DSTS_USBLNKST(reg);
79	}
80
81	/**
82	* dwc3_gadget_set_link_state - sets usb link to a particular state
83	* @dwc: pointer to our context structure
84	* @state: the state to put link into
85	*
86	* Caller should take care of locking. This function will
87	* return 0 on success or -ETIMEDOUT.
88	*/
89	int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
90	{
91	int retries = 10000;
92	u32 reg;
93
94	/*
95	* Wait until device controller is ready. Only applies to 1.94a and
96	* later RTL.
97	*/
98	if (!DWC3_VER_IS_PRIOR(DWC3, 194A)) {
99	while (--retries) {
100	reg = dwc3_readl(base: dwc->regs, DWC3_DSTS);
101	if (reg & DWC3_DSTS_DCNRD)
102	udelay(5);
103	else
104	break;
105	}
106
107	if (retries <= 0)
108	return -ETIMEDOUT;
109	}
110
111	reg = dwc3_readl(base: dwc->regs, DWC3_DCTL);
112	reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
113
114	/* set no action before sending new link state change */
115	dwc3_writel(base: dwc->regs, DWC3_DCTL, value: reg);
116
117	/* set requested state */
118	reg |= DWC3_DCTL_ULSTCHNGREQ(state);
119	dwc3_writel(base: dwc->regs, DWC3_DCTL, value: reg);
120
121	/*
122	* The following code is racy when called from dwc3_gadget_wakeup,
123	* and is not needed, at least on newer versions
124	*/
125	if (!DWC3_VER_IS_PRIOR(DWC3, 194A))
126	return 0;
127
128	/* wait for a change in DSTS */
129	retries = 10000;
130	while (--retries) {
131	reg = dwc3_readl(base: dwc->regs, DWC3_DSTS);
132
133	if (DWC3_DSTS_USBLNKST(reg) == state)
134	return 0;
135
136	udelay(5);
137	}
138
139	return -ETIMEDOUT;
140	}
141
142	static void dwc3_ep0_reset_state(struct dwc3 *dwc)
143	{
144	unsigned int dir;
145
146	if (dwc->ep0state != EP0_SETUP_PHASE) {
147	dir = !!dwc->ep0_expect_in;
148	if (dwc->ep0state == EP0_DATA_PHASE)
149	dwc3_ep0_end_control_data(dwc, dep: dwc->eps[dir]);
150	else
151	dwc3_ep0_end_control_data(dwc, dep: dwc->eps[!dir]);
152
153	dwc->eps[0]->trb_enqueue = 0;
154	dwc->eps[1]->trb_enqueue = 0;
155
156	dwc3_ep0_stall_and_restart(dwc);
157	}
158	}
159
160	/**
161	* dwc3_ep_inc_trb - increment a trb index.
162	* @index: Pointer to the TRB index to increment.
163	*
164	* The index should never point to the link TRB. After incrementing,
165	* if it is point to the link TRB, wrap around to the beginning. The
166	* link TRB is always at the last TRB entry.
167	*/
168	static void dwc3_ep_inc_trb(u8 *index)
169	{
170	(*index)++;
171	if (*index == (DWC3_TRB_NUM - 1))
172	*index = 0;
173	}
174
175	/**
176	* dwc3_ep_inc_enq - increment endpoint's enqueue pointer
177	* @dep: The endpoint whose enqueue pointer we're incrementing
178	*/
179	static void dwc3_ep_inc_enq(struct dwc3_ep *dep)
180	{
181	dwc3_ep_inc_trb(index: &dep->trb_enqueue);
182	}
183
184	/**
185	* dwc3_ep_inc_deq - increment endpoint's dequeue pointer
186	* @dep: The endpoint whose enqueue pointer we're incrementing
187	*/
188	static void dwc3_ep_inc_deq(struct dwc3_ep *dep)
189	{
190	dwc3_ep_inc_trb(index: &dep->trb_dequeue);
191	}
192
193	static void dwc3_gadget_del_and_unmap_request(struct dwc3_ep *dep,
194	struct dwc3_request *req, int status)
195	{
196	struct dwc3 *dwc = dep->dwc;
197
198	list_del(entry: &req->list);
199	req->remaining = 0;
200	req->needs_extra_trb = false;
201	req->num_trbs = 0;
202
203	if (req->request.status == -EINPROGRESS)
204	req->request.status = status;
205
206	if (req->trb)
207	usb_gadget_unmap_request_by_dev(dev: dwc->sysdev,
208	req: &req->request, is_in: req->direction);
209
210	req->trb = NULL;
211	trace_dwc3_gadget_giveback(req);
212
213	if (dep->number > 1)
214	pm_runtime_put(dev: dwc->dev);
215	}
216
217	/**
218	* dwc3_gadget_giveback - call struct usb_request's ->complete callback
219	* @dep: The endpoint to whom the request belongs to
220	* @req: The request we're giving back
221	* @status: completion code for the request
222	*
223	* Must be called with controller's lock held and interrupts disabled. This
224	* function will unmap @req and call its ->complete() callback to notify upper
225	* layers that it has completed.
226	*/
227	void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
228	int status)
229	{
230	struct dwc3 *dwc = dep->dwc;
231
232	dwc3_gadget_del_and_unmap_request(dep, req, status);
233	req->status = DWC3_REQUEST_STATUS_COMPLETED;
234
235	spin_unlock(lock: &dwc->lock);
236	usb_gadget_giveback_request(ep: &dep->endpoint, req: &req->request);
237	spin_lock(lock: &dwc->lock);
238	}
239
240	/**
241	* dwc3_send_gadget_generic_command - issue a generic command for the controller
242	* @dwc: pointer to the controller context
243	* @cmd: the command to be issued
244	* @param: command parameter
245	*
246	* Caller should take care of locking. Issue @cmd with a given @param to @dwc
247	* and wait for its completion.
248	*/
249	int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned int cmd,
250	u32 param)
251	{
252	u32 timeout = 500;
253	int status = 0;
254	int ret = 0;
255	u32 reg;
256
257	dwc3_writel(base: dwc->regs, DWC3_DGCMDPAR, value: param);
258	dwc3_writel(base: dwc->regs, DWC3_DGCMD, value: cmd | DWC3_DGCMD_CMDACT);
259
260	do {
261	reg = dwc3_readl(base: dwc->regs, DWC3_DGCMD);
262	if (!(reg & DWC3_DGCMD_CMDACT)) {
263	status = DWC3_DGCMD_STATUS(reg);
264	if (status)
265	ret = -EINVAL;
266	break;
267	}
268	} while (--timeout);
269
270	if (!timeout) {
271	ret = -ETIMEDOUT;
272	status = -ETIMEDOUT;
273	}
274
275	trace_dwc3_gadget_generic_cmd(cmd, param, status);
276
277	return ret;
278	}
279
280	static int __dwc3_gadget_wakeup(struct dwc3 *dwc, bool async);
281
282	/**
283	* dwc3_send_gadget_ep_cmd - issue an endpoint command
284	* @dep: the endpoint to which the command is going to be issued
285	* @cmd: the command to be issued
286	* @params: parameters to the command
287	*
288	* Caller should handle locking. This function will issue @cmd with given
289	* @params to @dep and wait for its completion.
290	*/
291	int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned int cmd,
292	struct dwc3_gadget_ep_cmd_params *params)
293	{
294	const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
295	struct dwc3 *dwc = dep->dwc;
296	u32 timeout = 5000;
297	u32 saved_config = 0;
298	u32 reg;
299
300	int cmd_status = 0;
301	int ret = -EINVAL;
302
303	/*
304	* When operating in USB 2.0 speeds (HS/FS), if GUSB2PHYCFG.ENBLSLPM or
305	* GUSB2PHYCFG.SUSPHY is set, it must be cleared before issuing an
306	* endpoint command.
307	*
308	* Save and clear both GUSB2PHYCFG.ENBLSLPM and GUSB2PHYCFG.SUSPHY
309	* settings. Restore them after the command is completed.
310	*
311	* DWC_usb3 3.30a and DWC_usb31 1.90a programming guide section 3.2.2
312	*/
313	if (dwc->gadget->speed <= USB_SPEED_HIGH ||
314	DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_ENDTRANSFER) {
315	reg = dwc3_readl(base: dwc->regs, DWC3_GUSB2PHYCFG(0));
316	if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) {
317	saved_config |= DWC3_GUSB2PHYCFG_SUSPHY;
318	reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
319	}
320
321	if (reg & DWC3_GUSB2PHYCFG_ENBLSLPM) {
322	saved_config |= DWC3_GUSB2PHYCFG_ENBLSLPM;
323	reg &= ~DWC3_GUSB2PHYCFG_ENBLSLPM;
324	}
325
326	if (saved_config)
327	dwc3_writel(base: dwc->regs, DWC3_GUSB2PHYCFG(0), value: reg);
328	}
329
330	if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
331	int link_state;
332
333	/*
334	* Initiate remote wakeup if the link state is in U3 when
335	* operating in SS/SSP or L1/L2 when operating in HS/FS. If the
336	* link state is in U1/U2, no remote wakeup is needed. The Start
337	* Transfer command will initiate the link recovery.
338	*/
339	link_state = dwc3_gadget_get_link_state(dwc);
340	switch (link_state) {
341	case DWC3_LINK_STATE_U2:
342	if (dwc->gadget->speed >= USB_SPEED_SUPER)
343	break;
344
345	fallthrough;
346	case DWC3_LINK_STATE_U3:
347	ret = __dwc3_gadget_wakeup(dwc, async: false);
348	dev_WARN_ONCE(dwc->dev, ret, "wakeup failed --> %d\n",
349	ret);
350	break;
351	}
352	}
353
354	/*
355	* For some commands such as Update Transfer command, DEPCMDPARn
356	* registers are reserved. Since the driver often sends Update Transfer
357	* command, don't write to DEPCMDPARn to avoid register write delays and
358	* improve performance.
359	*/
360	if (DWC3_DEPCMD_CMD(cmd) != DWC3_DEPCMD_UPDATETRANSFER) {
361	dwc3_writel(base: dep->regs, DWC3_DEPCMDPAR0, value: params->param0);
362	dwc3_writel(base: dep->regs, DWC3_DEPCMDPAR1, value: params->param1);
363	dwc3_writel(base: dep->regs, DWC3_DEPCMDPAR2, value: params->param2);
364	}
365
366	/*
367	* Synopsys Databook 2.60a states in section 6.3.2.5.6 of that if we're
368	* not relying on XferNotReady, we can make use of a special "No
369	* Response Update Transfer" command where we should clear both CmdAct
370	* and CmdIOC bits.
371	*
372	* With this, we don't need to wait for command completion and can
373	* straight away issue further commands to the endpoint.
374	*
375	* NOTICE: We're making an assumption that control endpoints will never
376	* make use of Update Transfer command. This is a safe assumption
377	* because we can never have more than one request at a time with
378	* Control Endpoints. If anybody changes that assumption, this chunk
379	* needs to be updated accordingly.
380	*/
381	if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_UPDATETRANSFER &&
382	!usb_endpoint_xfer_isoc(epd: desc))
383	cmd &= ~(DWC3_DEPCMD_CMDIOC | DWC3_DEPCMD_CMDACT);
384	else
385	cmd |= DWC3_DEPCMD_CMDACT;
386
387	dwc3_writel(base: dep->regs, DWC3_DEPCMD, value: cmd);
388
389	if (!(cmd & DWC3_DEPCMD_CMDACT) ||
390	(DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_ENDTRANSFER &&
391	!(cmd & DWC3_DEPCMD_CMDIOC))) {
392	ret = 0;
393	goto skip_status;
394	}
395
396	do {
397	reg = dwc3_readl(base: dep->regs, DWC3_DEPCMD);
398	if (!(reg & DWC3_DEPCMD_CMDACT)) {
399	cmd_status = DWC3_DEPCMD_STATUS(reg);
400
401	switch (cmd_status) {
402	case 0:
403	ret = 0;
404	break;
405	case DEPEVT_TRANSFER_NO_RESOURCE:
406	dev_WARN(dwc->dev, "No resource for %s\n",
407	dep->name);
408	ret = -EINVAL;
409	break;
410	case DEPEVT_TRANSFER_BUS_EXPIRY:
411	/*
412	* SW issues START TRANSFER command to
413	* isochronous ep with future frame interval. If
414	* future interval time has already passed when
415	* core receives the command, it will respond
416	* with an error status of 'Bus Expiry'.
417	*
418	* Instead of always returning -EINVAL, let's
419	* give a hint to the gadget driver that this is
420	* the case by returning -EAGAIN.
421	*/
422	ret = -EAGAIN;
423	break;
424	default:
425	dev_WARN(dwc->dev, "UNKNOWN cmd status\n");
426	}
427
428	break;
429	}
430	} while (--timeout);
431
432	if (timeout == 0) {
433	ret = -ETIMEDOUT;
434	cmd_status = -ETIMEDOUT;
435	}
436
437	skip_status:
438	trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status);
439
440	if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
441	if (ret == 0)
442	dep->flags |= DWC3_EP_TRANSFER_STARTED;
443
444	if (ret != -ETIMEDOUT)
445	dwc3_gadget_ep_get_transfer_index(dep);
446	}
447
448	if (saved_config) {
449	reg = dwc3_readl(base: dwc->regs, DWC3_GUSB2PHYCFG(0));
450	reg |= saved_config;
451	dwc3_writel(base: dwc->regs, DWC3_GUSB2PHYCFG(0), value: reg);
452	}
453
454	return ret;
455	}
456
457	static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep)
458	{
459	struct dwc3 *dwc = dep->dwc;
460	struct dwc3_gadget_ep_cmd_params params;
461	u32 cmd = DWC3_DEPCMD_CLEARSTALL;
462
463	/*
464	* As of core revision 2.60a the recommended programming model
465	* is to set the ClearPendIN bit when issuing a Clear Stall EP
466	* command for IN endpoints. This is to prevent an issue where
467	* some (non-compliant) hosts may not send ACK TPs for pending
468	* IN transfers due to a mishandled error condition. Synopsys
469	* STAR 9000614252.
470	*/
471	if (dep->direction &&
472	!DWC3_VER_IS_PRIOR(DWC3, 260A) &&
473	(dwc->gadget->speed >= USB_SPEED_SUPER))
474	cmd |= DWC3_DEPCMD_CLEARPENDIN;
475
476	memset(&params, 0, sizeof(params));
477
478	return dwc3_send_gadget_ep_cmd(dep, cmd, params: &params);
479	}
480
481	static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
482	struct dwc3_trb *trb)
483	{
484	u32 offset = (char *) trb - (char *) dep->trb_pool;
485
486	return dep->trb_pool_dma + offset;
487	}
488
489	static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
490	{
491	struct dwc3 *dwc = dep->dwc;
492
493	if (dep->trb_pool)
494	return 0;
495
496	dep->trb_pool = dma_alloc_coherent(dev: dwc->sysdev,
497	size: sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
498	dma_handle: &dep->trb_pool_dma, GFP_KERNEL);
499	if (!dep->trb_pool) {
500	dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
501	dep->name);
502	return -ENOMEM;
503	}
504
505	return 0;
506	}
507
508	static void dwc3_free_trb_pool(struct dwc3_ep *dep)
509	{
510	struct dwc3 *dwc = dep->dwc;
511
512	dma_free_coherent(dev: dwc->sysdev, size: sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
513	cpu_addr: dep->trb_pool, dma_handle: dep->trb_pool_dma);
514
515	dep->trb_pool = NULL;
516	dep->trb_pool_dma = 0;
517	}
518
519	static int dwc3_gadget_set_xfer_resource(struct dwc3_ep *dep)
520	{
521	struct dwc3_gadget_ep_cmd_params params;
522	int ret;
523
524	if (dep->flags & DWC3_EP_RESOURCE_ALLOCATED)
525	return 0;
526
527	memset(&params, 0x00, sizeof(params));
528
529	params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
530
531	ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE,
532	params: &params);
533	if (ret)
534	return ret;
535
536	dep->flags |= DWC3_EP_RESOURCE_ALLOCATED;
537	return 0;
538	}
539
540	/**
541	* dwc3_gadget_start_config - reset endpoint resources
542	* @dwc: pointer to the DWC3 context
543	* @resource_index: DEPSTARTCFG.XferRscIdx value (must be 0 or 2)
544	*
545	* Set resource_index=0 to reset all endpoints' resources allocation. Do this as
546	* part of the power-on/soft-reset initialization.
547	*
548	* Set resource_index=2 to reset only non-control endpoints' resources. Do this
549	* on receiving the SET_CONFIGURATION request or hibernation resume.
550	*/
551	int dwc3_gadget_start_config(struct dwc3 *dwc, unsigned int resource_index)
552	{
553	struct dwc3_gadget_ep_cmd_params params;
554	u32 cmd;
555	int i;
556	int ret;
557
558	if (resource_index != 0 && resource_index != 2)
559	return -EINVAL;
560
561	memset(&params, 0x00, sizeof(params));
562	cmd = DWC3_DEPCMD_DEPSTARTCFG;
563	cmd |= DWC3_DEPCMD_PARAM(resource_index);
564
565	ret = dwc3_send_gadget_ep_cmd(dep: dwc->eps[0], cmd, params: &params);
566	if (ret)
567	return ret;
568
569	/* Reset resource allocation flags */
570	for (i = resource_index; i < dwc->num_eps && dwc->eps[i]; i++)
571	dwc->eps[i]->flags &= ~DWC3_EP_RESOURCE_ALLOCATED;
572
573	return 0;
574	}
575
576	static int dwc3_gadget_set_ep_config(struct dwc3_ep *dep, unsigned int action)
577	{
578	const struct usb_ss_ep_comp_descriptor *comp_desc;
579	const struct usb_endpoint_descriptor *desc;
580	struct dwc3_gadget_ep_cmd_params params;
581	struct dwc3 *dwc = dep->dwc;
582
583	comp_desc = dep->endpoint.comp_desc;
584	desc = dep->endpoint.desc;
585
586	memset(&params, 0x00, sizeof(params));
587
588	params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
589	| DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc));
590
591	/* Burst size is only needed in SuperSpeed mode */
592	if (dwc->gadget->speed >= USB_SPEED_SUPER) {
593	u32 burst = dep->endpoint.maxburst;
594
595	params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1);
596	}
597
598	params.param0 |= action;
599	if (action == DWC3_DEPCFG_ACTION_RESTORE)
600	params.param2 |= dep->saved_state;
601
602	if (usb_endpoint_xfer_control(epd: desc))
603	params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN;
604
605	if (dep->number <= 1 || usb_endpoint_xfer_isoc(epd: desc))
606	params.param1 |= DWC3_DEPCFG_XFER_NOT_READY_EN;
607
608	if (usb_ss_max_streams(comp: comp_desc) && usb_endpoint_xfer_bulk(epd: desc)) {
609	params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
610	| DWC3_DEPCFG_XFER_COMPLETE_EN
611	| DWC3_DEPCFG_STREAM_EVENT_EN;
612	dep->stream_capable = true;
613	}
614
615	if (!usb_endpoint_xfer_control(epd: desc))
616	params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
617
618	/*
619	* We are doing 1:1 mapping for endpoints, meaning
620	* Physical Endpoints 2 maps to Logical Endpoint 2 and
621	* so on. We consider the direction bit as part of the physical
622	* endpoint number. So USB endpoint 0x81 is 0x03.
623	*/
624	params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
625
626	/*
627	* We must use the lower 16 TX FIFOs even though
628	* HW might have more
629	*/
630	if (dep->direction)
631	params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
632
633	if (desc->bInterval) {
634	u8 bInterval_m1;
635
636	/*
637	* Valid range for DEPCFG.bInterval_m1 is from 0 to 13.
638	*
639	* NOTE: The programming guide incorrectly stated bInterval_m1
640	* must be set to 0 when operating in fullspeed. Internally the
641	* controller does not have this limitation. See DWC_usb3x
642	* programming guide section 3.2.2.1.
643	*/
644	bInterval_m1 = min_t(u8, desc->bInterval - 1, 13);
645
646	if (usb_endpoint_type(epd: desc) == USB_ENDPOINT_XFER_INT &&
647	dwc->gadget->speed == USB_SPEED_FULL)
648	dep->interval = desc->bInterval;
649	else
650	dep->interval = 1 << (desc->bInterval - 1);
651
652	params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(bInterval_m1);
653	}
654
655	return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, params: &params);
656	}
657
658	/**
659	* dwc3_gadget_calc_tx_fifo_size - calculates the txfifo size value
660	* @dwc: pointer to the DWC3 context
661	* @mult: multiplier to be used when calculating the fifo_size
662	*
663	* Calculates the size value based on the equation below:
664	*
665	* DWC3 revision 280A and prior:
666	* fifo_size = mult * (max_packet / mdwidth) + 1;
667	*
668	* DWC3 revision 290A and onwards:
669	* fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1
670	*
671	* The max packet size is set to 1024, as the txfifo requirements mainly apply
672	* to super speed USB use cases. However, it is safe to overestimate the fifo
673	* allocations for other scenarios, i.e. high speed USB.
674	*/
675	static int dwc3_gadget_calc_tx_fifo_size(struct dwc3 *dwc, int mult)
676	{
677	int max_packet = 1024;
678	int fifo_size;
679	int mdwidth;
680
681	mdwidth = dwc3_mdwidth(dwc);
682
683	/* MDWIDTH is represented in bits, we need it in bytes */
684	mdwidth >>= 3;
685
686	if (DWC3_VER_IS_PRIOR(DWC3, 290A))
687	fifo_size = mult * (max_packet / mdwidth) + 1;
688	else
689	fifo_size = mult * ((max_packet + mdwidth) / mdwidth) + 1;
690	return fifo_size;
691	}
692
693	/**
694	* dwc3_gadget_clear_tx_fifos - Clears txfifo allocation
695	* @dwc: pointer to the DWC3 context
696	*
697	* Iterates through all the endpoint registers and clears the previous txfifo
698	* allocations.
699	*/
700	void dwc3_gadget_clear_tx_fifos(struct dwc3 *dwc)
701	{
702	struct dwc3_ep *dep;
703	int fifo_depth;
704	int size;
705	int num;
706
707	if (!dwc->do_fifo_resize)
708	return;
709
710	/* Read ep0IN related TXFIFO size */
711	dep = dwc->eps[1];
712	size = dwc3_readl(base: dwc->regs, DWC3_GTXFIFOSIZ(0));
713	if (DWC3_IP_IS(DWC3))
714	fifo_depth = DWC3_GTXFIFOSIZ_TXFDEP(size);
715	else
716	fifo_depth = DWC31_GTXFIFOSIZ_TXFDEP(size);
717
718	dwc->last_fifo_depth = fifo_depth;
719	/* Clear existing TXFIFO for all IN eps except ep0 */
720	for (num = 3; num < min_t(int, dwc->num_eps, DWC3_ENDPOINTS_NUM);
721	num += 2) {
722	dep = dwc->eps[num];
723	/* Don't change TXFRAMNUM on usb31 version */
724	size = DWC3_IP_IS(DWC3) ? 0 :
725	dwc3_readl(base: dwc->regs, DWC3_GTXFIFOSIZ(num >> 1)) &
726	DWC31_GTXFIFOSIZ_TXFRAMNUM;
727
728	dwc3_writel(base: dwc->regs, DWC3_GTXFIFOSIZ(num >> 1), value: size);
729	dep->flags &= ~DWC3_EP_TXFIFO_RESIZED;
730	}
731	dwc->num_ep_resized = 0;
732	}
733
734	/*
735	* dwc3_gadget_resize_tx_fifos - reallocate fifo spaces for current use-case
736	* @dwc: pointer to our context structure
737	*
738	* This function will a best effort FIFO allocation in order
739	* to improve FIFO usage and throughput, while still allowing
740	* us to enable as many endpoints as possible.
741	*
742	* Keep in mind that this operation will be highly dependent
743	* on the configured size for RAM1 - which contains TxFifo -,
744	* the amount of endpoints enabled on coreConsultant tool, and
745	* the width of the Master Bus.
746	*
747	* In general, FIFO depths are represented with the following equation:
748	*
749	* fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1
750	*
751	* In conjunction with dwc3_gadget_check_config(), this resizing logic will
752	* ensure that all endpoints will have enough internal memory for one max
753	* packet per endpoint.
754	*/
755	static int dwc3_gadget_resize_tx_fifos(struct dwc3_ep *dep)
756	{
757	struct dwc3 *dwc = dep->dwc;
758	int fifo_0_start;
759	int ram1_depth;
760	int fifo_size;
761	int min_depth;
762	int num_in_ep;
763	int remaining;
764	int num_fifos = 1;
765	int fifo;
766	int tmp;
767
768	if (!dwc->do_fifo_resize)
769	return 0;
770
771	/* resize IN endpoints except ep0 */
772	if (!usb_endpoint_dir_in(epd: dep->endpoint.desc) || dep->number <= 1)
773	return 0;
774
775	/* bail if already resized */
776	if (dep->flags & DWC3_EP_TXFIFO_RESIZED)
777	return 0;
778
779	ram1_depth = DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7);
780
781	if ((dep->endpoint.maxburst > 1 &&
782	usb_endpoint_xfer_bulk(epd: dep->endpoint.desc)) ||
783	usb_endpoint_xfer_isoc(epd: dep->endpoint.desc))
784	num_fifos = 3;
785
786	if (dep->endpoint.maxburst > 6 &&
787	(usb_endpoint_xfer_bulk(epd: dep->endpoint.desc) ||
788	usb_endpoint_xfer_isoc(epd: dep->endpoint.desc)) && DWC3_IP_IS(DWC31))
789	num_fifos = dwc->tx_fifo_resize_max_num;
790
791	/* FIFO size for a single buffer */
792	fifo = dwc3_gadget_calc_tx_fifo_size(dwc, mult: 1);
793
794	/* Calculate the number of remaining EPs w/o any FIFO */
795	num_in_ep = dwc->max_cfg_eps;
796	num_in_ep -= dwc->num_ep_resized;
797
798	/* Reserve at least one FIFO for the number of IN EPs */
799	min_depth = num_in_ep * (fifo + 1);
800	remaining = ram1_depth - min_depth - dwc->last_fifo_depth;
801	remaining = max_t(int, 0, remaining);
802	/*
803	* We've already reserved 1 FIFO per EP, so check what we can fit in
804	* addition to it. If there is not enough remaining space, allocate
805	* all the remaining space to the EP.
806	*/
807	fifo_size = (num_fifos - 1) * fifo;
808	if (remaining < fifo_size)
809	fifo_size = remaining;
810
811	fifo_size += fifo;
812	/* Last increment according to the TX FIFO size equation */
813	fifo_size++;
814
815	/* Check if TXFIFOs start at non-zero addr */
816	tmp = dwc3_readl(base: dwc->regs, DWC3_GTXFIFOSIZ(0));
817	fifo_0_start = DWC3_GTXFIFOSIZ_TXFSTADDR(tmp);
818
819	fifo_size |= (fifo_0_start + (dwc->last_fifo_depth << 16));
820	if (DWC3_IP_IS(DWC3))
821	dwc->last_fifo_depth += DWC3_GTXFIFOSIZ_TXFDEP(fifo_size);
822	else
823	dwc->last_fifo_depth += DWC31_GTXFIFOSIZ_TXFDEP(fifo_size);
824
825	/* Check fifo size allocation doesn't exceed available RAM size. */
826	if (dwc->last_fifo_depth >= ram1_depth) {
827	dev_err(dwc->dev, "Fifosize(%d) > RAM size(%d) %s depth:%d\n",
828	dwc->last_fifo_depth, ram1_depth,
829	dep->endpoint.name, fifo_size);
830	if (DWC3_IP_IS(DWC3))
831	fifo_size = DWC3_GTXFIFOSIZ_TXFDEP(fifo_size);
832	else
833	fifo_size = DWC31_GTXFIFOSIZ_TXFDEP(fifo_size);
834
835	dwc->last_fifo_depth -= fifo_size;
836	return -ENOMEM;
837	}
838
839	dwc3_writel(base: dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1), value: fifo_size);
840	dep->flags |= DWC3_EP_TXFIFO_RESIZED;
841	dwc->num_ep_resized++;
842
843	return 0;
844	}
845
846	/**
847	* __dwc3_gadget_ep_enable - initializes a hw endpoint
848	* @dep: endpoint to be initialized
849	* @action: one of INIT, MODIFY or RESTORE
850	*
851	* Caller should take care of locking. Execute all necessary commands to
852	* initialize a HW endpoint so it can be used by a gadget driver.
853	*/
854	static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, unsigned int action)
855	{
856	const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
857	struct dwc3 *dwc = dep->dwc;
858
859	u32 reg;
860	int ret;
861
862	if (!(dep->flags & DWC3_EP_ENABLED)) {
863	ret = dwc3_gadget_resize_tx_fifos(dep);
864	if (ret)
865	return ret;
866	}
867
868	ret = dwc3_gadget_set_ep_config(dep, action);
869	if (ret)
870	return ret;
871
872	if (!(dep->flags & DWC3_EP_RESOURCE_ALLOCATED)) {
873	ret = dwc3_gadget_set_xfer_resource(dep);
874	if (ret)
875	return ret;
876	}
877
878	if (!(dep->flags & DWC3_EP_ENABLED)) {
879	struct dwc3_trb *trb_st_hw;
880	struct dwc3_trb *trb_link;
881
882	dep->type = usb_endpoint_type(epd: desc);
883	dep->flags |= DWC3_EP_ENABLED;
884
885	reg = dwc3_readl(base: dwc->regs, DWC3_DALEPENA);
886	reg |= DWC3_DALEPENA_EP(dep->number);
887	dwc3_writel(base: dwc->regs, DWC3_DALEPENA, value: reg);
888
889	dep->trb_dequeue = 0;
890	dep->trb_enqueue = 0;
891
892	if (usb_endpoint_xfer_control(epd: desc))
893	goto out;
894
895	/* Initialize the TRB ring */
896	memset(dep->trb_pool, 0,
897	sizeof(struct dwc3_trb) * DWC3_TRB_NUM);
898
899	/* Link TRB. The HWO bit is never reset */
900	trb_st_hw = &dep->trb_pool[0];
901
902	trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
903	trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
904	trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
905	trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;
906	trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
907	}
908
909	/*
910	* Issue StartTransfer here with no-op TRB so we can always rely on No
911	* Response Update Transfer command.
912	*/
913	if (usb_endpoint_xfer_bulk(epd: desc) ||
914	usb_endpoint_xfer_int(epd: desc)) {
915	struct dwc3_gadget_ep_cmd_params params;
916	struct dwc3_trb *trb;
917	dma_addr_t trb_dma;
918	u32 cmd;
919
920	memset(&params, 0, sizeof(params));
921	trb = &dep->trb_pool[0];
922	trb_dma = dwc3_trb_dma_offset(dep, trb);
923
924	params.param0 = upper_32_bits(trb_dma);
925	params.param1 = lower_32_bits(trb_dma);
926
927	cmd = DWC3_DEPCMD_STARTTRANSFER;
928
929	ret = dwc3_send_gadget_ep_cmd(dep, cmd, params: &params);
930	if (ret < 0)
931	return ret;
932
933	if (dep->stream_capable) {
934	/*
935	* For streams, at start, there maybe a race where the
936	* host primes the endpoint before the function driver
937	* queues a request to initiate a stream. In that case,
938	* the controller will not see the prime to generate the
939	* ERDY and start stream. To workaround this, issue a
940	* no-op TRB as normal, but end it immediately. As a
941	* result, when the function driver queues the request,
942	* the next START_TRANSFER command will cause the
943	* controller to generate an ERDY to initiate the
944	* stream.
945	*/
946	dwc3_stop_active_transfer(dep, force: true, interrupt: true);
947
948	/*
949	* All stream eps will reinitiate stream on NoStream
950	* rejection until we can determine that the host can
951	* prime after the first transfer.
952	*
953	* However, if the controller is capable of
954	* TXF_FLUSH_BYPASS, then IN direction endpoints will
955	* automatically restart the stream without the driver
956	* initiation.
957	*/
958	if (!dep->direction ||
959	!(dwc->hwparams.hwparams9 &
960	DWC3_GHWPARAMS9_DEV_TXF_FLUSH_BYPASS))
961	dep->flags |= DWC3_EP_FORCE_RESTART_STREAM;
962	}
963	}
964
965	out:
966	trace_dwc3_gadget_ep_enable(dep);
967
968	return 0;
969	}
970
971	void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep, int status)
972	{
973	struct dwc3_request *req;
974
975	dwc3_stop_active_transfer(dep, force: true, interrupt: false);
976
977	/* If endxfer is delayed, avoid unmapping requests */
978	if (dep->flags & DWC3_EP_DELAY_STOP)
979	return;
980
981	/* - giveback all requests to gadget driver */
982	while (!list_empty(head: &dep->started_list)) {
983	req = next_request(list: &dep->started_list);
984
985	dwc3_gadget_giveback(dep, req, status);
986	}
987
988	while (!list_empty(head: &dep->pending_list)) {
989	req = next_request(list: &dep->pending_list);
990
991	dwc3_gadget_giveback(dep, req, status);
992	}
993
994	while (!list_empty(head: &dep->cancelled_list)) {
995	req = next_request(list: &dep->cancelled_list);
996
997	dwc3_gadget_giveback(dep, req, status);
998	}
999	}
1000
1001	/**
1002	* __dwc3_gadget_ep_disable - disables a hw endpoint
1003	* @dep: the endpoint to disable
1004	*
1005	* This function undoes what __dwc3_gadget_ep_enable did and also removes
1006	* requests which are currently being processed by the hardware and those which
1007	* are not yet scheduled.
1008	*
1009	* Caller should take care of locking.
1010	*/
1011	static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
1012	{
1013	struct dwc3 *dwc = dep->dwc;
1014	u32 reg;
1015	u32 mask;
1016
1017	trace_dwc3_gadget_ep_disable(dep);
1018
1019	/* make sure HW endpoint isn't stalled */
1020	if (dep->flags & DWC3_EP_STALL)
1021	__dwc3_gadget_ep_set_halt(dep, value: 0, protocol: false);
1022
1023	reg = dwc3_readl(base: dwc->regs, DWC3_DALEPENA);
1024	reg &= ~DWC3_DALEPENA_EP(dep->number);
1025	dwc3_writel(base: dwc->regs, DWC3_DALEPENA, value: reg);
1026
1027	dwc3_remove_requests(dwc, dep, status: -ESHUTDOWN);
1028
1029	dep->stream_capable = false;
1030	dep->type = 0;
1031	mask = DWC3_EP_TXFIFO_RESIZED | DWC3_EP_RESOURCE_ALLOCATED;
1032	/*
1033	* dwc3_remove_requests() can exit early if DWC3 EP delayed stop is
1034	* set. Do not clear DEP flags, so that the end transfer command will
1035	* be reattempted during the next SETUP stage.
1036	*/
1037	if (dep->flags & DWC3_EP_DELAY_STOP)
1038	mask |= (DWC3_EP_DELAY_STOP | DWC3_EP_TRANSFER_STARTED);
1039	dep->flags &= mask;
1040
1041	/* Clear out the ep descriptors for non-ep0 */
1042	if (dep->number > 1) {
1043	dep->endpoint.comp_desc = NULL;
1044	dep->endpoint.desc = NULL;
1045	}
1046
1047	return 0;
1048	}
1049
1050	/* -------------------------------------------------------------------------- */
1051
1052	static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
1053	const struct usb_endpoint_descriptor *desc)
1054	{
1055	return -EINVAL;
1056	}
1057
1058	static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
1059	{
1060	return -EINVAL;
1061	}
1062
1063	/* -------------------------------------------------------------------------- */
1064
1065	static int dwc3_gadget_ep_enable(struct usb_ep *ep,
1066	const struct usb_endpoint_descriptor *desc)
1067	{
1068	struct dwc3_ep *dep;
1069	struct dwc3 *dwc;
1070	unsigned long flags;
1071	int ret;
1072
1073	if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
1074	pr_debug("dwc3: invalid parameters\n");
1075	return -EINVAL;
1076	}
1077
1078	if (!desc->wMaxPacketSize) {
1079	pr_debug("dwc3: missing wMaxPacketSize\n");
1080	return -EINVAL;
1081	}
1082
1083	dep = to_dwc3_ep(ep);
1084	dwc = dep->dwc;
1085
1086	if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED,
1087	"%s is already enabled\n",
1088	dep->name))
1089	return 0;
1090
1091	spin_lock_irqsave(&dwc->lock, flags);
1092	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
1093	spin_unlock_irqrestore(lock: &dwc->lock, flags);
1094
1095	return ret;
1096	}
1097
1098	static int dwc3_gadget_ep_disable(struct usb_ep *ep)
1099	{
1100	struct dwc3_ep *dep;
1101	struct dwc3 *dwc;
1102	unsigned long flags;
1103	int ret;
1104
1105	if (!ep) {
1106	pr_debug("dwc3: invalid parameters\n");
1107	return -EINVAL;
1108	}
1109
1110	dep = to_dwc3_ep(ep);
1111	dwc = dep->dwc;
1112
1113	if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED),
1114	"%s is already disabled\n",
1115	dep->name))
1116	return 0;
1117
1118	spin_lock_irqsave(&dwc->lock, flags);
1119	ret = __dwc3_gadget_ep_disable(dep);
1120	spin_unlock_irqrestore(lock: &dwc->lock, flags);
1121
1122	return ret;
1123	}
1124
1125	static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
1126	gfp_t gfp_flags)
1127	{
1128	struct dwc3_request *req;
1129	struct dwc3_ep *dep = to_dwc3_ep(ep);
1130
1131	req = kzalloc(size: sizeof(*req), flags: gfp_flags);
1132	if (!req)
1133	return NULL;
1134
1135	req->direction = dep->direction;
1136	req->epnum = dep->number;
1137	req->dep = dep;
1138	req->status = DWC3_REQUEST_STATUS_UNKNOWN;
1139
1140	trace_dwc3_alloc_request(req);
1141
1142	return &req->request;
1143	}
1144
1145	static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
1146	struct usb_request *request)
1147	{
1148	struct dwc3_request *req = to_dwc3_request(request);
1149
1150	trace_dwc3_free_request(req);
1151	kfree(objp: req);
1152	}
1153
1154	/**
1155	* dwc3_ep_prev_trb - returns the previous TRB in the ring
1156	* @dep: The endpoint with the TRB ring
1157	* @index: The index of the current TRB in the ring
1158	*
1159	* Returns the TRB prior to the one pointed to by the index. If the
1160	* index is 0, we will wrap backwards, skip the link TRB, and return
1161	* the one just before that.
1162	*/
1163	static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index)
1164	{
1165	u8 tmp = index;
1166
1167	if (!tmp)
1168	tmp = DWC3_TRB_NUM - 1;
1169
1170	return &dep->trb_pool[tmp - 1];
1171	}
1172
1173	static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep)
1174	{
1175	u8 trbs_left;
1176
1177	/*
1178	* If the enqueue & dequeue are equal then the TRB ring is either full
1179	* or empty. It's considered full when there are DWC3_TRB_NUM-1 of TRBs
1180	* pending to be processed by the driver.
1181	*/
1182	if (dep->trb_enqueue == dep->trb_dequeue) {
1183	/*
1184	* If there is any request remained in the started_list at
1185	* this point, that means there is no TRB available.
1186	*/
1187	if (!list_empty(head: &dep->started_list))
1188	return 0;
1189
1190	return DWC3_TRB_NUM - 1;
1191	}
1192
1193	trbs_left = dep->trb_dequeue - dep->trb_enqueue;
1194	trbs_left &= (DWC3_TRB_NUM - 1);
1195
1196	if (dep->trb_dequeue < dep->trb_enqueue)
1197	trbs_left--;
1198
1199	return trbs_left;
1200	}
1201
1202	/**
1203	* dwc3_prepare_one_trb - setup one TRB from one request
1204	* @dep: endpoint for which this request is prepared
1205	* @req: dwc3_request pointer
1206	* @trb_length: buffer size of the TRB
1207	* @chain: should this TRB be chained to the next?
1208	* @node: only for isochronous endpoints. First TRB needs different type.
1209	* @use_bounce_buffer: set to use bounce buffer
1210	* @must_interrupt: set to interrupt on TRB completion
1211	*/
1212	static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
1213	struct dwc3_request *req, unsigned int trb_length,
1214	unsigned int chain, unsigned int node, bool use_bounce_buffer,
1215	bool must_interrupt)
1216	{
1217	struct dwc3_trb *trb;
1218	dma_addr_t dma;
1219	unsigned int stream_id = req->request.stream_id;
1220	unsigned int short_not_ok = req->request.short_not_ok;
1221	unsigned int no_interrupt = req->request.no_interrupt;
1222	unsigned int is_last = req->request.is_last;
1223	struct dwc3 *dwc = dep->dwc;
1224	struct usb_gadget *gadget = dwc->gadget;
1225	enum usb_device_speed speed = gadget->speed;
1226
1227	if (use_bounce_buffer)
1228	dma = dep->dwc->bounce_addr;
1229	else if (req->request.num_sgs > 0)
1230	dma = sg_dma_address(req->start_sg);
1231	else
1232	dma = req->request.dma;
1233
1234	trb = &dep->trb_pool[dep->trb_enqueue];
1235
1236	if (!req->trb) {
1237	dwc3_gadget_move_started_request(req);
1238	req->trb = trb;
1239	req->trb_dma = dwc3_trb_dma_offset(dep, trb);
1240	}
1241
1242	req->num_trbs++;
1243
1244	trb->size = DWC3_TRB_SIZE_LENGTH(trb_length);
1245	trb->bpl = lower_32_bits(dma);
1246	trb->bph = upper_32_bits(dma);
1247
1248	switch (usb_endpoint_type(epd: dep->endpoint.desc)) {
1249	case USB_ENDPOINT_XFER_CONTROL:
1250	trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;
1251	break;
1252
1253	case USB_ENDPOINT_XFER_ISOC:
1254	if (!node) {
1255	trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
1256
1257	/*
1258	* USB Specification 2.0 Section 5.9.2 states that: "If
1259	* there is only a single transaction in the microframe,
1260	* only a DATA0 data packet PID is used. If there are
1261	* two transactions per microframe, DATA1 is used for
1262	* the first transaction data packet and DATA0 is used
1263	* for the second transaction data packet. If there are
1264	* three transactions per microframe, DATA2 is used for
1265	* the first transaction data packet, DATA1 is used for
1266	* the second, and DATA0 is used for the third."
1267	*
1268	* IOW, we should satisfy the following cases:
1269	*
1270	* 1) length <= maxpacket
1271	* - DATA0
1272	*
1273	* 2) maxpacket < length <= (2 * maxpacket)
1274	* - DATA1, DATA0
1275	*
1276	* 3) (2 * maxpacket) < length <= (3 * maxpacket)
1277	* - DATA2, DATA1, DATA0
1278	*/
1279	if (speed == USB_SPEED_HIGH) {
1280	struct usb_ep *ep = &dep->endpoint;
1281	unsigned int mult = 2;
1282	unsigned int maxp = usb_endpoint_maxp(epd: ep->desc);
1283
1284	if (req->request.length <= (2 * maxp))
1285	mult--;
1286
1287	if (req->request.length <= maxp)
1288	mult--;
1289
1290	trb->size |= DWC3_TRB_SIZE_PCM1(mult);
1291	}
1292	} else {
1293	trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
1294	}
1295
1296	if (!no_interrupt && !chain)
1297	trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
1298	break;
1299
1300	case USB_ENDPOINT_XFER_BULK:
1301	case USB_ENDPOINT_XFER_INT:
1302	trb->ctrl = DWC3_TRBCTL_NORMAL;
1303	break;
1304	default:
1305	/*
1306	* This is only possible with faulty memory because we
1307	* checked it already :)
1308	*/
1309	dev_WARN(dwc->dev, "Unknown endpoint type %d\n",
1310	usb_endpoint_type(dep->endpoint.desc));
1311	}
1312
1313	/*
1314	* Enable Continue on Short Packet
1315	* when endpoint is not a stream capable
1316	*/
1317	if (usb_endpoint_dir_out(epd: dep->endpoint.desc)) {
1318	if (!dep->stream_capable)
1319	trb->ctrl |= DWC3_TRB_CTRL_CSP;
1320
1321	if (short_not_ok)
1322	trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
1323	}
1324
1325	/* All TRBs setup for MST must set CSP=1 when LST=0 */
1326	if (dep->stream_capable && DWC3_MST_CAPABLE(&dwc->hwparams))
1327	trb->ctrl |= DWC3_TRB_CTRL_CSP;
1328
1329	if ((!no_interrupt && !chain) || must_interrupt)
1330	trb->ctrl |= DWC3_TRB_CTRL_IOC;
1331
1332	if (chain)
1333	trb->ctrl |= DWC3_TRB_CTRL_CHN;
1334	else if (dep->stream_capable && is_last &&
1335	!DWC3_MST_CAPABLE(&dwc->hwparams))
1336	trb->ctrl |= DWC3_TRB_CTRL_LST;
1337
1338	if (usb_endpoint_xfer_bulk(epd: dep->endpoint.desc) && dep->stream_capable)
1339	trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(stream_id);
1340
1341	/*
1342	* As per data book 4.2.3.2TRB Control Bit Rules section
1343	*
1344	* The controller autonomously checks the HWO field of a TRB to determine if the
1345	* entire TRB is valid. Therefore, software must ensure that the rest of the TRB
1346	* is valid before setting the HWO field to '1'. In most systems, this means that
1347	* software must update the fourth DWORD of a TRB last.
1348	*
1349	* However there is a possibility of CPU re-ordering here which can cause
1350	* controller to observe the HWO bit set prematurely.
1351	* Add a write memory barrier to prevent CPU re-ordering.
1352	*/
1353	wmb();
1354	trb->ctrl |= DWC3_TRB_CTRL_HWO;
1355
1356	dwc3_ep_inc_enq(dep);
1357
1358	trace_dwc3_prepare_trb(dep, trb);
1359	}
1360
1361	static bool dwc3_needs_extra_trb(struct dwc3_ep *dep, struct dwc3_request *req)
1362	{
1363	unsigned int maxp = usb_endpoint_maxp(epd: dep->endpoint.desc);
1364	unsigned int rem = req->request.length % maxp;
1365
1366	if ((req->request.length && req->request.zero && !rem &&
1367	!usb_endpoint_xfer_isoc(epd: dep->endpoint.desc)) ||
1368	(!req->direction && rem))
1369	return true;
1370
1371	return false;
1372	}
1373
1374	/**
1375	* dwc3_prepare_last_sg - prepare TRBs for the last SG entry
1376	* @dep: The endpoint that the request belongs to
1377	* @req: The request to prepare
1378	* @entry_length: The last SG entry size
1379	* @node: Indicates whether this is not the first entry (for isoc only)
1380	*
1381	* Return the number of TRBs prepared.
1382	*/
1383	static int dwc3_prepare_last_sg(struct dwc3_ep *dep,
1384	struct dwc3_request *req, unsigned int entry_length,
1385	unsigned int node)
1386	{
1387	unsigned int maxp = usb_endpoint_maxp(epd: dep->endpoint.desc);
1388	unsigned int rem = req->request.length % maxp;
1389	unsigned int num_trbs = 1;
1390
1391	if (dwc3_needs_extra_trb(dep, req))
1392	num_trbs++;
1393
1394	if (dwc3_calc_trbs_left(dep) < num_trbs)
1395	return 0;
1396
1397	req->needs_extra_trb = num_trbs > 1;
1398
1399	/* Prepare a normal TRB */
1400	if (req->direction || req->request.length)
1401	dwc3_prepare_one_trb(dep, req, trb_length: entry_length,
1402	chain: req->needs_extra_trb, node, use_bounce_buffer: false, must_interrupt: false);
1403
1404	/* Prepare extra TRBs for ZLP and MPS OUT transfer alignment */
1405	if ((!req->direction && !req->request.length) || req->needs_extra_trb)
1406	dwc3_prepare_one_trb(dep, req,
1407	trb_length: req->direction ? 0 : maxp - rem,
1408	chain: false, node: 1, use_bounce_buffer: true, must_interrupt: false);
1409
1410	return num_trbs;
1411	}
1412
1413	static int dwc3_prepare_trbs_sg(struct dwc3_ep *dep,
1414	struct dwc3_request *req)
1415	{
1416	struct scatterlist *sg = req->start_sg;
1417	struct scatterlist *s;
1418	int i;
1419	unsigned int length = req->request.length;
1420	unsigned int remaining = req->request.num_mapped_sgs
1421	- req->num_queued_sgs;
1422	unsigned int num_trbs = req->num_trbs;
1423	bool needs_extra_trb = dwc3_needs_extra_trb(dep, req);
1424
1425	/*
1426	* If we resume preparing the request, then get the remaining length of
1427	* the request and resume where we left off.
1428	*/
1429	for_each_sg(req->request.sg, s, req->num_queued_sgs, i)
1430	length -= sg_dma_len(s);
1431
1432	for_each_sg(sg, s, remaining, i) {
1433	unsigned int num_trbs_left = dwc3_calc_trbs_left(dep);
1434	unsigned int trb_length;
1435	bool must_interrupt = false;
1436	bool last_sg = false;
1437
1438	trb_length = min_t(unsigned int, length, sg_dma_len(s));
1439
1440	length -= trb_length;
1441
1442	/*
1443	* IOMMU driver is coalescing the list of sgs which shares a
1444	* page boundary into one and giving it to USB driver. With
1445	* this the number of sgs mapped is not equal to the number of
1446	* sgs passed. So mark the chain bit to false if it isthe last
1447	* mapped sg.
1448	*/
1449	if ((i == remaining - 1) || !length)
1450	last_sg = true;
1451
1452	if (!num_trbs_left)
1453	break;
1454
1455	if (last_sg) {
1456	if (!dwc3_prepare_last_sg(dep, req, entry_length: trb_length, node: i))
1457	break;
1458	} else {
1459	/*
1460	* Look ahead to check if we have enough TRBs for the
1461	* next SG entry. If not, set interrupt on this TRB to
1462	* resume preparing the next SG entry when more TRBs are
1463	* free.
1464	*/
1465	if (num_trbs_left == 1 || (needs_extra_trb &&
1466	num_trbs_left <= 2 &&
1467	sg_dma_len(sg_next(s)) >= length)) {
1468	struct dwc3_request *r;
1469
1470	/* Check if previous requests already set IOC */
1471	list_for_each_entry(r, &dep->started_list, list) {
1472	if (r != req && !r->request.no_interrupt)
1473	break;
1474
1475	if (r == req)
1476	must_interrupt = true;
1477	}
1478	}
1479
1480	dwc3_prepare_one_trb(dep, req, trb_length, chain: 1, node: i, use_bounce_buffer: false,
1481	must_interrupt);
1482	}
1483
1484	/*
1485	* There can be a situation where all sgs in sglist are not
1486	* queued because of insufficient trb number. To handle this
1487	* case, update start_sg to next sg to be queued, so that
1488	* we have free trbs we can continue queuing from where we
1489	* previously stopped
1490	*/
1491	if (!last_sg)
1492	req->start_sg = sg_next(s);
1493
1494	req->num_queued_sgs++;
1495	req->num_pending_sgs--;
1496
1497	/*
1498	* The number of pending SG entries may not correspond to the
1499	* number of mapped SG entries. If all the data are queued, then
1500	* don't include unused SG entries.
1501	*/
1502	if (length == 0) {
1503	req->num_pending_sgs = 0;
1504	break;
1505	}
1506
1507	if (must_interrupt)
1508	break;
1509	}
1510
1511	return req->num_trbs - num_trbs;
1512	}
1513
1514	static int dwc3_prepare_trbs_linear(struct dwc3_ep *dep,
1515	struct dwc3_request *req)
1516	{
1517	return dwc3_prepare_last_sg(dep, req, entry_length: req->request.length, node: 0);
1518	}
1519
1520	/*
1521	* dwc3_prepare_trbs - setup TRBs from requests
1522	* @dep: endpoint for which requests are being prepared
1523	*
1524	* The function goes through the requests list and sets up TRBs for the
1525	* transfers. The function returns once there are no more TRBs available or
1526	* it runs out of requests.
1527	*
1528	* Returns the number of TRBs prepared or negative errno.
1529	*/
1530	static int dwc3_prepare_trbs(struct dwc3_ep *dep)
1531	{
1532	struct dwc3_request *req, *n;
1533	int ret = 0;
1534
1535	BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
1536
1537	/*
1538	* We can get in a situation where there's a request in the started list
1539	* but there weren't enough TRBs to fully kick it in the first time
1540	* around, so it has been waiting for more TRBs to be freed up.
1541	*
1542	* In that case, we should check if we have a request with pending_sgs
1543	* in the started list and prepare TRBs for that request first,
1544	* otherwise we will prepare TRBs completely out of order and that will
1545	* break things.
1546	*/
1547	list_for_each_entry(req, &dep->started_list, list) {
1548	if (req->num_pending_sgs > 0) {
1549	ret = dwc3_prepare_trbs_sg(dep, req);
1550	if (!ret || req->num_pending_sgs)
1551	return ret;
1552	}
1553
1554	if (!dwc3_calc_trbs_left(dep))
1555	return ret;
1556
1557	/*
1558	* Don't prepare beyond a transfer. In DWC_usb32, its transfer
1559	* burst capability may try to read and use TRBs beyond the
1560	* active transfer instead of stopping.
1561	*/
1562	if (dep->stream_capable && req->request.is_last &&
1563	!DWC3_MST_CAPABLE(&dep->dwc->hwparams))
1564	return ret;
1565	}
1566
1567	list_for_each_entry_safe(req, n, &dep->pending_list, list) {
1568	struct dwc3 *dwc = dep->dwc;
1569
1570	ret = usb_gadget_map_request_by_dev(dev: dwc->sysdev, req: &req->request,
1571	is_in: dep->direction);
1572	if (ret)
1573	return ret;
1574
1575	req->sg = req->request.sg;
1576	req->start_sg = req->sg;
1577	req->num_queued_sgs = 0;
1578	req->num_pending_sgs = req->request.num_mapped_sgs;
1579
1580	if (req->num_pending_sgs > 0) {
1581	ret = dwc3_prepare_trbs_sg(dep, req);
1582	if (req->num_pending_sgs)
1583	return ret;
1584	} else {
1585	ret = dwc3_prepare_trbs_linear(dep, req);
1586	}
1587
1588	if (!ret || !dwc3_calc_trbs_left(dep))
1589	return ret;
1590
1591	/*
1592	* Don't prepare beyond a transfer. In DWC_usb32, its transfer
1593	* burst capability may try to read and use TRBs beyond the
1594	* active transfer instead of stopping.
1595	*/
1596	if (dep->stream_capable && req->request.is_last &&
1597	!DWC3_MST_CAPABLE(&dwc->hwparams))
1598	return ret;
1599	}
1600
1601	return ret;
1602	}
1603
1604	static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep);
1605
1606	static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep)
1607	{
1608	struct dwc3_gadget_ep_cmd_params params;
1609	struct dwc3_request *req;
1610	int starting;
1611	int ret;
1612	u32 cmd;
1613
1614	/*
1615	* Note that it's normal to have no new TRBs prepared (i.e. ret == 0).
1616	* This happens when we need to stop and restart a transfer such as in
1617	* the case of reinitiating a stream or retrying an isoc transfer.
1618	*/
1619	ret = dwc3_prepare_trbs(dep);
1620	if (ret < 0)
1621	return ret;
1622
1623	starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED);
1624
1625	/*
1626	* If there's no new TRB prepared and we don't need to restart a
1627	* transfer, there's no need to update the transfer.
1628	*/
1629	if (!ret && !starting)
1630	return ret;
1631
1632	req = next_request(list: &dep->started_list);
1633	if (!req) {
1634	dep->flags |= DWC3_EP_PENDING_REQUEST;
1635	return 0;
1636	}
1637
1638	memset(&params, 0, sizeof(params));
1639
1640	if (starting) {
1641	params.param0 = upper_32_bits(req->trb_dma);
1642	params.param1 = lower_32_bits(req->trb_dma);
1643	cmd = DWC3_DEPCMD_STARTTRANSFER;
1644
1645	if (dep->stream_capable)
1646	cmd |= DWC3_DEPCMD_PARAM(req->request.stream_id);
1647
1648	if (usb_endpoint_xfer_isoc(epd: dep->endpoint.desc))
1649	cmd |= DWC3_DEPCMD_PARAM(dep->frame_number);
1650	} else {
1651	cmd = DWC3_DEPCMD_UPDATETRANSFER |
1652	DWC3_DEPCMD_PARAM(dep->resource_index);
1653	}
1654
1655	ret = dwc3_send_gadget_ep_cmd(dep, cmd, params: &params);
1656	if (ret < 0) {
1657	struct dwc3_request *tmp;
1658
1659	if (ret == -EAGAIN)
1660	return ret;
1661
1662	dwc3_stop_active_transfer(dep, force: true, interrupt: true);
1663
1664	list_for_each_entry_safe(req, tmp, &dep->started_list, list)
1665	dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_DEQUEUED);
1666
1667	/* If ep isn't started, then there's no end transfer pending */
1668	if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
1669	dwc3_gadget_ep_cleanup_cancelled_requests(dep);
1670
1671	return ret;
1672	}
1673
1674	if (dep->stream_capable && req->request.is_last &&
1675	!DWC3_MST_CAPABLE(&dep->dwc->hwparams))
1676	dep->flags |= DWC3_EP_WAIT_TRANSFER_COMPLETE;
1677
1678	return 0;
1679	}
1680
1681	static int __dwc3_gadget_get_frame(struct dwc3 *dwc)
1682	{
1683	u32 reg;
1684
1685	reg = dwc3_readl(base: dwc->regs, DWC3_DSTS);
1686	return DWC3_DSTS_SOFFN(reg);
1687	}
1688
1689	/**
1690	* __dwc3_stop_active_transfer - stop the current active transfer
1691	* @dep: isoc endpoint
1692	* @force: set forcerm bit in the command
1693	* @interrupt: command complete interrupt after End Transfer command
1694	*
1695	* When setting force, the ForceRM bit will be set. In that case
1696	* the controller won't update the TRB progress on command
1697	* completion. It also won't clear the HWO bit in the TRB.
1698	* The command will also not complete immediately in that case.
1699	*/
1700	static int __dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, bool interrupt)
1701	{
1702	struct dwc3 *dwc = dep->dwc;
1703	struct dwc3_gadget_ep_cmd_params params;
1704	u32 cmd;
1705	int ret;
1706
1707	cmd = DWC3_DEPCMD_ENDTRANSFER;
1708	cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0;
1709	cmd |= interrupt ? DWC3_DEPCMD_CMDIOC : 0;
1710	cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
1711	memset(&params, 0, sizeof(params));
1712	ret = dwc3_send_gadget_ep_cmd(dep, cmd, params: &params);
1713	/*
1714	* If the End Transfer command was timed out while the device is
1715	* not in SETUP phase, it's possible that an incoming Setup packet
1716	* may prevent the command's completion. Let's retry when the
1717	* ep0state returns to EP0_SETUP_PHASE.
1718	*/
1719	if (ret == -ETIMEDOUT && dep->dwc->ep0state != EP0_SETUP_PHASE) {
1720	dep->flags |= DWC3_EP_DELAY_STOP;
1721	return 0;
1722	}
1723	WARN_ON_ONCE(ret);
1724	dep->resource_index = 0;
1725
1726	if (!interrupt) {
1727	if (!DWC3_IP_IS(DWC3) || DWC3_VER_IS_PRIOR(DWC3, 310A))
1728	mdelay(1);
1729	dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
1730	} else if (!ret) {
1731	dep->flags |= DWC3_EP_END_TRANSFER_PENDING;
1732	}
1733
1734	dep->flags &= ~DWC3_EP_DELAY_STOP;
1735	return ret;
1736	}
1737
1738	/**
1739	* dwc3_gadget_start_isoc_quirk - workaround invalid frame number
1740	* @dep: isoc endpoint
1741	*
1742	* This function tests for the correct combination of BIT[15:14] from the 16-bit
1743	* microframe number reported by the XferNotReady event for the future frame
1744	* number to start the isoc transfer.
1745	*
1746	* In DWC_usb31 version 1.70a-ea06 and prior, for highspeed and fullspeed
1747	* isochronous IN, BIT[15:14] of the 16-bit microframe number reported by the
1748	* XferNotReady event are invalid. The driver uses this number to schedule the
1749	* isochronous transfer and passes it to the START TRANSFER command. Because
1750	* this number is invalid, the command may fail. If BIT[15:14] matches the
1751	* internal 16-bit microframe, the START TRANSFER command will pass and the
1752	* transfer will start at the scheduled time, if it is off by 1, the command
1753	* will still pass, but the transfer will start 2 seconds in the future. For all
1754	* other conditions, the START TRANSFER command will fail with bus-expiry.
1755	*
1756	* In order to workaround this issue, we can test for the correct combination of
1757	* BIT[15:14] by sending START TRANSFER commands with different values of
1758	* BIT[15:14]: 'b00, 'b01, 'b10, and 'b11. Each combination is 2^14 uframe apart
1759	* (or 2 seconds). 4 seconds into the future will result in a bus-expiry status.
1760	* As the result, within the 4 possible combinations for BIT[15:14], there will
1761	* be 2 successful and 2 failure START COMMAND status. One of the 2 successful
1762	* command status will result in a 2-second delay start. The smaller BIT[15:14]
1763	* value is the correct combination.
1764	*
1765	* Since there are only 4 outcomes and the results are ordered, we can simply
1766	* test 2 START TRANSFER commands with BIT[15:14] combinations 'b00 and 'b01 to
1767	* deduce the smaller successful combination.
1768	*
1769	* Let test0 = test status for combination 'b00 and test1 = test status for 'b01
1770	* of BIT[15:14]. The correct combination is as follow:
1771	*
1772	* if test0 fails and test1 passes, BIT[15:14] is 'b01
1773	* if test0 fails and test1 fails, BIT[15:14] is 'b10
1774	* if test0 passes and test1 fails, BIT[15:14] is 'b11
1775	* if test0 passes and test1 passes, BIT[15:14] is 'b00
1776	*
1777	* Synopsys STAR 9001202023: Wrong microframe number for isochronous IN
1778	* endpoints.
1779	*/
1780	static int dwc3_gadget_start_isoc_quirk(struct dwc3_ep *dep)
1781	{
1782	int cmd_status = 0;
1783	bool test0;
1784	bool test1;
1785
1786	while (dep->combo_num < 2) {
1787	struct dwc3_gadget_ep_cmd_params params;
1788	u32 test_frame_number;
1789	u32 cmd;
1790
1791	/*
1792	* Check if we can start isoc transfer on the next interval or
1793	* 4 uframes in the future with BIT[15:14] as dep->combo_num
1794	*/
1795	test_frame_number = dep->frame_number & DWC3_FRNUMBER_MASK;
1796	test_frame_number |= dep->combo_num << 14;
1797	test_frame_number += max_t(u32, 4, dep->interval);
1798
1799	params.param0 = upper_32_bits(dep->dwc->bounce_addr);
1800	params.param1 = lower_32_bits(dep->dwc->bounce_addr);
1801
1802	cmd = DWC3_DEPCMD_STARTTRANSFER;
1803	cmd |= DWC3_DEPCMD_PARAM(test_frame_number);
1804	cmd_status = dwc3_send_gadget_ep_cmd(dep, cmd, params: &params);
1805
1806	/* Redo if some other failure beside bus-expiry is received */
1807	if (cmd_status && cmd_status != -EAGAIN) {
1808	dep->start_cmd_status = 0;
1809	dep->combo_num = 0;
1810	return 0;
1811	}
1812
1813	/* Store the first test status */
1814	if (dep->combo_num == 0)
1815	dep->start_cmd_status = cmd_status;
1816
1817	dep->combo_num++;
1818
1819	/*
1820	* End the transfer if the START_TRANSFER command is successful
1821	* to wait for the next XferNotReady to test the command again
1822	*/
1823	if (cmd_status == 0) {
1824	dwc3_stop_active_transfer(dep, force: true, interrupt: true);
1825	return 0;
1826	}
1827	}
1828
1829	/* test0 and test1 are both completed at this point */
1830	test0 = (dep->start_cmd_status == 0);
1831	test1 = (cmd_status == 0);
1832
1833	if (!test0 && test1)
1834	dep->combo_num = 1;
1835	else if (!test0 && !test1)
1836	dep->combo_num = 2;
1837	else if (test0 && !test1)
1838	dep->combo_num = 3;
1839	else if (test0 && test1)
1840	dep->combo_num = 0;
1841
1842	dep->frame_number &= DWC3_FRNUMBER_MASK;
1843	dep->frame_number |= dep->combo_num << 14;
1844	dep->frame_number += max_t(u32, 4, dep->interval);
1845
1846	/* Reinitialize test variables */
1847	dep->start_cmd_status = 0;
1848	dep->combo_num = 0;
1849
1850	return __dwc3_gadget_kick_transfer(dep);
1851	}
1852
1853	static int __dwc3_gadget_start_isoc(struct dwc3_ep *dep)
1854	{
1855	const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
1856	struct dwc3 *dwc = dep->dwc;
1857	int ret;
1858	int i;
1859
1860	if (list_empty(head: &dep->pending_list) &&
1861	list_empty(head: &dep->started_list)) {
1862	dep->flags |= DWC3_EP_PENDING_REQUEST;
1863	return -EAGAIN;
1864	}
1865
1866	if (!dwc->dis_start_transfer_quirk &&
1867	(DWC3_VER_IS_PRIOR(DWC31, 170A) ||
1868	DWC3_VER_TYPE_IS_WITHIN(DWC31, 170A, EA01, EA06))) {
1869	if (dwc->gadget->speed <= USB_SPEED_HIGH && dep->direction)
1870	return dwc3_gadget_start_isoc_quirk(dep);
1871	}
1872
1873	if (desc->bInterval <= 14 &&
1874	dwc->gadget->speed >= USB_SPEED_HIGH) {
1875	u32 frame = __dwc3_gadget_get_frame(dwc);
1876	bool rollover = frame <
1877	(dep->frame_number & DWC3_FRNUMBER_MASK);
1878
1879	/*
1880	* frame_number is set from XferNotReady and may be already
1881	* out of date. DSTS only provides the lower 14 bit of the
1882	* current frame number. So add the upper two bits of
1883	* frame_number and handle a possible rollover.
1884	* This will provide the correct frame_number unless more than
1885	* rollover has happened since XferNotReady.
1886	*/
1887
1888	dep->frame_number = (dep->frame_number & ~DWC3_FRNUMBER_MASK) |
1889	frame;
1890	if (rollover)
1891	dep->frame_number += BIT(14);
1892	}
1893
1894	for (i = 0; i < DWC3_ISOC_MAX_RETRIES; i++) {
1895	int future_interval = i + 1;
1896
1897	/* Give the controller at least 500us to schedule transfers */
1898	if (desc->bInterval < 3)
1899	future_interval += 3 - desc->bInterval;
1900
1901	dep->frame_number = DWC3_ALIGN_FRAME(dep, future_interval);
1902
1903	ret = __dwc3_gadget_kick_transfer(dep);
1904	if (ret != -EAGAIN)
1905	break;
1906	}
1907
1908	/*
1909	* After a number of unsuccessful start attempts due to bus-expiry
1910	* status, issue END_TRANSFER command and retry on the next XferNotReady
1911	* event.
1912	*/
1913	if (ret == -EAGAIN)
1914	ret = __dwc3_stop_active_transfer(dep, force: false, interrupt: true);
1915
1916	return ret;
1917	}
1918
1919	static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
1920	{
1921	struct dwc3 *dwc = dep->dwc;
1922
1923	if (!dep->endpoint.desc || !dwc->pullups_connected || !dwc->connected) {
1924	dev_dbg(dwc->dev, "%s: can't queue to disabled endpoint\n",
1925	dep->name);
1926	return -ESHUTDOWN;
1927	}
1928
1929	if (WARN(req->dep != dep, "request %pK belongs to '%s'\n",
1930	&req->request, req->dep->name))
1931	return -EINVAL;
1932
1933	if (WARN(req->status < DWC3_REQUEST_STATUS_COMPLETED,
1934	"%s: request %pK already in flight\n",
1935	dep->name, &req->request))
1936	return -EINVAL;
1937
1938	pm_runtime_get(dev: dwc->dev);
1939
1940	req->request.actual = 0;
1941	req->request.status = -EINPROGRESS;
1942
1943	trace_dwc3_ep_queue(req);
1944
1945	list_add_tail(new: &req->list, head: &dep->pending_list);
1946	req->status = DWC3_REQUEST_STATUS_QUEUED;
1947
1948	if (dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE)
1949	return 0;
1950
1951	/*
1952	* Start the transfer only after the END_TRANSFER is completed
1953	* and endpoint STALL is cleared.
1954	*/
1955	if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) ||
1956	(dep->flags & DWC3_EP_WEDGE) ||
1957	(dep->flags & DWC3_EP_DELAY_STOP) ||
1958	(dep->flags & DWC3_EP_STALL)) {
1959	dep->flags |= DWC3_EP_DELAY_START;
1960	return 0;
1961	}
1962
1963	/*
1964	* NOTICE: Isochronous endpoints should NEVER be prestarted. We must
1965	* wait for a XferNotReady event so we will know what's the current
1966	* (micro-)frame number.
1967	*
1968	* Without this trick, we are very, very likely gonna get Bus Expiry
1969	* errors which will force us issue EndTransfer command.
1970	*/
1971	if (usb_endpoint_xfer_isoc(epd: dep->endpoint.desc)) {
1972	if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) {
1973	if ((dep->flags & DWC3_EP_PENDING_REQUEST))
1974	return __dwc3_gadget_start_isoc(dep);
1975
1976	return 0;
1977	}
1978	}
1979
1980	__dwc3_gadget_kick_transfer(dep);
1981
1982	return 0;
1983	}
1984
1985	static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
1986	gfp_t gfp_flags)
1987	{
1988	struct dwc3_request *req = to_dwc3_request(request);
1989	struct dwc3_ep *dep = to_dwc3_ep(ep);
1990	struct dwc3 *dwc = dep->dwc;
1991
1992	unsigned long flags;
1993
1994	int ret;
1995
1996	spin_lock_irqsave(&dwc->lock, flags);
1997	ret = __dwc3_gadget_ep_queue(dep, req);
1998	spin_unlock_irqrestore(lock: &dwc->lock, flags);
1999
2000	return ret;
2001	}
2002
2003	static void dwc3_gadget_ep_skip_trbs(struct dwc3_ep *dep, struct dwc3_request *req)
2004	{
2005	int i;
2006
2007	/* If req->trb is not set, then the request has not started */
2008	if (!req->trb)
2009	return;
2010
2011	/*
2012	* If request was already started, this means we had to
2013	* stop the transfer. With that we also need to ignore
2014	* all TRBs used by the request, however TRBs can only
2015	* be modified after completion of END_TRANSFER
2016	* command. So what we do here is that we wait for
2017	* END_TRANSFER completion and only after that, we jump
2018	* over TRBs by clearing HWO and incrementing dequeue
2019	* pointer.
2020	*/
2021	for (i = 0; i < req->num_trbs; i++) {
2022	struct dwc3_trb *trb;
2023
2024	trb = &dep->trb_pool[dep->trb_dequeue];
2025	trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
2026	dwc3_ep_inc_deq(dep);
2027	}
2028
2029	req->num_trbs = 0;
2030	}
2031
2032	static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep)
2033	{
2034	struct dwc3_request *req;
2035	struct dwc3 *dwc = dep->dwc;
2036
2037	while (!list_empty(head: &dep->cancelled_list)) {
2038	req = next_request(list: &dep->cancelled_list);
2039	dwc3_gadget_ep_skip_trbs(dep, req);
2040	switch (req->status) {
2041	case DWC3_REQUEST_STATUS_DISCONNECTED:
2042	dwc3_gadget_giveback(dep, req, status: -ESHUTDOWN);
2043	break;
2044	case DWC3_REQUEST_STATUS_DEQUEUED:
2045	dwc3_gadget_giveback(dep, req, status: -ECONNRESET);
2046	break;
2047	case DWC3_REQUEST_STATUS_STALLED:
2048	dwc3_gadget_giveback(dep, req, status: -EPIPE);
2049	break;
2050	default:
2051	dev_err(dwc->dev, "request cancelled with wrong reason:%d\n", req->status);
2052	dwc3_gadget_giveback(dep, req, status: -ECONNRESET);
2053	break;
2054	}
2055	/*
2056	* The endpoint is disabled, let the dwc3_remove_requests()
2057	* handle the cleanup.
2058	*/
2059	if (!dep->endpoint.desc)
2060	break;
2061	}
2062	}
2063
2064	static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
2065	struct usb_request *request)
2066	{
2067	struct dwc3_request *req = to_dwc3_request(request);
2068	struct dwc3_request *r = NULL;
2069
2070	struct dwc3_ep *dep = to_dwc3_ep(ep);
2071	struct dwc3 *dwc = dep->dwc;
2072
2073	unsigned long flags;
2074	int ret = 0;
2075
2076	trace_dwc3_ep_dequeue(req);
2077
2078	spin_lock_irqsave(&dwc->lock, flags);
2079
2080	list_for_each_entry(r, &dep->cancelled_list, list) {
2081	if (r == req)
2082	goto out;
2083	}
2084
2085	list_for_each_entry(r, &dep->pending_list, list) {
2086	if (r == req) {
2087	/*
2088	* Explicitly check for EP0/1 as dequeue for those
2089	* EPs need to be handled differently. Control EP
2090	* only deals with one USB req, and giveback will
2091	* occur during dwc3_ep0_stall_and_restart(). EP0
2092	* requests are never added to started_list.
2093	*/
2094	if (dep->number > 1)
2095	dwc3_gadget_giveback(dep, req, status: -ECONNRESET);
2096	else
2097	dwc3_ep0_reset_state(dwc);
2098	goto out;
2099	}
2100	}
2101
2102	list_for_each_entry(r, &dep->started_list, list) {
2103	if (r == req) {
2104	struct dwc3_request *t;
2105
2106	/* wait until it is processed */
2107	dwc3_stop_active_transfer(dep, force: true, interrupt: true);
2108
2109	/*
2110	* Remove any started request if the transfer is
2111	* cancelled.
2112	*/
2113	list_for_each_entry_safe(r, t, &dep->started_list, list)
2114	dwc3_gadget_move_cancelled_request(req: r,
2115	DWC3_REQUEST_STATUS_DEQUEUED);
2116
2117	dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE;
2118
2119	goto out;
2120	}
2121	}
2122
2123	dev_err(dwc->dev, "request %pK was not queued to %s\n",
2124	request, ep->name);
2125	ret = -EINVAL;
2126	out:
2127	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2128
2129	return ret;
2130	}
2131
2132	int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
2133	{
2134	struct dwc3_gadget_ep_cmd_params params;
2135	struct dwc3 *dwc = dep->dwc;
2136	struct dwc3_request *req;
2137	struct dwc3_request *tmp;
2138	int ret;
2139
2140	if (usb_endpoint_xfer_isoc(epd: dep->endpoint.desc)) {
2141	dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
2142	return -EINVAL;
2143	}
2144
2145	memset(&params, 0x00, sizeof(params));
2146
2147	if (value) {
2148	struct dwc3_trb *trb;
2149
2150	unsigned int transfer_in_flight;
2151	unsigned int started;
2152
2153	if (dep->number > 1)
2154	trb = dwc3_ep_prev_trb(dep, index: dep->trb_enqueue);
2155	else
2156	trb = &dwc->ep0_trb[dep->trb_enqueue];
2157
2158	transfer_in_flight = trb->ctrl & DWC3_TRB_CTRL_HWO;
2159	started = !list_empty(head: &dep->started_list);
2160
2161	if (!protocol && ((dep->direction && transfer_in_flight) ||
2162	(!dep->direction && started))) {
2163	return -EAGAIN;
2164	}
2165
2166	ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETSTALL,
2167	params: &params);
2168	if (ret)
2169	dev_err(dwc->dev, "failed to set STALL on %s\n",
2170	dep->name);
2171	else
2172	dep->flags |= DWC3_EP_STALL;
2173	} else {
2174	/*
2175	* Don't issue CLEAR_STALL command to control endpoints. The
2176	* controller automatically clears the STALL when it receives
2177	* the SETUP token.
2178	*/
2179	if (dep->number <= 1) {
2180	dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
2181	return 0;
2182	}
2183
2184	dwc3_stop_active_transfer(dep, force: true, interrupt: true);
2185
2186	list_for_each_entry_safe(req, tmp, &dep->started_list, list)
2187	dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_STALLED);
2188
2189	if (dep->flags & DWC3_EP_END_TRANSFER_PENDING ||
2190	(dep->flags & DWC3_EP_DELAY_STOP)) {
2191	dep->flags |= DWC3_EP_PENDING_CLEAR_STALL;
2192	if (protocol)
2193	dwc->clear_stall_protocol = dep->number;
2194
2195	return 0;
2196	}
2197
2198	dwc3_gadget_ep_cleanup_cancelled_requests(dep);
2199
2200	ret = dwc3_send_clear_stall_ep_cmd(dep);
2201	if (ret) {
2202	dev_err(dwc->dev, "failed to clear STALL on %s\n",
2203	dep->name);
2204	return ret;
2205	}
2206
2207	dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
2208
2209	if ((dep->flags & DWC3_EP_DELAY_START) &&
2210	!usb_endpoint_xfer_isoc(epd: dep->endpoint.desc))
2211	__dwc3_gadget_kick_transfer(dep);
2212
2213	dep->flags &= ~DWC3_EP_DELAY_START;
2214	}
2215
2216	return ret;
2217	}
2218
2219	static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
2220	{
2221	struct dwc3_ep *dep = to_dwc3_ep(ep);
2222	struct dwc3 *dwc = dep->dwc;
2223
2224	unsigned long flags;
2225
2226	int ret;
2227
2228	spin_lock_irqsave(&dwc->lock, flags);
2229	ret = __dwc3_gadget_ep_set_halt(dep, value, protocol: false);
2230	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2231
2232	return ret;
2233	}
2234
2235	static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
2236	{
2237	struct dwc3_ep *dep = to_dwc3_ep(ep);
2238	struct dwc3 *dwc = dep->dwc;
2239	unsigned long flags;
2240	int ret;
2241
2242	spin_lock_irqsave(&dwc->lock, flags);
2243	dep->flags |= DWC3_EP_WEDGE;
2244
2245	if (dep->number == 0 || dep->number == 1)
2246	ret = __dwc3_gadget_ep0_set_halt(ep, value: 1);
2247	else
2248	ret = __dwc3_gadget_ep_set_halt(dep, value: 1, protocol: false);
2249	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2250
2251	return ret;
2252	}
2253
2254	/* -------------------------------------------------------------------------- */
2255
2256	static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
2257	.bLength = USB_DT_ENDPOINT_SIZE,
2258	.bDescriptorType = USB_DT_ENDPOINT,
2259	.bmAttributes = USB_ENDPOINT_XFER_CONTROL,
2260	};
2261
2262	static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
2263	.enable = dwc3_gadget_ep0_enable,
2264	.disable = dwc3_gadget_ep0_disable,
2265	.alloc_request = dwc3_gadget_ep_alloc_request,
2266	.free_request = dwc3_gadget_ep_free_request,
2267	.queue = dwc3_gadget_ep0_queue,
2268	.dequeue = dwc3_gadget_ep_dequeue,
2269	.set_halt = dwc3_gadget_ep0_set_halt,
2270	.set_wedge = dwc3_gadget_ep_set_wedge,
2271	};
2272
2273	static const struct usb_ep_ops dwc3_gadget_ep_ops = {
2274	.enable = dwc3_gadget_ep_enable,
2275	.disable = dwc3_gadget_ep_disable,
2276	.alloc_request = dwc3_gadget_ep_alloc_request,
2277	.free_request = dwc3_gadget_ep_free_request,
2278	.queue = dwc3_gadget_ep_queue,
2279	.dequeue = dwc3_gadget_ep_dequeue,
2280	.set_halt = dwc3_gadget_ep_set_halt,
2281	.set_wedge = dwc3_gadget_ep_set_wedge,
2282	};
2283
2284	/* -------------------------------------------------------------------------- */
2285
2286	static void dwc3_gadget_enable_linksts_evts(struct dwc3 *dwc, bool set)
2287	{
2288	u32 reg;
2289
2290	if (DWC3_VER_IS_PRIOR(DWC3, 250A))
2291	return;
2292
2293	reg = dwc3_readl(base: dwc->regs, DWC3_DEVTEN);
2294	if (set)
2295	reg |= DWC3_DEVTEN_ULSTCNGEN;
2296	else
2297	reg &= ~DWC3_DEVTEN_ULSTCNGEN;
2298
2299	dwc3_writel(base: dwc->regs, DWC3_DEVTEN, value: reg);
2300	}
2301
2302	static int dwc3_gadget_get_frame(struct usb_gadget *g)
2303	{
2304	struct dwc3 *dwc = gadget_to_dwc(g);
2305
2306	return __dwc3_gadget_get_frame(dwc);
2307	}
2308
2309	static int __dwc3_gadget_wakeup(struct dwc3 *dwc, bool async)
2310	{
2311	int retries;
2312
2313	int ret;
2314	u32 reg;
2315
2316	u8 link_state;
2317
2318	/*
2319	* According to the Databook Remote wakeup request should
2320	* be issued only when the device is in early suspend state.
2321	*
2322	* We can check that via USB Link State bits in DSTS register.
2323	*/
2324	reg = dwc3_readl(base: dwc->regs, DWC3_DSTS);
2325
2326	link_state = DWC3_DSTS_USBLNKST(reg);
2327
2328	switch (link_state) {
2329	case DWC3_LINK_STATE_RESET:
2330	case DWC3_LINK_STATE_RX_DET: /* in HS, means Early Suspend */
2331	case DWC3_LINK_STATE_U3: /* in HS, means SUSPEND */
2332	case DWC3_LINK_STATE_U2: /* in HS, means Sleep (L1) */
2333	case DWC3_LINK_STATE_U1:
2334	case DWC3_LINK_STATE_RESUME:
2335	break;
2336	default:
2337	return -EINVAL;
2338	}
2339
2340	if (async)
2341	dwc3_gadget_enable_linksts_evts(dwc, set: true);
2342
2343	ret = dwc3_gadget_set_link_state(dwc, state: DWC3_LINK_STATE_RECOV);
2344	if (ret < 0) {
2345	dev_err(dwc->dev, "failed to put link in Recovery\n");
2346	dwc3_gadget_enable_linksts_evts(dwc, set: false);
2347	return ret;
2348	}
2349
2350	/* Recent versions do this automatically */
2351	if (DWC3_VER_IS_PRIOR(DWC3, 194A)) {
2352	/* write zeroes to Link Change Request */
2353	reg = dwc3_readl(base: dwc->regs, DWC3_DCTL);
2354	reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
2355	dwc3_writel(base: dwc->regs, DWC3_DCTL, value: reg);
2356	}
2357
2358	/*
2359	* Since link status change events are enabled we will receive
2360	* an U0 event when wakeup is successful. So bail out.
2361	*/
2362	if (async)
2363	return 0;
2364
2365	/* poll until Link State changes to ON */
2366	retries = 20000;
2367
2368	while (retries--) {
2369	reg = dwc3_readl(base: dwc->regs, DWC3_DSTS);
2370
2371	/* in HS, means ON */
2372	if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
2373	break;
2374	}
2375
2376	if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
2377	dev_err(dwc->dev, "failed to send remote wakeup\n");
2378	return -EINVAL;
2379	}
2380
2381	return 0;
2382	}
2383
2384	static int dwc3_gadget_wakeup(struct usb_gadget *g)
2385	{
2386	struct dwc3 *dwc = gadget_to_dwc(g);
2387	unsigned long flags;
2388	int ret;
2389
2390	if (!dwc->wakeup_configured) {
2391	dev_err(dwc->dev, "remote wakeup not configured\n");
2392	return -EINVAL;
2393	}
2394
2395	spin_lock_irqsave(&dwc->lock, flags);
2396	if (!dwc->gadget->wakeup_armed) {
2397	dev_err(dwc->dev, "not armed for remote wakeup\n");
2398	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2399	return -EINVAL;
2400	}
2401	ret = __dwc3_gadget_wakeup(dwc, async: true);
2402
2403	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2404
2405	return ret;
2406	}
2407
2408	static void dwc3_resume_gadget(struct dwc3 *dwc);
2409
2410	static int dwc3_gadget_func_wakeup(struct usb_gadget *g, int intf_id)
2411	{
2412	struct dwc3 *dwc = gadget_to_dwc(g);
2413	unsigned long flags;
2414	int ret;
2415	int link_state;
2416
2417	if (!dwc->wakeup_configured) {
2418	dev_err(dwc->dev, "remote wakeup not configured\n");
2419	return -EINVAL;
2420	}
2421
2422	spin_lock_irqsave(&dwc->lock, flags);
2423	/*
2424	* If the link is in U3, signal for remote wakeup and wait for the
2425	* link to transition to U0 before sending device notification.
2426	*/
2427	link_state = dwc3_gadget_get_link_state(dwc);
2428	if (link_state == DWC3_LINK_STATE_U3) {
2429	ret = __dwc3_gadget_wakeup(dwc, async: false);
2430	if (ret) {
2431	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2432	return -EINVAL;
2433	}
2434	dwc3_resume_gadget(dwc);
2435	dwc->suspended = false;
2436	dwc->link_state = DWC3_LINK_STATE_U0;
2437	}
2438
2439	ret = dwc3_send_gadget_generic_command(dwc, DWC3_DGCMD_DEV_NOTIFICATION,
2440	DWC3_DGCMDPAR_DN_FUNC_WAKE |
2441	DWC3_DGCMDPAR_INTF_SEL(intf_id));
2442	if (ret)
2443	dev_err(dwc->dev, "function remote wakeup failed, ret:%d\n", ret);
2444
2445	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2446
2447	return ret;
2448	}
2449
2450	static int dwc3_gadget_set_remote_wakeup(struct usb_gadget *g, int set)
2451	{
2452	struct dwc3 *dwc = gadget_to_dwc(g);
2453	unsigned long flags;
2454
2455	spin_lock_irqsave(&dwc->lock, flags);
2456	dwc->wakeup_configured = !!set;
2457	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2458
2459	return 0;
2460	}
2461
2462	static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
2463	int is_selfpowered)
2464	{
2465	struct dwc3 *dwc = gadget_to_dwc(g);
2466	unsigned long flags;
2467
2468	spin_lock_irqsave(&dwc->lock, flags);
2469	g->is_selfpowered = !!is_selfpowered;
2470	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2471
2472	return 0;
2473	}
2474
2475	static void dwc3_stop_active_transfers(struct dwc3 *dwc)
2476	{
2477	u32 epnum;
2478
2479	for (epnum = 2; epnum < dwc->num_eps; epnum++) {
2480	struct dwc3_ep *dep;
2481
2482	dep = dwc->eps[epnum];
2483	if (!dep)
2484	continue;
2485
2486	dwc3_remove_requests(dwc, dep, status: -ESHUTDOWN);
2487	}
2488	}
2489
2490	static void __dwc3_gadget_set_ssp_rate(struct dwc3 *dwc)
2491	{
2492	enum usb_ssp_rate ssp_rate = dwc->gadget_ssp_rate;
2493	u32 reg;
2494
2495	if (ssp_rate == USB_SSP_GEN_UNKNOWN)
2496	ssp_rate = dwc->max_ssp_rate;
2497
2498	reg = dwc3_readl(base: dwc->regs, DWC3_DCFG);
2499	reg &= ~DWC3_DCFG_SPEED_MASK;
2500	reg &= ~DWC3_DCFG_NUMLANES(~0);
2501
2502	if (ssp_rate == USB_SSP_GEN_1x2)
2503	reg |= DWC3_DCFG_SUPERSPEED;
2504	else if (dwc->max_ssp_rate != USB_SSP_GEN_1x2)
2505	reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2506
2507	if (ssp_rate != USB_SSP_GEN_2x1 &&
2508	dwc->max_ssp_rate != USB_SSP_GEN_2x1)
2509	reg |= DWC3_DCFG_NUMLANES(1);
2510
2511	dwc3_writel(base: dwc->regs, DWC3_DCFG, value: reg);
2512	}
2513
2514	static void __dwc3_gadget_set_speed(struct dwc3 *dwc)
2515	{
2516	enum usb_device_speed speed;
2517	u32 reg;
2518
2519	speed = dwc->gadget_max_speed;
2520	if (speed == USB_SPEED_UNKNOWN || speed > dwc->maximum_speed)
2521	speed = dwc->maximum_speed;
2522
2523	if (speed == USB_SPEED_SUPER_PLUS &&
2524	DWC3_IP_IS(DWC32)) {
2525	__dwc3_gadget_set_ssp_rate(dwc);
2526	return;
2527	}
2528
2529	reg = dwc3_readl(base: dwc->regs, DWC3_DCFG);
2530	reg &= ~(DWC3_DCFG_SPEED_MASK);
2531
2532	/*
2533	* WORKAROUND: DWC3 revision < 2.20a have an issue
2534	* which would cause metastability state on Run/Stop
2535	* bit if we try to force the IP to USB2-only mode.
2536	*
2537	* Because of that, we cannot configure the IP to any
2538	* speed other than the SuperSpeed
2539	*
2540	* Refers to:
2541	*
2542	* STAR#9000525659: Clock Domain Crossing on DCTL in
2543	* USB 2.0 Mode
2544	*/
2545	if (DWC3_VER_IS_PRIOR(DWC3, 220A) &&
2546	!dwc->dis_metastability_quirk) {
2547	reg |= DWC3_DCFG_SUPERSPEED;
2548	} else {
2549	switch (speed) {
2550	case USB_SPEED_FULL:
2551	reg |= DWC3_DCFG_FULLSPEED;
2552	break;
2553	case USB_SPEED_HIGH:
2554	reg |= DWC3_DCFG_HIGHSPEED;
2555	break;
2556	case USB_SPEED_SUPER:
2557	reg |= DWC3_DCFG_SUPERSPEED;
2558	break;
2559	case USB_SPEED_SUPER_PLUS:
2560	if (DWC3_IP_IS(DWC3))
2561	reg |= DWC3_DCFG_SUPERSPEED;
2562	else
2563	reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2564	break;
2565	default:
2566	dev_err(dwc->dev, "invalid speed (%d)\n", speed);
2567
2568	if (DWC3_IP_IS(DWC3))
2569	reg |= DWC3_DCFG_SUPERSPEED;
2570	else
2571	reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2572	}
2573	}
2574
2575	if (DWC3_IP_IS(DWC32) &&
2576	speed > USB_SPEED_UNKNOWN &&
2577	speed < USB_SPEED_SUPER_PLUS)
2578	reg &= ~DWC3_DCFG_NUMLANES(~0);
2579
2580	dwc3_writel(base: dwc->regs, DWC3_DCFG, value: reg);
2581	}
2582
2583	static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on)
2584	{
2585	u32 reg;
2586	u32 timeout = 2000;
2587
2588	if (pm_runtime_suspended(dev: dwc->dev))
2589	return 0;
2590
2591	reg = dwc3_readl(base: dwc->regs, DWC3_DCTL);
2592	if (is_on) {
2593	if (DWC3_VER_IS_WITHIN(DWC3, ANY, 187A)) {
2594	reg &= ~DWC3_DCTL_TRGTULST_MASK;
2595	reg |= DWC3_DCTL_TRGTULST_RX_DET;
2596	}
2597
2598	if (!DWC3_VER_IS_PRIOR(DWC3, 194A))
2599	reg &= ~DWC3_DCTL_KEEP_CONNECT;
2600	reg |= DWC3_DCTL_RUN_STOP;
2601
2602	__dwc3_gadget_set_speed(dwc);
2603	dwc->pullups_connected = true;
2604	} else {
2605	reg &= ~DWC3_DCTL_RUN_STOP;
2606
2607	dwc->pullups_connected = false;
2608	}
2609
2610	dwc3_gadget_dctl_write_safe(dwc, value: reg);
2611
2612	do {
2613	usleep_range(min: 1000, max: 2000);
2614	reg = dwc3_readl(base: dwc->regs, DWC3_DSTS);
2615	reg &= DWC3_DSTS_DEVCTRLHLT;
2616	} while (--timeout && !(!is_on ^ !reg));
2617
2618	if (!timeout)
2619	return -ETIMEDOUT;
2620
2621	return 0;
2622	}
2623
2624	static void dwc3_gadget_disable_irq(struct dwc3 *dwc);
2625	static void __dwc3_gadget_stop(struct dwc3 *dwc);
2626	static int __dwc3_gadget_start(struct dwc3 *dwc);
2627
2628	static int dwc3_gadget_soft_disconnect(struct dwc3 *dwc)
2629	{
2630	unsigned long flags;
2631	int ret;
2632
2633	spin_lock_irqsave(&dwc->lock, flags);
2634	if (!dwc->pullups_connected) {
2635	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2636	return 0;
2637	}
2638
2639	dwc->connected = false;
2640
2641	/*
2642	* Attempt to end pending SETUP status phase, and not wait for the
2643	* function to do so.
2644	*/
2645	if (dwc->delayed_status)
2646	dwc3_ep0_send_delayed_status(dwc);
2647
2648	/*
2649	* In the Synopsys DesignWare Cores USB3 Databook Rev. 3.30a
2650	* Section 4.1.8 Table 4-7, it states that for a device-initiated
2651	* disconnect, the SW needs to ensure that it sends "a DEPENDXFER
2652	* command for any active transfers" before clearing the RunStop
2653	* bit.
2654	*/
2655	dwc3_stop_active_transfers(dwc);
2656	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2657
2658	/*
2659	* Per databook, when we want to stop the gadget, if a control transfer
2660	* is still in process, complete it and get the core into setup phase.
2661	* In case the host is unresponsive to a SETUP transaction, forcefully
2662	* stall the transfer, and move back to the SETUP phase, so that any
2663	* pending endxfers can be executed.
2664	*/
2665	if (dwc->ep0state != EP0_SETUP_PHASE) {
2666	reinit_completion(x: &dwc->ep0_in_setup);
2667
2668	ret = wait_for_completion_timeout(x: &dwc->ep0_in_setup,
2669	timeout: msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT));
2670	if (ret == 0) {
2671	dev_warn(dwc->dev, "wait for SETUP phase timed out\n");
2672	spin_lock_irqsave(&dwc->lock, flags);
2673	dwc3_ep0_reset_state(dwc);
2674	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2675	}
2676	}
2677
2678	/*
2679	* Note: if the GEVNTCOUNT indicates events in the event buffer, the
2680	* driver needs to acknowledge them before the controller can halt.
2681	* Simply let the interrupt handler acknowledges and handle the
2682	* remaining event generated by the controller while polling for
2683	* DSTS.DEVCTLHLT.
2684	*/
2685	ret = dwc3_gadget_run_stop(dwc, is_on: false);
2686
2687	/*
2688	* Stop the gadget after controller is halted, so that if needed, the
2689	* events to update EP0 state can still occur while the run/stop
2690	* routine polls for the halted state. DEVTEN is cleared as part of
2691	* gadget stop.
2692	*/
2693	spin_lock_irqsave(&dwc->lock, flags);
2694	__dwc3_gadget_stop(dwc);
2695	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2696
2697	return ret;
2698	}
2699
2700	static int dwc3_gadget_soft_connect(struct dwc3 *dwc)
2701	{
2702	int ret;
2703
2704	/*
2705	* In the Synopsys DWC_usb31 1.90a programming guide section
2706	* 4.1.9, it specifies that for a reconnect after a
2707	* device-initiated disconnect requires a core soft reset
2708	* (DCTL.CSftRst) before enabling the run/stop bit.
2709	*/
2710	ret = dwc3_core_soft_reset(dwc);
2711	if (ret)
2712	return ret;
2713
2714	dwc3_event_buffers_setup(dwc);
2715	__dwc3_gadget_start(dwc);
2716	return dwc3_gadget_run_stop(dwc, is_on: true);
2717	}
2718
2719	static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
2720	{
2721	struct dwc3 *dwc = gadget_to_dwc(g);
2722	int ret;
2723
2724	is_on = !!is_on;
2725
2726	dwc->softconnect = is_on;
2727
2728	/*
2729	* Avoid issuing a runtime resume if the device is already in the
2730	* suspended state during gadget disconnect. DWC3 gadget was already
2731	* halted/stopped during runtime suspend.
2732	*/
2733	if (!is_on) {
2734	pm_runtime_barrier(dev: dwc->dev);
2735	if (pm_runtime_suspended(dev: dwc->dev))
2736	return 0;
2737	}
2738
2739	/*
2740	* Check the return value for successful resume, or error. For a
2741	* successful resume, the DWC3 runtime PM resume routine will handle
2742	* the run stop sequence, so avoid duplicate operations here.
2743	*/
2744	ret = pm_runtime_get_sync(dev: dwc->dev);
2745	if (!ret || ret < 0) {
2746	pm_runtime_put(dev: dwc->dev);
2747	if (ret < 0)
2748	pm_runtime_set_suspended(dev: dwc->dev);
2749	return ret;
2750	}
2751
2752	if (dwc->pullups_connected == is_on) {
2753	pm_runtime_put(dev: dwc->dev);
2754	return 0;
2755	}
2756
2757	synchronize_irq(irq: dwc->irq_gadget);
2758
2759	if (!is_on)
2760	ret = dwc3_gadget_soft_disconnect(dwc);
2761	else
2762	ret = dwc3_gadget_soft_connect(dwc);
2763
2764	pm_runtime_put(dev: dwc->dev);
2765
2766	return ret;
2767	}
2768
2769	static void dwc3_gadget_enable_irq(struct dwc3 *dwc)
2770	{
2771	u32 reg;
2772
2773	/* Enable all but Start and End of Frame IRQs */
2774	reg = (DWC3_DEVTEN_EVNTOVERFLOWEN |
2775	DWC3_DEVTEN_CMDCMPLTEN |
2776	DWC3_DEVTEN_ERRTICERREN |
2777	DWC3_DEVTEN_WKUPEVTEN |
2778	DWC3_DEVTEN_CONNECTDONEEN |
2779	DWC3_DEVTEN_USBRSTEN |
2780	DWC3_DEVTEN_DISCONNEVTEN);
2781
2782	if (DWC3_VER_IS_PRIOR(DWC3, 250A))
2783	reg |= DWC3_DEVTEN_ULSTCNGEN;
2784
2785	/* On 2.30a and above this bit enables U3/L2-L1 Suspend Events */
2786	if (!DWC3_VER_IS_PRIOR(DWC3, 230A))
2787	reg |= DWC3_DEVTEN_U3L2L1SUSPEN;
2788
2789	dwc3_writel(base: dwc->regs, DWC3_DEVTEN, value: reg);
2790	}
2791
2792	static void dwc3_gadget_disable_irq(struct dwc3 *dwc)
2793	{
2794	/* mask all interrupts */
2795	dwc3_writel(base: dwc->regs, DWC3_DEVTEN, value: 0x00);
2796	}
2797
2798	static irqreturn_t dwc3_interrupt(int irq, void *_dwc);
2799	static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc);
2800
2801	/**
2802	* dwc3_gadget_setup_nump - calculate and initialize NUMP field of %DWC3_DCFG
2803	* @dwc: pointer to our context structure
2804	*
2805	* The following looks like complex but it's actually very simple. In order to
2806	* calculate the number of packets we can burst at once on OUT transfers, we're
2807	* gonna use RxFIFO size.
2808	*
2809	* To calculate RxFIFO size we need two numbers:
2810	* MDWIDTH = size, in bits, of the internal memory bus
2811	* RAM2_DEPTH = depth, in MDWIDTH, of internal RAM2 (where RxFIFO sits)
2812	*
2813	* Given these two numbers, the formula is simple:
2814	*
2815	* RxFIFO Size = (RAM2_DEPTH * MDWIDTH / 8) - 24 - 16;
2816	*
2817	* 24 bytes is for 3x SETUP packets
2818	* 16 bytes is a clock domain crossing tolerance
2819	*
2820	* Given RxFIFO Size, NUMP = RxFIFOSize / 1024;
2821	*/
2822	static void dwc3_gadget_setup_nump(struct dwc3 *dwc)
2823	{
2824	u32 ram2_depth;
2825	u32 mdwidth;
2826	u32 nump;
2827	u32 reg;
2828
2829	ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7);
2830	mdwidth = dwc3_mdwidth(dwc);
2831
2832	nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024;
2833	nump = min_t(u32, nump, 16);
2834
2835	/* update NumP */
2836	reg = dwc3_readl(base: dwc->regs, DWC3_DCFG);
2837	reg &= ~DWC3_DCFG_NUMP_MASK;
2838	reg |= nump << DWC3_DCFG_NUMP_SHIFT;
2839	dwc3_writel(base: dwc->regs, DWC3_DCFG, value: reg);
2840	}
2841
2842	static int __dwc3_gadget_start(struct dwc3 *dwc)
2843	{
2844	struct dwc3_ep *dep;
2845	int ret = 0;
2846	u32 reg;
2847
2848	/*
2849	* Use IMOD if enabled via dwc->imod_interval. Otherwise, if
2850	* the core supports IMOD, disable it.
2851	*/
2852	if (dwc->imod_interval) {
2853	dwc3_writel(base: dwc->regs, DWC3_DEV_IMOD(0), value: dwc->imod_interval);
2854	dwc3_writel(base: dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
2855	} else if (dwc3_has_imod(dwc)) {
2856	dwc3_writel(base: dwc->regs, DWC3_DEV_IMOD(0), value: 0);
2857	}
2858
2859	/*
2860	* We are telling dwc3 that we want to use DCFG.NUMP as ACK TP's NUMP
2861	* field instead of letting dwc3 itself calculate that automatically.
2862	*
2863	* This way, we maximize the chances that we'll be able to get several
2864	* bursts of data without going through any sort of endpoint throttling.
2865	*/
2866	reg = dwc3_readl(base: dwc->regs, DWC3_GRXTHRCFG);
2867	if (DWC3_IP_IS(DWC3))
2868	reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL;
2869	else
2870	reg &= ~DWC31_GRXTHRCFG_PKTCNTSEL;
2871
2872	dwc3_writel(base: dwc->regs, DWC3_GRXTHRCFG, value: reg);
2873
2874	dwc3_gadget_setup_nump(dwc);
2875
2876	/*
2877	* Currently the controller handles single stream only. So, Ignore
2878	* Packet Pending bit for stream selection and don't search for another
2879	* stream if the host sends Data Packet with PP=0 (for OUT direction) or
2880	* ACK with NumP=0 and PP=0 (for IN direction). This slightly improves
2881	* the stream performance.
2882	*/
2883	reg = dwc3_readl(base: dwc->regs, DWC3_DCFG);
2884	reg |= DWC3_DCFG_IGNSTRMPP;
2885	dwc3_writel(base: dwc->regs, DWC3_DCFG, value: reg);
2886
2887	/* Enable MST by default if the device is capable of MST */
2888	if (DWC3_MST_CAPABLE(&dwc->hwparams)) {
2889	reg = dwc3_readl(base: dwc->regs, DWC3_DCFG1);
2890	reg &= ~DWC3_DCFG1_DIS_MST_ENH;
2891	dwc3_writel(base: dwc->regs, DWC3_DCFG1, value: reg);
2892	}
2893
2894	/* Start with SuperSpeed Default */
2895	dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2896
2897	ret = dwc3_gadget_start_config(dwc, resource_index: 0);
2898	if (ret) {
2899	dev_err(dwc->dev, "failed to config endpoints\n");
2900	return ret;
2901	}
2902
2903	dep = dwc->eps[0];
2904	dep->flags = 0;
2905	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
2906	if (ret) {
2907	dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2908	goto err0;
2909	}
2910
2911	dep = dwc->eps[1];
2912	dep->flags = 0;
2913	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
2914	if (ret) {
2915	dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2916	goto err1;
2917	}
2918
2919	/* begin to receive SETUP packets */
2920	dwc->ep0state = EP0_SETUP_PHASE;
2921	dwc->ep0_bounced = false;
2922	dwc->link_state = DWC3_LINK_STATE_SS_DIS;
2923	dwc->delayed_status = false;
2924	dwc3_ep0_out_start(dwc);
2925
2926	dwc3_gadget_enable_irq(dwc);
2927
2928	return 0;
2929
2930	err1:
2931	__dwc3_gadget_ep_disable(dep: dwc->eps[0]);
2932
2933	err0:
2934	return ret;
2935	}
2936
2937	static int dwc3_gadget_start(struct usb_gadget *g,
2938	struct usb_gadget_driver *driver)
2939	{
2940	struct dwc3 *dwc = gadget_to_dwc(g);
2941	unsigned long flags;
2942	int ret;
2943	int irq;
2944
2945	irq = dwc->irq_gadget;
2946	ret = request_threaded_irq(irq, handler: dwc3_interrupt, thread_fn: dwc3_thread_interrupt,
2947	IRQF_SHARED, name: "dwc3", dev: dwc->ev_buf);
2948	if (ret) {
2949	dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
2950	irq, ret);
2951	return ret;
2952	}
2953
2954	spin_lock_irqsave(&dwc->lock, flags);
2955	dwc->gadget_driver = driver;
2956	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2957
2958	if (dwc->sys_wakeup)
2959	device_wakeup_enable(dev: dwc->sysdev);
2960
2961	return 0;
2962	}
2963
2964	static void __dwc3_gadget_stop(struct dwc3 *dwc)
2965	{
2966	dwc3_gadget_disable_irq(dwc);
2967	__dwc3_gadget_ep_disable(dep: dwc->eps[0]);
2968	__dwc3_gadget_ep_disable(dep: dwc->eps[1]);
2969	}
2970
2971	static int dwc3_gadget_stop(struct usb_gadget *g)
2972	{
2973	struct dwc3 *dwc = gadget_to_dwc(g);
2974	unsigned long flags;
2975
2976	if (dwc->sys_wakeup)
2977	device_wakeup_disable(dev: dwc->sysdev);
2978
2979	spin_lock_irqsave(&dwc->lock, flags);
2980	dwc->gadget_driver = NULL;
2981	dwc->max_cfg_eps = 0;
2982	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2983
2984	free_irq(dwc->irq_gadget, dwc->ev_buf);
2985
2986	return 0;
2987	}
2988
2989	static void dwc3_gadget_config_params(struct usb_gadget *g,
2990	struct usb_dcd_config_params *params)
2991	{
2992	struct dwc3 *dwc = gadget_to_dwc(g);
2993
2994	params->besl_baseline = USB_DEFAULT_BESL_UNSPECIFIED;
2995	params->besl_deep = USB_DEFAULT_BESL_UNSPECIFIED;
2996
2997	/* Recommended BESL */
2998	if (!dwc->dis_enblslpm_quirk) {
2999	/*
3000	* If the recommended BESL baseline is 0 or if the BESL deep is
3001	* less than 2, Microsoft's Windows 10 host usb stack will issue
3002	* a usb reset immediately after it receives the extended BOS
3003	* descriptor and the enumeration will fail. To maintain
3004	* compatibility with the Windows' usb stack, let's set the
3005	* recommended BESL baseline to 1 and clamp the BESL deep to be
3006	* within 2 to 15.
3007	*/
3008	params->besl_baseline = 1;
3009	if (dwc->is_utmi_l1_suspend)
3010	params->besl_deep =
3011	clamp_t(u8, dwc->hird_threshold, 2, 15);
3012	}
3013
3014	/* U1 Device exit Latency */
3015	if (dwc->dis_u1_entry_quirk)
3016	params->bU1devExitLat = 0;
3017	else
3018	params->bU1devExitLat = DWC3_DEFAULT_U1_DEV_EXIT_LAT;
3019
3020	/* U2 Device exit Latency */
3021	if (dwc->dis_u2_entry_quirk)
3022	params->bU2DevExitLat = 0;
3023	else
3024	params->bU2DevExitLat =
3025	cpu_to_le16(DWC3_DEFAULT_U2_DEV_EXIT_LAT);
3026	}
3027
3028	static void dwc3_gadget_set_speed(struct usb_gadget *g,
3029	enum usb_device_speed speed)
3030	{
3031	struct dwc3 *dwc = gadget_to_dwc(g);
3032	unsigned long flags;
3033
3034	spin_lock_irqsave(&dwc->lock, flags);
3035	dwc->gadget_max_speed = speed;
3036	spin_unlock_irqrestore(lock: &dwc->lock, flags);
3037	}
3038
3039	static void dwc3_gadget_set_ssp_rate(struct usb_gadget *g,
3040	enum usb_ssp_rate rate)
3041	{
3042	struct dwc3 *dwc = gadget_to_dwc(g);
3043	unsigned long flags;
3044
3045	spin_lock_irqsave(&dwc->lock, flags);
3046	dwc->gadget_max_speed = USB_SPEED_SUPER_PLUS;
3047	dwc->gadget_ssp_rate = rate;
3048	spin_unlock_irqrestore(lock: &dwc->lock, flags);
3049	}
3050
3051	static int dwc3_gadget_vbus_draw(struct usb_gadget *g, unsigned int mA)
3052	{
3053	struct dwc3 *dwc = gadget_to_dwc(g);
3054	union power_supply_propval val = {0};
3055	int ret;
3056
3057	if (dwc->usb2_phy)
3058	return usb_phy_set_power(x: dwc->usb2_phy, mA);
3059
3060	if (!dwc->usb_psy)
3061	return -EOPNOTSUPP;
3062
3063	val.intval = 1000 * mA;
3064	ret = power_supply_set_property(psy: dwc->usb_psy, psp: POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT, val: &val);
3065
3066	return ret;
3067	}
3068
3069	/**
3070	* dwc3_gadget_check_config - ensure dwc3 can support the USB configuration
3071	* @g: pointer to the USB gadget
3072	*
3073	* Used to record the maximum number of endpoints being used in a USB composite
3074	* device. (across all configurations) This is to be used in the calculation
3075	* of the TXFIFO sizes when resizing internal memory for individual endpoints.
3076	* It will help ensured that the resizing logic reserves enough space for at
3077	* least one max packet.
3078	*/
3079	static int dwc3_gadget_check_config(struct usb_gadget *g)
3080	{
3081	struct dwc3 *dwc = gadget_to_dwc(g);
3082	struct usb_ep *ep;
3083	int fifo_size = 0;
3084	int ram1_depth;
3085	int ep_num = 0;
3086
3087	if (!dwc->do_fifo_resize)
3088	return 0;
3089
3090	list_for_each_entry(ep, &g->ep_list, ep_list) {
3091	/* Only interested in the IN endpoints */
3092	if (ep->claimed && (ep->address & USB_DIR_IN))
3093	ep_num++;
3094	}
3095
3096	if (ep_num <= dwc->max_cfg_eps)
3097	return 0;
3098
3099	/* Update the max number of eps in the composition */
3100	dwc->max_cfg_eps = ep_num;
3101
3102	fifo_size = dwc3_gadget_calc_tx_fifo_size(dwc, mult: dwc->max_cfg_eps);
3103	/* Based on the equation, increment by one for every ep */
3104	fifo_size += dwc->max_cfg_eps;
3105
3106	/* Check if we can fit a single fifo per endpoint */
3107	ram1_depth = DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7);
3108	if (fifo_size > ram1_depth)
3109	return -ENOMEM;
3110
3111	return 0;
3112	}
3113
3114	static void dwc3_gadget_async_callbacks(struct usb_gadget *g, bool enable)
3115	{
3116	struct dwc3 *dwc = gadget_to_dwc(g);
3117	unsigned long flags;
3118
3119	spin_lock_irqsave(&dwc->lock, flags);
3120	dwc->async_callbacks = enable;
3121	spin_unlock_irqrestore(lock: &dwc->lock, flags);
3122	}
3123
3124	static const struct usb_gadget_ops dwc3_gadget_ops = {
3125	.get_frame = dwc3_gadget_get_frame,
3126	.wakeup = dwc3_gadget_wakeup,
3127	.func_wakeup = dwc3_gadget_func_wakeup,
3128	.set_remote_wakeup = dwc3_gadget_set_remote_wakeup,
3129	.set_selfpowered = dwc3_gadget_set_selfpowered,
3130	.pullup = dwc3_gadget_pullup,
3131	.udc_start = dwc3_gadget_start,
3132	.udc_stop = dwc3_gadget_stop,
3133	.udc_set_speed = dwc3_gadget_set_speed,
3134	.udc_set_ssp_rate = dwc3_gadget_set_ssp_rate,
3135	.get_config_params = dwc3_gadget_config_params,
3136	.vbus_draw = dwc3_gadget_vbus_draw,
3137	.check_config = dwc3_gadget_check_config,
3138	.udc_async_callbacks = dwc3_gadget_async_callbacks,
3139	};
3140
3141	/* -------------------------------------------------------------------------- */
3142
3143	static int dwc3_gadget_init_control_endpoint(struct dwc3_ep *dep)
3144	{
3145	struct dwc3 *dwc = dep->dwc;
3146
3147	usb_ep_set_maxpacket_limit(ep: &dep->endpoint, maxpacket_limit: 512);
3148	dep->endpoint.maxburst = 1;
3149	dep->endpoint.ops = &dwc3_gadget_ep0_ops;
3150	if (!dep->direction)
3151	dwc->gadget->ep0 = &dep->endpoint;
3152
3153	dep->endpoint.caps.type_control = true;
3154
3155	return 0;
3156	}
3157
3158	static int dwc3_gadget_init_in_endpoint(struct dwc3_ep *dep)
3159	{
3160	struct dwc3 *dwc = dep->dwc;
3161	u32 mdwidth;
3162	int size;
3163	int maxpacket;
3164
3165	mdwidth = dwc3_mdwidth(dwc);
3166
3167	/* MDWIDTH is represented in bits, we need it in bytes */
3168	mdwidth /= 8;
3169
3170	size = dwc3_readl(base: dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1));
3171	if (DWC3_IP_IS(DWC3))
3172	size = DWC3_GTXFIFOSIZ_TXFDEP(size);
3173	else
3174	size = DWC31_GTXFIFOSIZ_TXFDEP(size);
3175
3176	/*
3177	* maxpacket size is determined as part of the following, after assuming
3178	* a mult value of one maxpacket:
3179	* DWC3 revision 280A and prior:
3180	* fifo_size = mult * (max_packet / mdwidth) + 1;
3181	* maxpacket = mdwidth * (fifo_size - 1);
3182	*
3183	* DWC3 revision 290A and onwards:
3184	* fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1
3185	* maxpacket = mdwidth * ((fifo_size - 1) - 1) - mdwidth;
3186	*/
3187	if (DWC3_VER_IS_PRIOR(DWC3, 290A))
3188	maxpacket = mdwidth * (size - 1);
3189	else
3190	maxpacket = mdwidth * ((size - 1) - 1) - mdwidth;
3191
3192	/* Functionally, space for one max packet is sufficient */
3193	size = min_t(int, maxpacket, 1024);
3194	usb_ep_set_maxpacket_limit(ep: &dep->endpoint, maxpacket_limit: size);
3195
3196	dep->endpoint.max_streams = 16;
3197	dep->endpoint.ops = &dwc3_gadget_ep_ops;
3198	list_add_tail(new: &dep->endpoint.ep_list,
3199	head: &dwc->gadget->ep_list);
3200	dep->endpoint.caps.type_iso = true;
3201	dep->endpoint.caps.type_bulk = true;
3202	dep->endpoint.caps.type_int = true;
3203
3204	return dwc3_alloc_trb_pool(dep);
3205	}
3206
3207	static int dwc3_gadget_init_out_endpoint(struct dwc3_ep *dep)
3208	{
3209	struct dwc3 *dwc = dep->dwc;
3210	u32 mdwidth;
3211	int size;
3212
3213	mdwidth = dwc3_mdwidth(dwc);
3214
3215	/* MDWIDTH is represented in bits, convert to bytes */
3216	mdwidth /= 8;
3217
3218	/* All OUT endpoints share a single RxFIFO space */
3219	size = dwc3_readl(base: dwc->regs, DWC3_GRXFIFOSIZ(0));
3220	if (DWC3_IP_IS(DWC3))
3221	size = DWC3_GRXFIFOSIZ_RXFDEP(size);
3222	else
3223	size = DWC31_GRXFIFOSIZ_RXFDEP(size);
3224
3225	/* FIFO depth is in MDWDITH bytes */
3226	size *= mdwidth;
3227
3228	/*
3229	* To meet performance requirement, a minimum recommended RxFIFO size
3230	* is defined as follow:
3231	* RxFIFO size >= (3 x MaxPacketSize) +
3232	* (3 x 8 bytes setup packets size) + (16 bytes clock crossing margin)
3233	*
3234	* Then calculate the max packet limit as below.
3235	*/
3236	size -= (3 * 8) + 16;
3237	if (size < 0)
3238	size = 0;
3239	else
3240	size /= 3;
3241
3242	usb_ep_set_maxpacket_limit(ep: &dep->endpoint, maxpacket_limit: size);
3243	dep->endpoint.max_streams = 16;
3244	dep->endpoint.ops = &dwc3_gadget_ep_ops;
3245	list_add_tail(new: &dep->endpoint.ep_list,
3246	head: &dwc->gadget->ep_list);
3247	dep->endpoint.caps.type_iso = true;
3248	dep->endpoint.caps.type_bulk = true;
3249	dep->endpoint.caps.type_int = true;
3250
3251	return dwc3_alloc_trb_pool(dep);
3252	}
3253
3254	static int dwc3_gadget_init_endpoint(struct dwc3 *dwc, u8 epnum)
3255	{
3256	struct dwc3_ep *dep;
3257	bool direction = epnum & 1;
3258	int ret;
3259	u8 num = epnum >> 1;
3260
3261	dep = kzalloc(size: sizeof(*dep), GFP_KERNEL);
3262	if (!dep)
3263	return -ENOMEM;
3264
3265	dep->dwc = dwc;
3266	dep->number = epnum;
3267	dep->direction = direction;
3268	dep->regs = dwc->regs + DWC3_DEP_BASE(epnum);
3269	dwc->eps[epnum] = dep;
3270	dep->combo_num = 0;
3271	dep->start_cmd_status = 0;
3272
3273	snprintf(buf: dep->name, size: sizeof(dep->name), fmt: "ep%u%s", num,
3274	direction ? "in" : "out");
3275
3276	dep->endpoint.name = dep->name;
3277
3278	if (!(dep->number > 1)) {
3279	dep->endpoint.desc = &dwc3_gadget_ep0_desc;
3280	dep->endpoint.comp_desc = NULL;
3281	}
3282
3283	if (num == 0)
3284	ret = dwc3_gadget_init_control_endpoint(dep);
3285	else if (direction)
3286	ret = dwc3_gadget_init_in_endpoint(dep);
3287	else
3288	ret = dwc3_gadget_init_out_endpoint(dep);
3289
3290	if (ret)
3291	return ret;
3292
3293	dep->endpoint.caps.dir_in = direction;
3294	dep->endpoint.caps.dir_out = !direction;
3295
3296	INIT_LIST_HEAD(list: &dep->pending_list);
3297	INIT_LIST_HEAD(list: &dep->started_list);
3298	INIT_LIST_HEAD(list: &dep->cancelled_list);
3299
3300	dwc3_debugfs_create_endpoint_dir(dep);
3301
3302	return 0;
3303	}
3304
3305	static int dwc3_gadget_init_endpoints(struct dwc3 *dwc, u8 total)
3306	{
3307	u8 epnum;
3308
3309	INIT_LIST_HEAD(list: &dwc->gadget->ep_list);
3310
3311	for (epnum = 0; epnum < total; epnum++) {
3312	int ret;
3313
3314	ret = dwc3_gadget_init_endpoint(dwc, epnum);
3315	if (ret)
3316	return ret;
3317	}
3318
3319	return 0;
3320	}
3321
3322	static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
3323	{
3324	struct dwc3_ep *dep;
3325	u8 epnum;
3326
3327	for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
3328	dep = dwc->eps[epnum];
3329	if (!dep)
3330	continue;
3331	/*
3332	* Physical endpoints 0 and 1 are special; they form the
3333	* bi-directional USB endpoint 0.
3334	*
3335	* For those two physical endpoints, we don't allocate a TRB
3336	* pool nor do we add them the endpoints list. Due to that, we
3337	* shouldn't do these two operations otherwise we would end up
3338	* with all sorts of bugs when removing dwc3.ko.
3339	*/
3340	if (epnum != 0 && epnum != 1) {
3341	dwc3_free_trb_pool(dep);
3342	list_del(entry: &dep->endpoint.ep_list);
3343	}
3344
3345	dwc3_debugfs_remove_endpoint_dir(dep);
3346	kfree(objp: dep);
3347	}
3348	}
3349
3350	/* -------------------------------------------------------------------------- */
3351
3352	static int dwc3_gadget_ep_reclaim_completed_trb(struct dwc3_ep *dep,
3353	struct dwc3_request *req, struct dwc3_trb *trb,
3354	const struct dwc3_event_depevt *event, int status, int chain)
3355	{
3356	unsigned int count;
3357
3358	dwc3_ep_inc_deq(dep);
3359
3360	trace_dwc3_complete_trb(dep, trb);
3361	req->num_trbs--;
3362
3363	/*
3364	* If we're in the middle of series of chained TRBs and we
3365	* receive a short transfer along the way, DWC3 will skip
3366	* through all TRBs including the last TRB in the chain (the
3367	* where CHN bit is zero. DWC3 will also avoid clearing HWO
3368	* bit and SW has to do it manually.
3369	*
3370	* We're going to do that here to avoid problems of HW trying
3371	* to use bogus TRBs for transfers.
3372	*/
3373	if (chain && (trb->ctrl & DWC3_TRB_CTRL_HWO))
3374	trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
3375
3376	/*
3377	* For isochronous transfers, the first TRB in a service interval must
3378	* have the Isoc-First type. Track and report its interval frame number.
3379	*/
3380	if (usb_endpoint_xfer_isoc(epd: dep->endpoint.desc) &&
3381	(trb->ctrl & DWC3_TRBCTL_ISOCHRONOUS_FIRST)) {
3382	unsigned int frame_number;
3383
3384	frame_number = DWC3_TRB_CTRL_GET_SID_SOFN(trb->ctrl);
3385	frame_number &= ~(dep->interval - 1);
3386	req->request.frame_number = frame_number;
3387	}
3388
3389	/*
3390	* We use bounce buffer for requests that needs extra TRB or OUT ZLP. If
3391	* this TRB points to the bounce buffer address, it's a MPS alignment
3392	* TRB. Don't add it to req->remaining calculation.
3393	*/
3394	if (trb->bpl == lower_32_bits(dep->dwc->bounce_addr) &&
3395	trb->bph == upper_32_bits(dep->dwc->bounce_addr)) {
3396	trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
3397	return 1;
3398	}
3399
3400	count = trb->size & DWC3_TRB_SIZE_MASK;
3401	req->remaining += count;
3402
3403	if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
3404	return 1;
3405
3406	if (event->status & DEPEVT_STATUS_SHORT && !chain)
3407	return 1;
3408
3409	if ((trb->ctrl & DWC3_TRB_CTRL_ISP_IMI) &&
3410	DWC3_TRB_SIZE_TRBSTS(trb->size) == DWC3_TRBSTS_MISSED_ISOC)
3411	return 1;
3412
3413	if ((trb->ctrl & DWC3_TRB_CTRL_IOC) ||
3414	(trb->ctrl & DWC3_TRB_CTRL_LST))
3415	return 1;
3416
3417	return 0;
3418	}
3419
3420	static int dwc3_gadget_ep_reclaim_trb_sg(struct dwc3_ep *dep,
3421	struct dwc3_request *req, const struct dwc3_event_depevt *event,
3422	int status)
3423	{
3424	struct dwc3_trb *trb;
3425	struct scatterlist *sg = req->sg;
3426	struct scatterlist *s;
3427	unsigned int num_queued = req->num_queued_sgs;
3428	unsigned int i;
3429	int ret = 0;
3430
3431	for_each_sg(sg, s, num_queued, i) {
3432	trb = &dep->trb_pool[dep->trb_dequeue];
3433
3434	req->sg = sg_next(s);
3435	req->num_queued_sgs--;
3436
3437	ret = dwc3_gadget_ep_reclaim_completed_trb(dep, req,
3438	trb, event, status, chain: true);
3439	if (ret)
3440	break;
3441	}
3442
3443	return ret;
3444	}
3445
3446	static int dwc3_gadget_ep_reclaim_trb_linear(struct dwc3_ep *dep,
3447	struct dwc3_request *req, const struct dwc3_event_depevt *event,
3448	int status)
3449	{
3450	struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
3451
3452	return dwc3_gadget_ep_reclaim_completed_trb(dep, req, trb,
3453	event, status, chain: false);
3454	}
3455
3456	static bool dwc3_gadget_ep_request_completed(struct dwc3_request *req)
3457	{
3458	return req->num_pending_sgs == 0 && req->num_queued_sgs == 0;
3459	}
3460
3461	static int dwc3_gadget_ep_cleanup_completed_request(struct dwc3_ep *dep,
3462	const struct dwc3_event_depevt *event,
3463	struct dwc3_request *req, int status)
3464	{
3465	int request_status;
3466	int ret;
3467
3468	if (req->request.num_mapped_sgs)
3469	ret = dwc3_gadget_ep_reclaim_trb_sg(dep, req, event,
3470	status);
3471	else
3472	ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
3473	status);
3474
3475	req->request.actual = req->request.length - req->remaining;
3476
3477	if (!dwc3_gadget_ep_request_completed(req))
3478	goto out;
3479
3480	if (req->needs_extra_trb) {
3481	ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
3482	status);
3483	req->needs_extra_trb = false;
3484	}
3485
3486	/*
3487	* The event status only reflects the status of the TRB with IOC set.
3488	* For the requests that don't set interrupt on completion, the driver
3489	* needs to check and return the status of the completed TRBs associated
3490	* with the request. Use the status of the last TRB of the request.
3491	*/
3492	if (req->request.no_interrupt) {
3493	struct dwc3_trb *trb;
3494
3495	trb = dwc3_ep_prev_trb(dep, index: dep->trb_dequeue);
3496	switch (DWC3_TRB_SIZE_TRBSTS(trb->size)) {
3497	case DWC3_TRBSTS_MISSED_ISOC:
3498	/* Isoc endpoint only */
3499	request_status = -EXDEV;
3500	break;
3501	case DWC3_TRB_STS_XFER_IN_PROG:
3502	/* Applicable when End Transfer with ForceRM=0 */
3503	case DWC3_TRBSTS_SETUP_PENDING:
3504	/* Control endpoint only */
3505	case DWC3_TRBSTS_OK:
3506	default:
3507	request_status = 0;
3508	break;
3509	}
3510	} else {
3511	request_status = status;
3512	}
3513
3514	dwc3_gadget_giveback(dep, req, status: request_status);
3515
3516	out:
3517	return ret;
3518	}
3519
3520	static void dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep *dep,
3521	const struct dwc3_event_depevt *event, int status)
3522	{
3523	struct dwc3_request *req;
3524
3525	while (!list_empty(head: &dep->started_list)) {
3526	int ret;
3527
3528	req = next_request(list: &dep->started_list);
3529	ret = dwc3_gadget_ep_cleanup_completed_request(dep, event,
3530	req, status);
3531	if (ret)
3532	break;
3533	/*
3534	* The endpoint is disabled, let the dwc3_remove_requests()
3535	* handle the cleanup.
3536	*/
3537	if (!dep->endpoint.desc)
3538	break;
3539	}
3540	}
3541
3542	static bool dwc3_gadget_ep_should_continue(struct dwc3_ep *dep)
3543	{
3544	struct dwc3_request *req;
3545	struct dwc3 *dwc = dep->dwc;
3546
3547	if (!dep->endpoint.desc || !dwc->pullups_connected ||
3548	!dwc->connected)
3549	return false;
3550
3551	if (!list_empty(head: &dep->pending_list))
3552	return true;
3553
3554	/*
3555	* We only need to check the first entry of the started list. We can
3556	* assume the completed requests are removed from the started list.
3557	*/
3558	req = next_request(list: &dep->started_list);
3559	if (!req)
3560	return false;
3561
3562	return !dwc3_gadget_ep_request_completed(req);
3563	}
3564
3565	static void dwc3_gadget_endpoint_frame_from_event(struct dwc3_ep *dep,
3566	const struct dwc3_event_depevt *event)
3567	{
3568	dep->frame_number = event->parameters;
3569	}
3570
3571	static bool dwc3_gadget_endpoint_trbs_complete(struct dwc3_ep *dep,
3572	const struct dwc3_event_depevt *event, int status)
3573	{
3574	struct dwc3 *dwc = dep->dwc;
3575	bool no_started_trb = true;
3576
3577	dwc3_gadget_ep_cleanup_completed_requests(dep, event, status);
3578
3579	if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)
3580	goto out;
3581
3582	if (!dep->endpoint.desc)
3583	return no_started_trb;
3584
3585	if (usb_endpoint_xfer_isoc(epd: dep->endpoint.desc) &&
3586	list_empty(head: &dep->started_list) &&
3587	(list_empty(head: &dep->pending_list) || status == -EXDEV))
3588	dwc3_stop_active_transfer(dep, force: true, interrupt: true);
3589	else if (dwc3_gadget_ep_should_continue(dep))
3590	if (__dwc3_gadget_kick_transfer(dep) == 0)
3591	no_started_trb = false;
3592
3593	out:
3594	/*
3595	* WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.
3596	* See dwc3_gadget_linksts_change_interrupt() for 1st half.
3597	*/
3598	if (DWC3_VER_IS_PRIOR(DWC3, 183A)) {
3599	u32 reg;
3600	int i;
3601
3602	for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
3603	dep = dwc->eps[i];
3604
3605	if (!(dep->flags & DWC3_EP_ENABLED))
3606	continue;
3607
3608	if (!list_empty(head: &dep->started_list))
3609	return no_started_trb;
3610	}
3611
3612	reg = dwc3_readl(base: dwc->regs, DWC3_DCTL);
3613	reg |= dwc->u1u2;
3614	dwc3_writel(base: dwc->regs, DWC3_DCTL, value: reg);
3615
3616	dwc->u1u2 = 0;
3617	}
3618
3619	return no_started_trb;
3620	}
3621
3622	static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep,
3623	const struct dwc3_event_depevt *event)
3624	{
3625	int status = 0;
3626
3627	if (!dep->endpoint.desc)
3628	return;
3629
3630	if (usb_endpoint_xfer_isoc(epd: dep->endpoint.desc))
3631	dwc3_gadget_endpoint_frame_from_event(dep, event);
3632
3633	if (event->status & DEPEVT_STATUS_BUSERR)
3634	status = -ECONNRESET;
3635
3636	if (event->status & DEPEVT_STATUS_MISSED_ISOC)
3637	status = -EXDEV;
3638
3639	dwc3_gadget_endpoint_trbs_complete(dep, event, status);
3640	}
3641
3642	static void dwc3_gadget_endpoint_transfer_complete(struct dwc3_ep *dep,
3643	const struct dwc3_event_depevt *event)
3644	{
3645	int status = 0;
3646
3647	dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
3648
3649	if (event->status & DEPEVT_STATUS_BUSERR)
3650	status = -ECONNRESET;
3651
3652	if (dwc3_gadget_endpoint_trbs_complete(dep, event, status))
3653	dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE;
3654	}
3655
3656	static void dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep *dep,
3657	const struct dwc3_event_depevt *event)
3658	{
3659	dwc3_gadget_endpoint_frame_from_event(dep, event);
3660
3661	/*
3662	* The XferNotReady event is generated only once before the endpoint
3663	* starts. It will be generated again when END_TRANSFER command is
3664	* issued. For some controller versions, the XferNotReady event may be
3665	* generated while the END_TRANSFER command is still in process. Ignore
3666	* it and wait for the next XferNotReady event after the command is
3667	* completed.
3668	*/
3669	if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)
3670	return;
3671
3672	(void) __dwc3_gadget_start_isoc(dep);
3673	}
3674
3675	static void dwc3_gadget_endpoint_command_complete(struct dwc3_ep *dep,
3676	const struct dwc3_event_depevt *event)
3677	{
3678	u8 cmd = DEPEVT_PARAMETER_CMD(event->parameters);
3679
3680	if (cmd != DWC3_DEPCMD_ENDTRANSFER)
3681	return;
3682
3683	/*
3684	* The END_TRANSFER command will cause the controller to generate a
3685	* NoStream Event, and it's not due to the host DP NoStream rejection.
3686	* Ignore the next NoStream event.
3687	*/
3688	if (dep->stream_capable)
3689	dep->flags |= DWC3_EP_IGNORE_NEXT_NOSTREAM;
3690
3691	dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
3692	dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
3693	dwc3_gadget_ep_cleanup_cancelled_requests(dep);
3694
3695	if (dep->flags & DWC3_EP_PENDING_CLEAR_STALL) {
3696	struct dwc3 *dwc = dep->dwc;
3697
3698	dep->flags &= ~DWC3_EP_PENDING_CLEAR_STALL;
3699	if (dwc3_send_clear_stall_ep_cmd(dep)) {
3700	struct usb_ep *ep0 = &dwc->eps[0]->endpoint;
3701
3702	dev_err(dwc->dev, "failed to clear STALL on %s\n", dep->name);
3703	if (dwc->delayed_status)
3704	__dwc3_gadget_ep0_set_halt(ep: ep0, value: 1);
3705	return;
3706	}
3707
3708	dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
3709	if (dwc->clear_stall_protocol == dep->number)
3710	dwc3_ep0_send_delayed_status(dwc);
3711	}
3712
3713	if ((dep->flags & DWC3_EP_DELAY_START) &&
3714	!usb_endpoint_xfer_isoc(epd: dep->endpoint.desc))
3715	__dwc3_gadget_kick_transfer(dep);
3716
3717	dep->flags &= ~DWC3_EP_DELAY_START;
3718	}
3719
3720	static void dwc3_gadget_endpoint_stream_event(struct dwc3_ep *dep,
3721	const struct dwc3_event_depevt *event)
3722	{
3723	struct dwc3 *dwc = dep->dwc;
3724
3725	if (event->status == DEPEVT_STREAMEVT_FOUND) {
3726	dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED;
3727	goto out;
3728	}
3729
3730	/* Note: NoStream rejection event param value is 0 and not 0xFFFF */
3731	switch (event->parameters) {
3732	case DEPEVT_STREAM_PRIME:
3733	/*
3734	* If the host can properly transition the endpoint state from
3735	* idle to prime after a NoStream rejection, there's no need to
3736	* force restarting the endpoint to reinitiate the stream. To
3737	* simplify the check, assume the host follows the USB spec if
3738	* it primed the endpoint more than once.
3739	*/
3740	if (dep->flags & DWC3_EP_FORCE_RESTART_STREAM) {
3741	if (dep->flags & DWC3_EP_FIRST_STREAM_PRIMED)
3742	dep->flags &= ~DWC3_EP_FORCE_RESTART_STREAM;
3743	else
3744	dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED;
3745	}
3746
3747	break;
3748	case DEPEVT_STREAM_NOSTREAM:
3749	if ((dep->flags & DWC3_EP_IGNORE_NEXT_NOSTREAM) ||
3750	!(dep->flags & DWC3_EP_FORCE_RESTART_STREAM) ||
3751	(!DWC3_MST_CAPABLE(&dwc->hwparams) &&
3752	!(dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE)))
3753	break;
3754
3755	/*
3756	* If the host rejects a stream due to no active stream, by the
3757	* USB and xHCI spec, the endpoint will be put back to idle
3758	* state. When the host is ready (buffer added/updated), it will
3759	* prime the endpoint to inform the usb device controller. This
3760	* triggers the device controller to issue ERDY to restart the
3761	* stream. However, some hosts don't follow this and keep the
3762	* endpoint in the idle state. No prime will come despite host
3763	* streams are updated, and the device controller will not be
3764	* triggered to generate ERDY to move the next stream data. To
3765	* workaround this and maintain compatibility with various
3766	* hosts, force to reinitiate the stream until the host is ready
3767	* instead of waiting for the host to prime the endpoint.
3768	*/
3769	if (DWC3_VER_IS_WITHIN(DWC32, 100A, ANY)) {
3770	unsigned int cmd = DWC3_DGCMD_SET_ENDPOINT_PRIME;
3771
3772	dwc3_send_gadget_generic_command(dwc, cmd, param: dep->number);
3773	} else {
3774	dep->flags |= DWC3_EP_DELAY_START;
3775	dwc3_stop_active_transfer(dep, force: true, interrupt: true);
3776	return;
3777	}
3778	break;
3779	}
3780
3781	out:
3782	dep->flags &= ~DWC3_EP_IGNORE_NEXT_NOSTREAM;
3783	}
3784
3785	static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
3786	const struct dwc3_event_depevt *event)
3787	{
3788	struct dwc3_ep *dep;
3789	u8 epnum = event->endpoint_number;
3790
3791	dep = dwc->eps[epnum];
3792
3793	if (!(dep->flags & DWC3_EP_ENABLED)) {
3794	if ((epnum > 1) && !(dep->flags & DWC3_EP_TRANSFER_STARTED))
3795	return;
3796
3797	/* Handle only EPCMDCMPLT when EP disabled */
3798	if ((event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT) &&
3799	!(epnum <= 1 && event->endpoint_event == DWC3_DEPEVT_XFERCOMPLETE))
3800	return;
3801	}
3802
3803	if (epnum == 0 || epnum == 1) {
3804	dwc3_ep0_interrupt(dwc, event);
3805	return;
3806	}
3807
3808	switch (event->endpoint_event) {
3809	case DWC3_DEPEVT_XFERINPROGRESS:
3810	dwc3_gadget_endpoint_transfer_in_progress(dep, event);
3811	break;
3812	case DWC3_DEPEVT_XFERNOTREADY:
3813	dwc3_gadget_endpoint_transfer_not_ready(dep, event);
3814	break;
3815	case DWC3_DEPEVT_EPCMDCMPLT:
3816	dwc3_gadget_endpoint_command_complete(dep, event);
3817	break;
3818	case DWC3_DEPEVT_XFERCOMPLETE:
3819	dwc3_gadget_endpoint_transfer_complete(dep, event);
3820	break;
3821	case DWC3_DEPEVT_STREAMEVT:
3822	dwc3_gadget_endpoint_stream_event(dep, event);
3823	break;
3824	case DWC3_DEPEVT_RXTXFIFOEVT:
3825	break;
3826	default:
3827	dev_err(dwc->dev, "unknown endpoint event %d\n", event->endpoint_event);
3828	break;
3829	}
3830	}
3831
3832	static void dwc3_disconnect_gadget(struct dwc3 *dwc)
3833	{
3834	if (dwc->async_callbacks && dwc->gadget_driver->disconnect) {
3835	spin_unlock(lock: &dwc->lock);
3836	dwc->gadget_driver->disconnect(dwc->gadget);
3837	spin_lock(lock: &dwc->lock);
3838	}
3839	}
3840
3841	static void dwc3_suspend_gadget(struct dwc3 *dwc)
3842	{
3843	if (dwc->async_callbacks && dwc->gadget_driver->suspend) {
3844	spin_unlock(lock: &dwc->lock);
3845	dwc->gadget_driver->suspend(dwc->gadget);
3846	spin_lock(lock: &dwc->lock);
3847	}
3848	}
3849
3850	static void dwc3_resume_gadget(struct dwc3 *dwc)
3851	{
3852	if (dwc->async_callbacks && dwc->gadget_driver->resume) {
3853	spin_unlock(lock: &dwc->lock);
3854	dwc->gadget_driver->resume(dwc->gadget);
3855	spin_lock(lock: &dwc->lock);
3856	}
3857	}
3858
3859	static void dwc3_reset_gadget(struct dwc3 *dwc)
3860	{
3861	if (!dwc->gadget_driver)
3862	return;
3863
3864	if (dwc->async_callbacks && dwc->gadget->speed != USB_SPEED_UNKNOWN) {
3865	spin_unlock(lock: &dwc->lock);
3866	usb_gadget_udc_reset(gadget: dwc->gadget, driver: dwc->gadget_driver);
3867	spin_lock(lock: &dwc->lock);
3868	}
3869	}
3870
3871	void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force,
3872	bool interrupt)
3873	{
3874	struct dwc3 *dwc = dep->dwc;
3875
3876	/*
3877	* Only issue End Transfer command to the control endpoint of a started
3878	* Data Phase. Typically we should only do so in error cases such as
3879	* invalid/unexpected direction as described in the control transfer
3880	* flow of the programming guide.
3881	*/
3882	if (dep->number <= 1 && dwc->ep0state != EP0_DATA_PHASE)
3883	return;
3884
3885	if (interrupt && (dep->flags & DWC3_EP_DELAY_STOP))
3886	return;
3887
3888	if (!(dep->flags & DWC3_EP_TRANSFER_STARTED) ||
3889	(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
3890	return;
3891
3892	/*
3893	* If a Setup packet is received but yet to DMA out, the controller will
3894	* not process the End Transfer command of any endpoint. Polling of its
3895	* DEPCMD.CmdAct may block setting up TRB for Setup packet, causing a
3896	* timeout. Delay issuing the End Transfer command until the Setup TRB is
3897	* prepared.
3898	*/
3899	if (dwc->ep0state != EP0_SETUP_PHASE && !dwc->delayed_status) {
3900	dep->flags |= DWC3_EP_DELAY_STOP;
3901	return;
3902	}
3903
3904	/*
3905	* NOTICE: We are violating what the Databook says about the
3906	* EndTransfer command. Ideally we would _always_ wait for the
3907	* EndTransfer Command Completion IRQ, but that's causing too
3908	* much trouble synchronizing between us and gadget driver.
3909	*
3910	* We have discussed this with the IP Provider and it was
3911	* suggested to giveback all requests here.
3912	*
3913	* Note also that a similar handling was tested by Synopsys
3914	* (thanks a lot Paul) and nothing bad has come out of it.
3915	* In short, what we're doing is issuing EndTransfer with
3916	* CMDIOC bit set and delay kicking transfer until the
3917	* EndTransfer command had completed.
3918	*
3919	* As of IP version 3.10a of the DWC_usb3 IP, the controller
3920	* supports a mode to work around the above limitation. The
3921	* software can poll the CMDACT bit in the DEPCMD register
3922	* after issuing a EndTransfer command. This mode is enabled
3923	* by writing GUCTL2[14]. This polling is already done in the
3924	* dwc3_send_gadget_ep_cmd() function so if the mode is
3925	* enabled, the EndTransfer command will have completed upon
3926	* returning from this function.
3927	*
3928	* This mode is NOT available on the DWC_usb31 IP. In this
3929	* case, if the IOC bit is not set, then delay by 1ms
3930	* after issuing the EndTransfer command. This allows for the
3931	* controller to handle the command completely before DWC3
3932	* remove requests attempts to unmap USB request buffers.
3933	*/
3934
3935	__dwc3_stop_active_transfer(dep, force, interrupt);
3936	}
3937
3938	static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
3939	{
3940	u32 epnum;
3941
3942	for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
3943	struct dwc3_ep *dep;
3944	int ret;
3945
3946	dep = dwc->eps[epnum];
3947	if (!dep)
3948	continue;
3949
3950	if (!(dep->flags & DWC3_EP_STALL))
3951	continue;
3952
3953	dep->flags &= ~DWC3_EP_STALL;
3954
3955	ret = dwc3_send_clear_stall_ep_cmd(dep);
3956	WARN_ON_ONCE(ret);
3957	}
3958	}
3959
3960	static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
3961	{
3962	int reg;
3963
3964	dwc->suspended = false;
3965
3966	dwc3_gadget_set_link_state(dwc, state: DWC3_LINK_STATE_RX_DET);
3967
3968	reg = dwc3_readl(base: dwc->regs, DWC3_DCTL);
3969	reg &= ~DWC3_DCTL_INITU1ENA;
3970	reg &= ~DWC3_DCTL_INITU2ENA;
3971	dwc3_gadget_dctl_write_safe(dwc, value: reg);
3972
3973	dwc->connected = false;
3974
3975	dwc3_disconnect_gadget(dwc);
3976
3977	dwc->gadget->speed = USB_SPEED_UNKNOWN;
3978	dwc->setup_packet_pending = false;
3979	dwc->gadget->wakeup_armed = false;
3980	dwc3_gadget_enable_linksts_evts(dwc, set: false);
3981	usb_gadget_set_state(gadget: dwc->gadget, state: USB_STATE_NOTATTACHED);
3982
3983	dwc3_ep0_reset_state(dwc);
3984
3985	/*
3986	* Request PM idle to address condition where usage count is
3987	* already decremented to zero, but waiting for the disconnect
3988	* interrupt to set dwc->connected to FALSE.
3989	*/
3990	pm_request_idle(dev: dwc->dev);
3991	}
3992
3993	static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
3994	{
3995	u32 reg;
3996
3997	dwc->suspended = false;
3998
3999	/*
4000	* Ideally, dwc3_reset_gadget() would trigger the function
4001	* drivers to stop any active transfers through ep disable.
4002	* However, for functions which defer ep disable, such as mass
4003	* storage, we will need to rely on the call to stop active
4004	* transfers here, and avoid allowing of request queuing.
4005	*/
4006	dwc->connected = false;
4007
4008	/*
4009	* WORKAROUND: DWC3 revisions <1.88a have an issue which
4010	* would cause a missing Disconnect Event if there's a
4011	* pending Setup Packet in the FIFO.
4012	*
4013	* There's no suggested workaround on the official Bug
4014	* report, which states that "unless the driver/application
4015	* is doing any special handling of a disconnect event,
4016	* there is no functional issue".
4017	*
4018	* Unfortunately, it turns out that we _do_ some special
4019	* handling of a disconnect event, namely complete all
4020	* pending transfers, notify gadget driver of the
4021	* disconnection, and so on.
4022	*
4023	* Our suggested workaround is to follow the Disconnect
4024	* Event steps here, instead, based on a setup_packet_pending
4025	* flag. Such flag gets set whenever we have a SETUP_PENDING
4026	* status for EP0 TRBs and gets cleared on XferComplete for the
4027	* same endpoint.
4028	*
4029	* Refers to:
4030	*
4031	* STAR#9000466709: RTL: Device : Disconnect event not
4032	* generated if setup packet pending in FIFO
4033	*/
4034	if (DWC3_VER_IS_PRIOR(DWC3, 188A)) {
4035	if (dwc->setup_packet_pending)
4036	dwc3_gadget_disconnect_interrupt(dwc);
4037	}
4038
4039	dwc3_reset_gadget(dwc);
4040
4041	/*
4042	* From SNPS databook section 8.1.2, the EP0 should be in setup
4043	* phase. So ensure that EP0 is in setup phase by issuing a stall
4044	* and restart if EP0 is not in setup phase.
4045	*/
4046	dwc3_ep0_reset_state(dwc);
4047
4048	/*
4049	* In the Synopsis DesignWare Cores USB3 Databook Rev. 3.30a
4050	* Section 4.1.2 Table 4-2, it states that during a USB reset, the SW
4051	* needs to ensure that it sends "a DEPENDXFER command for any active
4052	* transfers."
4053	*/
4054	dwc3_stop_active_transfers(dwc);
4055	dwc->connected = true;
4056
4057	reg = dwc3_readl(base: dwc->regs, DWC3_DCTL);
4058	reg &= ~DWC3_DCTL_TSTCTRL_MASK;
4059	dwc3_gadget_dctl_write_safe(dwc, value: reg);
4060	dwc->test_mode = false;
4061	dwc->gadget->wakeup_armed = false;
4062	dwc3_gadget_enable_linksts_evts(dwc, set: false);
4063	dwc3_clear_stall_all_ep(dwc);
4064
4065	/* Reset device address to zero */
4066	reg = dwc3_readl(base: dwc->regs, DWC3_DCFG);
4067	reg &= ~(DWC3_DCFG_DEVADDR_MASK);
4068	dwc3_writel(base: dwc->regs, DWC3_DCFG, value: reg);
4069	}
4070
4071	static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
4072	{
4073	struct dwc3_ep *dep;
4074	int ret;
4075	u32 reg;
4076	u8 lanes = 1;
4077	u8 speed;
4078
4079	if (!dwc->softconnect)
4080	return;
4081
4082	reg = dwc3_readl(base: dwc->regs, DWC3_DSTS);
4083	speed = reg & DWC3_DSTS_CONNECTSPD;
4084	dwc->speed = speed;
4085
4086	if (DWC3_IP_IS(DWC32))
4087	lanes = DWC3_DSTS_CONNLANES(reg) + 1;
4088
4089	dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN;
4090
4091	/*
4092	* RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
4093	* each time on Connect Done.
4094	*
4095	* Currently we always use the reset value. If any platform
4096	* wants to set this to a different value, we need to add a
4097	* setting and update GCTL.RAMCLKSEL here.
4098	*/
4099
4100	switch (speed) {
4101	case DWC3_DSTS_SUPERSPEED_PLUS:
4102	dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
4103	dwc->gadget->ep0->maxpacket = 512;
4104	dwc->gadget->speed = USB_SPEED_SUPER_PLUS;
4105
4106	if (lanes > 1)
4107	dwc->gadget->ssp_rate = USB_SSP_GEN_2x2;
4108	else
4109	dwc->gadget->ssp_rate = USB_SSP_GEN_2x1;
4110	break;
4111	case DWC3_DSTS_SUPERSPEED:
4112	/*
4113	* WORKAROUND: DWC3 revisions <1.90a have an issue which
4114	* would cause a missing USB3 Reset event.
4115	*
4116	* In such situations, we should force a USB3 Reset
4117	* event by calling our dwc3_gadget_reset_interrupt()
4118	* routine.
4119	*
4120	* Refers to:
4121	*
4122	* STAR#9000483510: RTL: SS : USB3 reset event may
4123	* not be generated always when the link enters poll
4124	*/
4125	if (DWC3_VER_IS_PRIOR(DWC3, 190A))
4126	dwc3_gadget_reset_interrupt(dwc);
4127
4128	dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
4129	dwc->gadget->ep0->maxpacket = 512;
4130	dwc->gadget->speed = USB_SPEED_SUPER;
4131
4132	if (lanes > 1) {
4133	dwc->gadget->speed = USB_SPEED_SUPER_PLUS;
4134	dwc->gadget->ssp_rate = USB_SSP_GEN_1x2;
4135	}
4136	break;
4137	case DWC3_DSTS_HIGHSPEED:
4138	dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
4139	dwc->gadget->ep0->maxpacket = 64;
4140	dwc->gadget->speed = USB_SPEED_HIGH;
4141	break;
4142	case DWC3_DSTS_FULLSPEED:
4143	dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
4144	dwc->gadget->ep0->maxpacket = 64;
4145	dwc->gadget->speed = USB_SPEED_FULL;
4146	break;
4147	}
4148
4149	dwc->eps[1]->endpoint.maxpacket = dwc->gadget->ep0->maxpacket;
4150
4151	/* Enable USB2 LPM Capability */
4152
4153	if (!DWC3_VER_IS_WITHIN(DWC3, ANY, 194A) &&
4154	!dwc->usb2_gadget_lpm_disable &&
4155	(speed != DWC3_DSTS_SUPERSPEED) &&
4156	(speed != DWC3_DSTS_SUPERSPEED_PLUS)) {
4157	reg = dwc3_readl(base: dwc->regs, DWC3_DCFG);
4158	reg |= DWC3_DCFG_LPM_CAP;
4159	dwc3_writel(base: dwc->regs, DWC3_DCFG, value: reg);
4160
4161	reg = dwc3_readl(base: dwc->regs, DWC3_DCTL);
4162	reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN);
4163
4164	reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold |
4165	(dwc->is_utmi_l1_suspend << 4));
4166
4167	/*
4168	* When dwc3 revisions >= 2.40a, LPM Erratum is enabled and
4169	* DCFG.LPMCap is set, core responses with an ACK and the
4170	* BESL value in the LPM token is less than or equal to LPM
4171	* NYET threshold.
4172	*/
4173	WARN_ONCE(DWC3_VER_IS_PRIOR(DWC3, 240A) && dwc->has_lpm_erratum,
4174	"LPM Erratum not available on dwc3 revisions < 2.40a\n");
4175
4176	if (dwc->has_lpm_erratum && !DWC3_VER_IS_PRIOR(DWC3, 240A))
4177	reg |= DWC3_DCTL_NYET_THRES(dwc->lpm_nyet_threshold);
4178
4179	dwc3_gadget_dctl_write_safe(dwc, value: reg);
4180	} else {
4181	if (dwc->usb2_gadget_lpm_disable) {
4182	reg = dwc3_readl(base: dwc->regs, DWC3_DCFG);
4183	reg &= ~DWC3_DCFG_LPM_CAP;
4184	dwc3_writel(base: dwc->regs, DWC3_DCFG, value: reg);
4185	}
4186
4187	reg = dwc3_readl(base: dwc->regs, DWC3_DCTL);
4188	reg &= ~DWC3_DCTL_HIRD_THRES_MASK;
4189	dwc3_gadget_dctl_write_safe(dwc, value: reg);
4190	}
4191
4192	dep = dwc->eps[0];
4193	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
4194	if (ret) {
4195	dev_err(dwc->dev, "failed to enable %s\n", dep->name);
4196	return;
4197	}
4198
4199	dep = dwc->eps[1];
4200	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
4201	if (ret) {
4202	dev_err(dwc->dev, "failed to enable %s\n", dep->name);
4203	return;
4204	}
4205
4206	/*
4207	* Configure PHY via GUSB3PIPECTLn if required.
4208	*
4209	* Update GTXFIFOSIZn
4210	*
4211	* In both cases reset values should be sufficient.
4212	*/
4213	}
4214
4215	static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc, unsigned int evtinfo)
4216	{
4217	dwc->suspended = false;
4218
4219	/*
4220	* TODO take core out of low power mode when that's
4221	* implemented.
4222	*/
4223
4224	if (dwc->async_callbacks && dwc->gadget_driver->resume) {
4225	spin_unlock(lock: &dwc->lock);
4226	dwc->gadget_driver->resume(dwc->gadget);
4227	spin_lock(lock: &dwc->lock);
4228	}
4229
4230	dwc->link_state = evtinfo & DWC3_LINK_STATE_MASK;
4231	}
4232
4233	static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
4234	unsigned int evtinfo)
4235	{
4236	enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
4237	unsigned int pwropt;
4238
4239	/*
4240	* WORKAROUND: DWC3 < 2.50a have an issue when configured without
4241	* Hibernation mode enabled which would show up when device detects
4242	* host-initiated U3 exit.
4243	*
4244	* In that case, device will generate a Link State Change Interrupt
4245	* from U3 to RESUME which is only necessary if Hibernation is
4246	* configured in.
4247	*
4248	* There are no functional changes due to such spurious event and we
4249	* just need to ignore it.
4250	*
4251	* Refers to:
4252	*
4253	* STAR#9000570034 RTL: SS Resume event generated in non-Hibernation
4254	* operational mode
4255	*/
4256	pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
4257	if (DWC3_VER_IS_PRIOR(DWC3, 250A) &&
4258	(pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
4259	if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
4260	(next == DWC3_LINK_STATE_RESUME)) {
4261	return;
4262	}
4263	}
4264
4265	/*
4266	* WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending
4267	* on the link partner, the USB session might do multiple entry/exit
4268	* of low power states before a transfer takes place.
4269	*
4270	* Due to this problem, we might experience lower throughput. The
4271	* suggested workaround is to disable DCTL[12:9] bits if we're
4272	* transitioning from U1/U2 to U0 and enable those bits again
4273	* after a transfer completes and there are no pending transfers
4274	* on any of the enabled endpoints.
4275	*
4276	* This is the first half of that workaround.
4277	*
4278	* Refers to:
4279	*
4280	* STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us
4281	* core send LGO_Ux entering U0
4282	*/
4283	if (DWC3_VER_IS_PRIOR(DWC3, 183A)) {
4284	if (next == DWC3_LINK_STATE_U0) {
4285	u32 u1u2;
4286	u32 reg;
4287
4288	switch (dwc->link_state) {
4289	case DWC3_LINK_STATE_U1:
4290	case DWC3_LINK_STATE_U2:
4291	reg = dwc3_readl(base: dwc->regs, DWC3_DCTL);
4292	u1u2 = reg & (DWC3_DCTL_INITU2ENA
4293	| DWC3_DCTL_ACCEPTU2ENA
4294	| DWC3_DCTL_INITU1ENA
4295	| DWC3_DCTL_ACCEPTU1ENA);
4296
4297	if (!dwc->u1u2)
4298	dwc->u1u2 = reg & u1u2;
4299
4300	reg &= ~u1u2;
4301
4302	dwc3_gadget_dctl_write_safe(dwc, value: reg);
4303	break;
4304	default:
4305	/* do nothing */
4306	break;
4307	}
4308	}
4309	}
4310
4311	switch (next) {
4312	case DWC3_LINK_STATE_U0:
4313	if (dwc->gadget->wakeup_armed) {
4314	dwc3_gadget_enable_linksts_evts(dwc, set: false);
4315	dwc3_resume_gadget(dwc);
4316	dwc->suspended = false;
4317	}
4318	break;
4319	case DWC3_LINK_STATE_U1:
4320	if (dwc->speed == USB_SPEED_SUPER)
4321	dwc3_suspend_gadget(dwc);
4322	break;
4323	case DWC3_LINK_STATE_U2:
4324	case DWC3_LINK_STATE_U3:
4325	dwc3_suspend_gadget(dwc);
4326	break;
4327	case DWC3_LINK_STATE_RESUME:
4328	dwc3_resume_gadget(dwc);
4329	break;
4330	default:
4331	/* do nothing */
4332	break;
4333	}
4334
4335	dwc->link_state = next;
4336	}
4337
4338	static void dwc3_gadget_suspend_interrupt(struct dwc3 *dwc,
4339	unsigned int evtinfo)
4340	{
4341	enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
4342
4343	if (!dwc->suspended && next == DWC3_LINK_STATE_U3) {
4344	dwc->suspended = true;
4345	dwc3_suspend_gadget(dwc);
4346	}
4347
4348	dwc->link_state = next;
4349	}
4350
4351	static void dwc3_gadget_interrupt(struct dwc3 *dwc,
4352	const struct dwc3_event_devt *event)
4353	{
4354	switch (event->type) {
4355	case DWC3_DEVICE_EVENT_DISCONNECT:
4356	dwc3_gadget_disconnect_interrupt(dwc);
4357	break;
4358	case DWC3_DEVICE_EVENT_RESET:
4359	dwc3_gadget_reset_interrupt(dwc);
4360	break;
4361	case DWC3_DEVICE_EVENT_CONNECT_DONE:
4362	dwc3_gadget_conndone_interrupt(dwc);
4363	break;
4364	case DWC3_DEVICE_EVENT_WAKEUP:
4365	dwc3_gadget_wakeup_interrupt(dwc, evtinfo: event->event_info);
4366	break;
4367	case DWC3_DEVICE_EVENT_HIBER_REQ:
4368	dev_WARN_ONCE(dwc->dev, true, "unexpected hibernation event\n");
4369	break;
4370	case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
4371	dwc3_gadget_linksts_change_interrupt(dwc, evtinfo: event->event_info);
4372	break;
4373	case DWC3_DEVICE_EVENT_SUSPEND:
4374	/* It changed to be suspend event for version 2.30a and above */
4375	if (!DWC3_VER_IS_PRIOR(DWC3, 230A))
4376	dwc3_gadget_suspend_interrupt(dwc, evtinfo: event->event_info);
4377	break;
4378	case DWC3_DEVICE_EVENT_SOF:
4379	case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
4380	case DWC3_DEVICE_EVENT_CMD_CMPL:
4381	case DWC3_DEVICE_EVENT_OVERFLOW:
4382	break;
4383	default:
4384	dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
4385	}
4386	}
4387
4388	static void dwc3_process_event_entry(struct dwc3 *dwc,
4389	const union dwc3_event *event)
4390	{
4391	trace_dwc3_event(event: event->raw, dwc);
4392
4393	if (!event->type.is_devspec)
4394	dwc3_endpoint_interrupt(dwc, event: &event->depevt);
4395	else if (event->type.type == DWC3_EVENT_TYPE_DEV)
4396	dwc3_gadget_interrupt(dwc, event: &event->devt);
4397	else
4398	dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
4399	}
4400
4401	static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt)
4402	{
4403	struct dwc3 *dwc = evt->dwc;
4404	irqreturn_t ret = IRQ_NONE;
4405	int left;
4406
4407	left = evt->count;
4408
4409	if (!(evt->flags & DWC3_EVENT_PENDING))
4410	return IRQ_NONE;
4411
4412	while (left > 0) {
4413	union dwc3_event event;
4414
4415	event.raw = *(u32 *) (evt->cache + evt->lpos);
4416
4417	dwc3_process_event_entry(dwc, event: &event);
4418
4419	/*
4420	* FIXME we wrap around correctly to the next entry as
4421	* almost all entries are 4 bytes in size. There is one
4422	* entry which has 12 bytes which is a regular entry
4423	* followed by 8 bytes data. ATM I don't know how
4424	* things are organized if we get next to the a
4425	* boundary so I worry about that once we try to handle
4426	* that.
4427	*/
4428	evt->lpos = (evt->lpos + 4) % evt->length;
4429	left -= 4;
4430	}
4431
4432	evt->count = 0;
4433	ret = IRQ_HANDLED;
4434
4435	/* Unmask interrupt */
4436	dwc3_writel(base: dwc->regs, DWC3_GEVNTSIZ(0),
4437	DWC3_GEVNTSIZ_SIZE(evt->length));
4438
4439	if (dwc->imod_interval) {
4440	dwc3_writel(base: dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
4441	dwc3_writel(base: dwc->regs, DWC3_DEV_IMOD(0), value: dwc->imod_interval);
4442	}
4443
4444	/* Keep the clearing of DWC3_EVENT_PENDING at the end */
4445	evt->flags &= ~DWC3_EVENT_PENDING;
4446
4447	return ret;
4448	}
4449
4450	static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt)
4451	{
4452	struct dwc3_event_buffer *evt = _evt;
4453	struct dwc3 *dwc = evt->dwc;
4454	unsigned long flags;
4455	irqreturn_t ret = IRQ_NONE;
4456
4457	local_bh_disable();
4458	spin_lock_irqsave(&dwc->lock, flags);
4459	ret = dwc3_process_event_buf(evt);
4460	spin_unlock_irqrestore(lock: &dwc->lock, flags);
4461	local_bh_enable();
4462
4463	return ret;
4464	}
4465
4466	static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt)
4467	{
4468	struct dwc3 *dwc = evt->dwc;
4469	u32 amount;
4470	u32 count;
4471
4472	if (pm_runtime_suspended(dev: dwc->dev)) {
4473	dwc->pending_events = true;
4474	/*
4475	* Trigger runtime resume. The get() function will be balanced
4476	* after processing the pending events in dwc3_process_pending
4477	* events().
4478	*/
4479	pm_runtime_get(dev: dwc->dev);
4480	disable_irq_nosync(irq: dwc->irq_gadget);
4481	return IRQ_HANDLED;
4482	}
4483
4484	/*
4485	* With PCIe legacy interrupt, test shows that top-half irq handler can
4486	* be called again after HW interrupt deassertion. Check if bottom-half
4487	* irq event handler completes before caching new event to prevent
4488	* losing events.
4489	*/
4490	if (evt->flags & DWC3_EVENT_PENDING)
4491	return IRQ_HANDLED;
4492
4493	count = dwc3_readl(base: dwc->regs, DWC3_GEVNTCOUNT(0));
4494	count &= DWC3_GEVNTCOUNT_MASK;
4495	if (!count)
4496	return IRQ_NONE;
4497
4498	evt->count = count;
4499	evt->flags |= DWC3_EVENT_PENDING;
4500
4501	/* Mask interrupt */
4502	dwc3_writel(base: dwc->regs, DWC3_GEVNTSIZ(0),
4503	DWC3_GEVNTSIZ_INTMASK | DWC3_GEVNTSIZ_SIZE(evt->length));
4504
4505	amount = min(count, evt->length - evt->lpos);
4506	memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount);
4507
4508	if (amount < count)
4509	memcpy(evt->cache, evt->buf, count - amount);
4510
4511	dwc3_writel(base: dwc->regs, DWC3_GEVNTCOUNT(0), value: count);
4512
4513	return IRQ_WAKE_THREAD;
4514	}
4515
4516	static irqreturn_t dwc3_interrupt(int irq, void *_evt)
4517	{
4518	struct dwc3_event_buffer *evt = _evt;
4519
4520	return dwc3_check_event_buf(evt);
4521	}
4522
4523	static int dwc3_gadget_get_irq(struct dwc3 *dwc)
4524	{
4525	struct platform_device *dwc3_pdev = to_platform_device(dwc->dev);
4526	int irq;
4527
4528	irq = platform_get_irq_byname_optional(dev: dwc3_pdev, name: "peripheral");
4529	if (irq > 0)
4530	goto out;
4531
4532	if (irq == -EPROBE_DEFER)
4533	goto out;
4534
4535	irq = platform_get_irq_byname_optional(dev: dwc3_pdev, name: "dwc_usb3");
4536	if (irq > 0)
4537	goto out;
4538
4539	if (irq == -EPROBE_DEFER)
4540	goto out;
4541
4542	irq = platform_get_irq(dwc3_pdev, 0);
4543
4544	out:
4545	return irq;
4546	}
4547
4548	static void dwc_gadget_release(struct device *dev)
4549	{
4550	struct usb_gadget *gadget = container_of(dev, struct usb_gadget, dev);
4551
4552	kfree(objp: gadget);
4553	}
4554
4555	/**
4556	* dwc3_gadget_init - initializes gadget related registers
4557	* @dwc: pointer to our controller context structure
4558	*
4559	* Returns 0 on success otherwise negative errno.
4560	*/
4561	int dwc3_gadget_init(struct dwc3 *dwc)
4562	{
4563	int ret;
4564	int irq;
4565	struct device *dev;
4566
4567	irq = dwc3_gadget_get_irq(dwc);
4568	if (irq < 0) {
4569	ret = irq;
4570	goto err0;
4571	}
4572
4573	dwc->irq_gadget = irq;
4574
4575	dwc->ep0_trb = dma_alloc_coherent(dev: dwc->sysdev,
4576	size: sizeof(*dwc->ep0_trb) * 2,
4577	dma_handle: &dwc->ep0_trb_addr, GFP_KERNEL);
4578	if (!dwc->ep0_trb) {
4579	dev_err(dwc->dev, "failed to allocate ep0 trb\n");
4580	ret = -ENOMEM;
4581	goto err0;
4582	}
4583
4584	dwc->setup_buf = kzalloc(DWC3_EP0_SETUP_SIZE, GFP_KERNEL);
4585	if (!dwc->setup_buf) {
4586	ret = -ENOMEM;
4587	goto err1;
4588	}
4589
4590	dwc->bounce = dma_alloc_coherent(dev: dwc->sysdev, DWC3_BOUNCE_SIZE,
4591	dma_handle: &dwc->bounce_addr, GFP_KERNEL);
4592	if (!dwc->bounce) {
4593	ret = -ENOMEM;
4594	goto err2;
4595	}
4596
4597	init_completion(x: &dwc->ep0_in_setup);
4598	dwc->gadget = kzalloc(size: sizeof(struct usb_gadget), GFP_KERNEL);
4599	if (!dwc->gadget) {
4600	ret = -ENOMEM;
4601	goto err3;
4602	}
4603
4604
4605	usb_initialize_gadget(parent: dwc->dev, gadget: dwc->gadget, release: dwc_gadget_release);
4606	dev = &dwc->gadget->dev;
4607	dev->platform_data = dwc;
4608	dwc->gadget->ops = &dwc3_gadget_ops;
4609	dwc->gadget->speed = USB_SPEED_UNKNOWN;
4610	dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN;
4611	dwc->gadget->sg_supported = true;
4612	dwc->gadget->name = "dwc3-gadget";
4613	dwc->gadget->lpm_capable = !dwc->usb2_gadget_lpm_disable;
4614	dwc->gadget->wakeup_capable = true;
4615
4616	/*
4617	* FIXME We might be setting max_speed to <SUPER, however versions
4618	* <2.20a of dwc3 have an issue with metastability (documented
4619	* elsewhere in this driver) which tells us we can't set max speed to
4620	* anything lower than SUPER.
4621	*
4622	* Because gadget.max_speed is only used by composite.c and function
4623	* drivers (i.e. it won't go into dwc3's registers) we are allowing this
4624	* to happen so we avoid sending SuperSpeed Capability descriptor
4625	* together with our BOS descriptor as that could confuse host into
4626	* thinking we can handle super speed.
4627	*
4628	* Note that, in fact, we won't even support GetBOS requests when speed
4629	* is less than super speed because we don't have means, yet, to tell
4630	* composite.c that we are USB 2.0 + LPM ECN.
4631	*/
4632	if (DWC3_VER_IS_PRIOR(DWC3, 220A) &&
4633	!dwc->dis_metastability_quirk)
4634	dev_info(dwc->dev, "changing max_speed on rev %08x\n",
4635	dwc->revision);
4636
4637	dwc->gadget->max_speed = dwc->maximum_speed;
4638	dwc->gadget->max_ssp_rate = dwc->max_ssp_rate;
4639
4640	/*
4641	* REVISIT: Here we should clear all pending IRQs to be
4642	* sure we're starting from a well known location.
4643	*/
4644
4645	ret = dwc3_gadget_init_endpoints(dwc, total: dwc->num_eps);
4646	if (ret)
4647	goto err4;
4648
4649	ret = usb_add_gadget(gadget: dwc->gadget);
4650	if (ret) {
4651	dev_err(dwc->dev, "failed to add gadget\n");
4652	goto err5;
4653	}
4654
4655	if (DWC3_IP_IS(DWC32) && dwc->maximum_speed == USB_SPEED_SUPER_PLUS)
4656	dwc3_gadget_set_ssp_rate(g: dwc->gadget, rate: dwc->max_ssp_rate);
4657	else
4658	dwc3_gadget_set_speed(g: dwc->gadget, speed: dwc->maximum_speed);
4659
4660	/* No system wakeup if no gadget driver bound */
4661	if (dwc->sys_wakeup)
4662	device_wakeup_disable(dev: dwc->sysdev);
4663
4664	return 0;
4665
4666	err5:
4667	dwc3_gadget_free_endpoints(dwc);
4668	err4:
4669	usb_put_gadget(gadget: dwc->gadget);
4670	dwc->gadget = NULL;
4671	err3:
4672	dma_free_coherent(dev: dwc->sysdev, DWC3_BOUNCE_SIZE, cpu_addr: dwc->bounce,
4673	dma_handle: dwc->bounce_addr);
4674
4675	err2:
4676	kfree(objp: dwc->setup_buf);
4677
4678	err1:
4679	dma_free_coherent(dev: dwc->sysdev, size: sizeof(*dwc->ep0_trb) * 2,
4680	cpu_addr: dwc->ep0_trb, dma_handle: dwc->ep0_trb_addr);
4681
4682	err0:
4683	return ret;
4684	}
4685
4686	/* -------------------------------------------------------------------------- */
4687
4688	void dwc3_gadget_exit(struct dwc3 *dwc)
4689	{
4690	if (!dwc->gadget)
4691	return;
4692
4693	usb_del_gadget(gadget: dwc->gadget);
4694	dwc3_gadget_free_endpoints(dwc);
4695	usb_put_gadget(gadget: dwc->gadget);
4696	dma_free_coherent(dev: dwc->sysdev, DWC3_BOUNCE_SIZE, cpu_addr: dwc->bounce,
4697	dma_handle: dwc->bounce_addr);
4698	kfree(objp: dwc->setup_buf);
4699	dma_free_coherent(dev: dwc->sysdev, size: sizeof(*dwc->ep0_trb) * 2,
4700	cpu_addr: dwc->ep0_trb, dma_handle: dwc->ep0_trb_addr);
4701	}
4702
4703	int dwc3_gadget_suspend(struct dwc3 *dwc)
4704	{
4705	unsigned long flags;
4706	int ret;
4707
4708	ret = dwc3_gadget_soft_disconnect(dwc);
4709	if (ret)
4710	goto err;
4711
4712	spin_lock_irqsave(&dwc->lock, flags);
4713	if (dwc->gadget_driver)
4714	dwc3_disconnect_gadget(dwc);
4715	spin_unlock_irqrestore(lock: &dwc->lock, flags);
4716
4717	return 0;
4718
4719	err:
4720	/*
4721	* Attempt to reset the controller's state. Likely no
4722	* communication can be established until the host
4723	* performs a port reset.
4724	*/
4725	if (dwc->softconnect)
4726	dwc3_gadget_soft_connect(dwc);
4727
4728	return ret;
4729	}
4730
4731	int dwc3_gadget_resume(struct dwc3 *dwc)
4732	{
4733	if (!dwc->gadget_driver || !dwc->softconnect)
4734	return 0;
4735
4736	return dwc3_gadget_soft_connect(dwc);
4737	}
4738
4739	void dwc3_gadget_process_pending_events(struct dwc3 *dwc)
4740	{
4741	if (dwc->pending_events) {
4742	dwc3_interrupt(irq: dwc->irq_gadget, evt: dwc->ev_buf);
4743	dwc3_thread_interrupt(irq: dwc->irq_gadget, evt: dwc->ev_buf);
4744	pm_runtime_put(dev: dwc->dev);
4745	dwc->pending_events = false;
4746	enable_irq(irq: dwc->irq_gadget);
4747	}
4748	}
4749