1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
4	*
5	* Copyright (C) 2010-2011 Texas Instruments Incorporated - https://www.ti.com
6	*
7	* Authors: Felipe Balbi <balbi@ti.com>,
8	* Sebastian Andrzej Siewior <bigeasy@linutronix.de>
9	*/
10
11	#include <linux/kernel.h>
12	#include <linux/delay.h>
13	#include <linux/slab.h>
14	#include <linux/spinlock.h>
15	#include <linux/platform_device.h>
16	#include <linux/pm_runtime.h>
17	#include <linux/interrupt.h>
18	#include <linux/io.h>
19	#include <linux/list.h>
20	#include <linux/dma-mapping.h>
21
22	#include <linux/usb/ch9.h>
23	#include <linux/usb/gadget.h>
24
25	#include "debug.h"
26	#include "core.h"
27	#include "gadget.h"
28	#include "io.h"
29
30	#define DWC3_ALIGN_FRAME(d, n) (((d)->frame_number + ((d)->interval * (n))) \
31	& ~((d)->interval - 1))
32
33	/**
34	* dwc3_gadget_set_test_mode - enables usb2 test modes
35	* @dwc: pointer to our context structure
36	* @mode: the mode to set (J, K SE0 NAK, Force Enable)
37	*
38	* Caller should take care of locking. This function will return 0 on
39	* success or -EINVAL if wrong Test Selector is passed.
40	*/
41	int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode)
42	{
43	u32 reg;
44
45	reg = dwc3_readl(base: dwc->regs, DWC3_DCTL);
46	reg &= ~DWC3_DCTL_TSTCTRL_MASK;
47
48	switch (mode) {
49	case USB_TEST_J:
50	case USB_TEST_K:
51	case USB_TEST_SE0_NAK:
52	case USB_TEST_PACKET:
53	case USB_TEST_FORCE_ENABLE:
54	reg |= mode << 1;
55	break;
56	default:
57	return -EINVAL;
58	}
59
60	dwc3_gadget_dctl_write_safe(dwc, value: reg);
61
62	return 0;
63	}
64
65	/**
66	* dwc3_gadget_get_link_state - gets current state of usb link
67	* @dwc: pointer to our context structure
68	*
69	* Caller should take care of locking. This function will
70	* return the link state on success (>= 0) or -ETIMEDOUT.
71	*/
72	int dwc3_gadget_get_link_state(struct dwc3 *dwc)
73	{
74	u32 reg;
75
76	reg = dwc3_readl(base: dwc->regs, DWC3_DSTS);
77
78	return DWC3_DSTS_USBLNKST(reg);
79	}
80
81	/**
82	* dwc3_gadget_set_link_state - sets usb link to a particular state
83	* @dwc: pointer to our context structure
84	* @state: the state to put link into
85	*
86	* Caller should take care of locking. This function will
87	* return 0 on success or -ETIMEDOUT.
88	*/
89	int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
90	{
91	int retries = 10000;
92	u32 reg;
93
94	/*
95	* Wait until device controller is ready. Only applies to 1.94a and
96	* later RTL.
97	*/
98	if (!DWC3_VER_IS_PRIOR(DWC3, 194A)) {
99	while (--retries) {
100	reg = dwc3_readl(base: dwc->regs, DWC3_DSTS);
101	if (reg & DWC3_DSTS_DCNRD)
102	udelay(usec: 5);
103	else
104	break;
105	}
106
107	if (retries <= 0)
108	return -ETIMEDOUT;
109	}
110
111	reg = dwc3_readl(base: dwc->regs, DWC3_DCTL);
112	reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
113
114	/* set no action before sending new link state change */
115	dwc3_writel(base: dwc->regs, DWC3_DCTL, value: reg);
116
117	/* set requested state */
118	reg |= DWC3_DCTL_ULSTCHNGREQ(state);
119	dwc3_writel(base: dwc->regs, DWC3_DCTL, value: reg);
120
121	/*
122	* The following code is racy when called from dwc3_gadget_wakeup,
123	* and is not needed, at least on newer versions
124	*/
125	if (!DWC3_VER_IS_PRIOR(DWC3, 194A))
126	return 0;
127
128	/* wait for a change in DSTS */
129	retries = 10000;
130	while (--retries) {
131	reg = dwc3_readl(base: dwc->regs, DWC3_DSTS);
132
133	if (DWC3_DSTS_USBLNKST(reg) == state)
134	return 0;
135
136	udelay(usec: 5);
137	}
138
139	return -ETIMEDOUT;
140	}
141
142	static void dwc3_ep0_reset_state(struct dwc3 *dwc)
143	{
144	unsigned int dir;
145
146	if (dwc->ep0state != EP0_SETUP_PHASE) {
147	dir = !!dwc->ep0_expect_in;
148	if (dwc->ep0state == EP0_DATA_PHASE)
149	dwc3_ep0_end_control_data(dwc, dep: dwc->eps[dir]);
150	else
151	dwc3_ep0_end_control_data(dwc, dep: dwc->eps[!dir]);
152
153	dwc->eps[0]->trb_enqueue = 0;
154	dwc->eps[1]->trb_enqueue = 0;
155
156	dwc3_ep0_stall_and_restart(dwc);
157	}
158	}
159
160	/**
161	* dwc3_ep_inc_trb - increment a trb index.
162	* @index: Pointer to the TRB index to increment.
163	*
164	* The index should never point to the link TRB. After incrementing,
165	* if it is point to the link TRB, wrap around to the beginning. The
166	* link TRB is always at the last TRB entry.
167	*/
168	static void dwc3_ep_inc_trb(u8 *index)
169	{
170	(*index)++;
171	if (*index == (DWC3_TRB_NUM - 1))
172	*index = 0;
173	}
174
175	/**
176	* dwc3_ep_inc_enq - increment endpoint's enqueue pointer
177	* @dep: The endpoint whose enqueue pointer we're incrementing
178	*/
179	static void dwc3_ep_inc_enq(struct dwc3_ep *dep)
180	{
181	dwc3_ep_inc_trb(index: &dep->trb_enqueue);
182	}
183
184	/**
185	* dwc3_ep_inc_deq - increment endpoint's dequeue pointer
186	* @dep: The endpoint whose enqueue pointer we're incrementing
187	*/
188	static void dwc3_ep_inc_deq(struct dwc3_ep *dep)
189	{
190	dwc3_ep_inc_trb(index: &dep->trb_dequeue);
191	}
192
193	static void dwc3_gadget_del_and_unmap_request(struct dwc3_ep *dep,
194	struct dwc3_request *req, int status)
195	{
196	struct dwc3 *dwc = dep->dwc;
197
198	list_del(entry: &req->list);
199	req->remaining = 0;
200	req->num_trbs = 0;
201
202	if (req->request.status == -EINPROGRESS)
203	req->request.status = status;
204
205	if (req->trb)
206	usb_gadget_unmap_request_by_dev(dev: dwc->sysdev,
207	req: &req->request, is_in: req->direction);
208
209	req->trb = NULL;
210	trace_dwc3_gadget_giveback(req);
211
212	if (dep->number > 1)
213	pm_runtime_put(dev: dwc->dev);
214	}
215
216	/**
217	* dwc3_gadget_giveback - call struct usb_request's ->complete callback
218	* @dep: The endpoint to whom the request belongs to
219	* @req: The request we're giving back
220	* @status: completion code for the request
221	*
222	* Must be called with controller's lock held and interrupts disabled. This
223	* function will unmap @req and call its ->complete() callback to notify upper
224	* layers that it has completed.
225	*/
226	void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
227	int status)
228	{
229	struct dwc3 *dwc = dep->dwc;
230
231	/*
232	* The request might have been processed and completed while the
233	* spinlock was released. Skip processing if already completed.
234	*/
235	if (req->status == DWC3_REQUEST_STATUS_COMPLETED)
236	return;
237
238	dwc3_gadget_del_and_unmap_request(dep, req, status);
239	req->status = DWC3_REQUEST_STATUS_COMPLETED;
240
241	spin_unlock(lock: &dwc->lock);
242	usb_gadget_giveback_request(ep: &dep->endpoint, req: &req->request);
243	spin_lock(lock: &dwc->lock);
244	}
245
246	/**
247	* dwc3_send_gadget_generic_command - issue a generic command for the controller
248	* @dwc: pointer to the controller context
249	* @cmd: the command to be issued
250	* @param: command parameter
251	*
252	* Caller should take care of locking. Issue @cmd with a given @param to @dwc
253	* and wait for its completion.
254	*/
255	int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned int cmd,
256	u32 param)
257	{
258	u32 timeout = 500;
259	int status = 0;
260	int ret = 0;
261	u32 reg;
262
263	dwc3_writel(base: dwc->regs, DWC3_DGCMDPAR, value: param);
264	dwc3_writel(base: dwc->regs, DWC3_DGCMD, value: cmd | DWC3_DGCMD_CMDACT);
265
266	do {
267	reg = dwc3_readl(base: dwc->regs, DWC3_DGCMD);
268	if (!(reg & DWC3_DGCMD_CMDACT)) {
269	status = DWC3_DGCMD_STATUS(reg);
270	if (status)
271	ret = -EINVAL;
272	break;
273	}
274	} while (--timeout);
275
276	if (!timeout) {
277	ret = -ETIMEDOUT;
278	status = -ETIMEDOUT;
279	}
280
281	trace_dwc3_gadget_generic_cmd(cmd, param, status);
282
283	return ret;
284	}
285
286	/**
287	* dwc3_send_gadget_ep_cmd - issue an endpoint command
288	* @dep: the endpoint to which the command is going to be issued
289	* @cmd: the command to be issued
290	* @params: parameters to the command
291	*
292	* Caller should handle locking. This function will issue @cmd with given
293	* @params to @dep and wait for its completion.
294	*
295	* According to the programming guide, if the link state is in L1/L2/U3,
296	* then sending the Start Transfer command may not complete. The
297	* programming guide suggested to bring the link state back to ON/U0 by
298	* performing remote wakeup prior to sending the command. However, don't
299	* initiate remote wakeup when the user/function does not send wakeup
300	* request via wakeup ops. Send the command when it's allowed.
301	*
302	* Notes:
303	* For L1 link state, issuing a command requires the clearing of
304	* GUSB2PHYCFG.SUSPENDUSB2, which turns on the signal required to complete
305	* the given command (usually within 50us). This should happen within the
306	* command timeout set by driver. No additional step is needed.
307	*
308	* For L2 or U3 link state, the gadget is in USB suspend. Care should be
309	* taken when sending Start Transfer command to ensure that it's done after
310	* USB resume.
311	*/
312	int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned int cmd,
313	struct dwc3_gadget_ep_cmd_params *params)
314	{
315	const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
316	struct dwc3 *dwc = dep->dwc;
317	u32 timeout = 5000;
318	u32 saved_config = 0;
319	u32 reg;
320
321	int cmd_status = 0;
322	int ret = -EINVAL;
323
324	/*
325	* When operating in USB 2.0 speeds (HS/FS), if GUSB2PHYCFG.ENBLSLPM or
326	* GUSB2PHYCFG.SUSPHY is set, it must be cleared before issuing an
327	* endpoint command.
328	*
329	* Save and clear both GUSB2PHYCFG.ENBLSLPM and GUSB2PHYCFG.SUSPHY
330	* settings. Restore them after the command is completed.
331	*
332	* DWC_usb3 3.30a and DWC_usb31 1.90a programming guide section 3.2.2
333	*/
334	if (dwc->gadget->speed <= USB_SPEED_HIGH ||
335	DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_ENDTRANSFER) {
336	reg = dwc3_readl(base: dwc->regs, DWC3_GUSB2PHYCFG(0));
337	if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) {
338	saved_config |= DWC3_GUSB2PHYCFG_SUSPHY;
339	reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
340	}
341
342	if (reg & DWC3_GUSB2PHYCFG_ENBLSLPM) {
343	saved_config |= DWC3_GUSB2PHYCFG_ENBLSLPM;
344	reg &= ~DWC3_GUSB2PHYCFG_ENBLSLPM;
345	}
346
347	if (saved_config)
348	dwc3_writel(base: dwc->regs, DWC3_GUSB2PHYCFG(0), value: reg);
349	}
350
351	/*
352	* For some commands such as Update Transfer command, DEPCMDPARn
353	* registers are reserved. Since the driver often sends Update Transfer
354	* command, don't write to DEPCMDPARn to avoid register write delays and
355	* improve performance.
356	*/
357	if (DWC3_DEPCMD_CMD(cmd) != DWC3_DEPCMD_UPDATETRANSFER) {
358	dwc3_writel(base: dep->regs, DWC3_DEPCMDPAR0, value: params->param0);
359	dwc3_writel(base: dep->regs, DWC3_DEPCMDPAR1, value: params->param1);
360	dwc3_writel(base: dep->regs, DWC3_DEPCMDPAR2, value: params->param2);
361	}
362
363	/*
364	* Synopsys Databook 2.60a states in section 6.3.2.5.6 of that if we're
365	* not relying on XferNotReady, we can make use of a special "No
366	* Response Update Transfer" command where we should clear both CmdAct
367	* and CmdIOC bits.
368	*
369	* With this, we don't need to wait for command completion and can
370	* straight away issue further commands to the endpoint.
371	*
372	* NOTICE: We're making an assumption that control endpoints will never
373	* make use of Update Transfer command. This is a safe assumption
374	* because we can never have more than one request at a time with
375	* Control Endpoints. If anybody changes that assumption, this chunk
376	* needs to be updated accordingly.
377	*/
378	if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_UPDATETRANSFER &&
379	!usb_endpoint_xfer_isoc(epd: desc))
380	cmd &= ~(DWC3_DEPCMD_CMDIOC | DWC3_DEPCMD_CMDACT);
381	else
382	cmd |= DWC3_DEPCMD_CMDACT;
383
384	dwc3_writel(base: dep->regs, DWC3_DEPCMD, value: cmd);
385
386	if (!(cmd & DWC3_DEPCMD_CMDACT) ||
387	(DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_ENDTRANSFER &&
388	!(cmd & DWC3_DEPCMD_CMDIOC))) {
389	ret = 0;
390	goto skip_status;
391	}
392
393	do {
394	reg = dwc3_readl(base: dep->regs, DWC3_DEPCMD);
395	if (!(reg & DWC3_DEPCMD_CMDACT)) {
396	cmd_status = DWC3_DEPCMD_STATUS(reg);
397
398	switch (cmd_status) {
399	case 0:
400	ret = 0;
401	break;
402	case DEPEVT_TRANSFER_NO_RESOURCE:
403	dev_WARN(dwc->dev, "No resource for %s\n",
404	dep->name);
405	ret = -EINVAL;
406	break;
407	case DEPEVT_TRANSFER_BUS_EXPIRY:
408	/*
409	* SW issues START TRANSFER command to
410	* isochronous ep with future frame interval. If
411	* future interval time has already passed when
412	* core receives the command, it will respond
413	* with an error status of 'Bus Expiry'.
414	*
415	* Instead of always returning -EINVAL, let's
416	* give a hint to the gadget driver that this is
417	* the case by returning -EAGAIN.
418	*/
419	ret = -EAGAIN;
420	break;
421	default:
422	dev_WARN(dwc->dev, "UNKNOWN cmd status\n");
423	}
424
425	break;
426	}
427	} while (--timeout);
428
429	if (timeout == 0) {
430	ret = -ETIMEDOUT;
431	cmd_status = -ETIMEDOUT;
432	}
433
434	skip_status:
435	trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status);
436
437	if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
438	if (ret == 0)
439	dep->flags |= DWC3_EP_TRANSFER_STARTED;
440
441	if (ret != -ETIMEDOUT)
442	dwc3_gadget_ep_get_transfer_index(dep);
443	}
444
445	if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_ENDTRANSFER &&
446	!(cmd & DWC3_DEPCMD_CMDIOC))
447	mdelay(1);
448
449	if (saved_config) {
450	reg = dwc3_readl(base: dwc->regs, DWC3_GUSB2PHYCFG(0));
451	reg |= saved_config;
452	dwc3_writel(base: dwc->regs, DWC3_GUSB2PHYCFG(0), value: reg);
453	}
454
455	return ret;
456	}
457
458	static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep)
459	{
460	struct dwc3 *dwc = dep->dwc;
461	struct dwc3_gadget_ep_cmd_params params;
462	u32 cmd = DWC3_DEPCMD_CLEARSTALL;
463
464	/*
465	* As of core revision 2.60a the recommended programming model
466	* is to set the ClearPendIN bit when issuing a Clear Stall EP
467	* command for IN endpoints. This is to prevent an issue where
468	* some (non-compliant) hosts may not send ACK TPs for pending
469	* IN transfers due to a mishandled error condition. Synopsys
470	* STAR 9000614252.
471	*/
472	if (dep->direction &&
473	!DWC3_VER_IS_PRIOR(DWC3, 260A) &&
474	(dwc->gadget->speed >= USB_SPEED_SUPER))
475	cmd |= DWC3_DEPCMD_CLEARPENDIN;
476
477	memset(&params, 0, sizeof(params));
478
479	return dwc3_send_gadget_ep_cmd(dep, cmd, params: &params);
480	}
481
482	static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
483	struct dwc3_trb *trb)
484	{
485	u32 offset = (char *) trb - (char *) dep->trb_pool;
486
487	return dep->trb_pool_dma + offset;
488	}
489
490	static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
491	{
492	struct dwc3 *dwc = dep->dwc;
493
494	if (dep->trb_pool)
495	return 0;
496
497	dep->trb_pool = dma_alloc_coherent(dev: dwc->sysdev,
498	size: sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
499	dma_handle: &dep->trb_pool_dma, GFP_KERNEL);
500	if (!dep->trb_pool) {
501	dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
502	dep->name);
503	return -ENOMEM;
504	}
505
506	return 0;
507	}
508
509	static void dwc3_free_trb_pool(struct dwc3_ep *dep)
510	{
511	struct dwc3 *dwc = dep->dwc;
512
513	dma_free_coherent(dev: dwc->sysdev, size: sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
514	cpu_addr: dep->trb_pool, dma_handle: dep->trb_pool_dma);
515
516	dep->trb_pool = NULL;
517	dep->trb_pool_dma = 0;
518	}
519
520	static int dwc3_gadget_set_xfer_resource(struct dwc3_ep *dep)
521	{
522	struct dwc3_gadget_ep_cmd_params params;
523	int ret;
524
525	if (dep->flags & DWC3_EP_RESOURCE_ALLOCATED)
526	return 0;
527
528	memset(&params, 0x00, sizeof(params));
529
530	params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
531
532	ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE,
533	params: &params);
534	if (ret)
535	return ret;
536
537	dep->flags |= DWC3_EP_RESOURCE_ALLOCATED;
538	return 0;
539	}
540
541	/**
542	* dwc3_gadget_start_config - reset endpoint resources
543	* @dwc: pointer to the DWC3 context
544	* @resource_index: DEPSTARTCFG.XferRscIdx value (must be 0 or 2)
545	*
546	* Set resource_index=0 to reset all endpoints' resources allocation. Do this as
547	* part of the power-on/soft-reset initialization.
548	*
549	* Set resource_index=2 to reset only non-control endpoints' resources. Do this
550	* on receiving the SET_CONFIGURATION request or hibernation resume.
551	*/
552	int dwc3_gadget_start_config(struct dwc3 *dwc, unsigned int resource_index)
553	{
554	struct dwc3_gadget_ep_cmd_params params;
555	struct dwc3_ep *dep;
556	u32 cmd;
557	int i;
558	int ret;
559
560	if (resource_index != 0 && resource_index != 2)
561	return -EINVAL;
562
563	memset(&params, 0x00, sizeof(params));
564	cmd = DWC3_DEPCMD_DEPSTARTCFG;
565	cmd |= DWC3_DEPCMD_PARAM(resource_index);
566
567	ret = dwc3_send_gadget_ep_cmd(dep: dwc->eps[0], cmd, params: &params);
568	if (ret)
569	return ret;
570
571	/* Reset resource allocation flags */
572	for (i = resource_index; i < dwc->num_eps; i++) {
573	dep = dwc->eps[i];
574	if (!dep)
575	continue;
576
577	dep->flags &= ~DWC3_EP_RESOURCE_ALLOCATED;
578	}
579
580	return 0;
581	}
582
583	static int dwc3_gadget_set_ep_config(struct dwc3_ep *dep, unsigned int action)
584	{
585	const struct usb_ss_ep_comp_descriptor *comp_desc;
586	const struct usb_endpoint_descriptor *desc;
587	struct dwc3_gadget_ep_cmd_params params;
588	struct dwc3 *dwc = dep->dwc;
589
590	comp_desc = dep->endpoint.comp_desc;
591	desc = dep->endpoint.desc;
592
593	memset(&params, 0x00, sizeof(params));
594
595	params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
596	| DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc));
597
598	/* Burst size is only needed in SuperSpeed mode */
599	if (dwc->gadget->speed >= USB_SPEED_SUPER) {
600	u32 burst = dep->endpoint.maxburst;
601
602	params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1);
603	}
604
605	params.param0 |= action;
606	if (action == DWC3_DEPCFG_ACTION_RESTORE)
607	params.param2 |= dep->saved_state;
608
609	if (usb_endpoint_xfer_control(epd: desc))
610	params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN;
611
612	if (dep->number <= 1 || usb_endpoint_xfer_isoc(epd: desc))
613	params.param1 |= DWC3_DEPCFG_XFER_NOT_READY_EN;
614
615	if (usb_ss_max_streams(comp: comp_desc) && usb_endpoint_xfer_bulk(epd: desc)) {
616	params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
617	| DWC3_DEPCFG_XFER_COMPLETE_EN
618	| DWC3_DEPCFG_STREAM_EVENT_EN;
619	dep->stream_capable = true;
620	}
621
622	if (!usb_endpoint_xfer_control(epd: desc))
623	params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
624
625	/*
626	* We are doing 1:1 mapping for endpoints, meaning
627	* Physical Endpoints 2 maps to Logical Endpoint 2 and
628	* so on. We consider the direction bit as part of the physical
629	* endpoint number. So USB endpoint 0x81 is 0x03.
630	*/
631	params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
632
633	/*
634	* We must use the lower 16 TX FIFOs even though
635	* HW might have more
636	*/
637	if (dep->direction)
638	params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
639
640	if (desc->bInterval) {
641	u8 bInterval_m1;
642
643	/*
644	* Valid range for DEPCFG.bInterval_m1 is from 0 to 13.
645	*
646	* NOTE: The programming guide incorrectly stated bInterval_m1
647	* must be set to 0 when operating in fullspeed. Internally the
648	* controller does not have this limitation. See DWC_usb3x
649	* programming guide section 3.2.2.1.
650	*/
651	bInterval_m1 = min_t(u8, desc->bInterval - 1, 13);
652
653	if (usb_endpoint_type(epd: desc) == USB_ENDPOINT_XFER_INT &&
654	dwc->gadget->speed == USB_SPEED_FULL)
655	dep->interval = desc->bInterval;
656	else
657	dep->interval = 1 << (desc->bInterval - 1);
658
659	params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(bInterval_m1);
660	}
661
662	return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, params: &params);
663	}
664
665	/**
666	* dwc3_gadget_calc_tx_fifo_size - calculates the txfifo size value
667	* @dwc: pointer to the DWC3 context
668	* @mult: multiplier to be used when calculating the fifo_size
669	*
670	* Calculates the size value based on the equation below:
671	*
672	* DWC3 revision 280A and prior:
673	* fifo_size = mult * (max_packet / mdwidth) + 1;
674	*
675	* DWC3 revision 290A and onwards:
676	* fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1
677	*
678	* The max packet size is set to 1024, as the txfifo requirements mainly apply
679	* to super speed USB use cases. However, it is safe to overestimate the fifo
680	* allocations for other scenarios, i.e. high speed USB.
681	*/
682	static int dwc3_gadget_calc_tx_fifo_size(struct dwc3 *dwc, int mult)
683	{
684	int max_packet = 1024;
685	int fifo_size;
686	int mdwidth;
687
688	mdwidth = dwc3_mdwidth(dwc);
689
690	/* MDWIDTH is represented in bits, we need it in bytes */
691	mdwidth >>= 3;
692
693	if (DWC3_VER_IS_PRIOR(DWC3, 290A))
694	fifo_size = mult * (max_packet / mdwidth) + 1;
695	else
696	fifo_size = mult * ((max_packet + mdwidth) / mdwidth) + 1;
697	return fifo_size;
698	}
699
700	/**
701	* dwc3_gadget_calc_ram_depth - calculates the ram depth for txfifo
702	* @dwc: pointer to the DWC3 context
703	*/
704	static int dwc3_gadget_calc_ram_depth(struct dwc3 *dwc)
705	{
706	int ram_depth;
707	int fifo_0_start;
708	bool is_single_port_ram;
709
710	/* Check supporting RAM type by HW */
711	is_single_port_ram = DWC3_SPRAM_TYPE(dwc->hwparams.hwparams1);
712
713	/*
714	* If a single port RAM is utilized, then allocate TxFIFOs from
715	* RAM0. otherwise, allocate them from RAM1.
716	*/
717	ram_depth = is_single_port_ram ? DWC3_RAM0_DEPTH(dwc->hwparams.hwparams6) :
718	DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7);
719
720	/*
721	* In a single port RAM configuration, the available RAM is shared
722	* between the RX and TX FIFOs. This means that the txfifo can begin
723	* at a non-zero address.
724	*/
725	if (is_single_port_ram) {
726	u32 reg;
727
728	/* Check if TXFIFOs start at non-zero addr */
729	reg = dwc3_readl(base: dwc->regs, DWC3_GTXFIFOSIZ(0));
730	fifo_0_start = DWC3_GTXFIFOSIZ_TXFSTADDR(reg);
731
732	ram_depth -= (fifo_0_start >> 16);
733	}
734
735	return ram_depth;
736	}
737
738	/**
739	* dwc3_gadget_clear_tx_fifos - Clears txfifo allocation
740	* @dwc: pointer to the DWC3 context
741	*
742	* Iterates through all the endpoint registers and clears the previous txfifo
743	* allocations.
744	*/
745	void dwc3_gadget_clear_tx_fifos(struct dwc3 *dwc)
746	{
747	struct dwc3_ep *dep;
748	int fifo_depth;
749	int size;
750	int num;
751
752	if (!dwc->do_fifo_resize)
753	return;
754
755	/* Read ep0IN related TXFIFO size */
756	dep = dwc->eps[1];
757	size = dwc3_readl(base: dwc->regs, DWC3_GTXFIFOSIZ(0));
758	if (DWC3_IP_IS(DWC3))
759	fifo_depth = DWC3_GTXFIFOSIZ_TXFDEP(size);
760	else
761	fifo_depth = DWC31_GTXFIFOSIZ_TXFDEP(size);
762
763	dwc->last_fifo_depth = fifo_depth;
764	/* Clear existing TXFIFO for all IN eps except ep0 */
765	for (num = 3; num < min_t(int, dwc->num_eps, DWC3_ENDPOINTS_NUM); num += 2) {
766	dep = dwc->eps[num];
767	if (!dep)
768	continue;
769
770	/* Don't change TXFRAMNUM on usb31 version */
771	size = DWC3_IP_IS(DWC3) ? 0 :
772	dwc3_readl(base: dwc->regs, DWC3_GTXFIFOSIZ(num >> 1)) &
773	DWC31_GTXFIFOSIZ_TXFRAMNUM;
774
775	dwc3_writel(base: dwc->regs, DWC3_GTXFIFOSIZ(num >> 1), value: size);
776	dep->flags &= ~DWC3_EP_TXFIFO_RESIZED;
777	}
778	dwc->num_ep_resized = 0;
779	}
780
781	/*
782	* dwc3_gadget_resize_tx_fifos - reallocate fifo spaces for current use-case
783	* @dwc: pointer to our context structure
784	*
785	* This function will a best effort FIFO allocation in order
786	* to improve FIFO usage and throughput, while still allowing
787	* us to enable as many endpoints as possible.
788	*
789	* Keep in mind that this operation will be highly dependent
790	* on the configured size for RAM1 - which contains TxFifo -,
791	* the amount of endpoints enabled on coreConsultant tool, and
792	* the width of the Master Bus.
793	*
794	* In general, FIFO depths are represented with the following equation:
795	*
796	* fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1
797	*
798	* In conjunction with dwc3_gadget_check_config(), this resizing logic will
799	* ensure that all endpoints will have enough internal memory for one max
800	* packet per endpoint.
801	*/
802	static int dwc3_gadget_resize_tx_fifos(struct dwc3_ep *dep)
803	{
804	struct dwc3 *dwc = dep->dwc;
805	int fifo_0_start;
806	int ram_depth;
807	int fifo_size;
808	int min_depth;
809	int num_in_ep;
810	int remaining;
811	int num_fifos = 1;
812	int fifo;
813	int tmp;
814
815	if (!dwc->do_fifo_resize)
816	return 0;
817
818	/* resize IN endpoints except ep0 */
819	if (!usb_endpoint_dir_in(epd: dep->endpoint.desc) || dep->number <= 1)
820	return 0;
821
822	/* bail if already resized */
823	if (dep->flags & DWC3_EP_TXFIFO_RESIZED)
824	return 0;
825
826	ram_depth = dwc3_gadget_calc_ram_depth(dwc);
827
828	switch (dwc->gadget->speed) {
829	case USB_SPEED_SUPER_PLUS:
830	case USB_SPEED_SUPER:
831	if (usb_endpoint_xfer_bulk(epd: dep->endpoint.desc) ||
832	usb_endpoint_xfer_isoc(epd: dep->endpoint.desc))
833	num_fifos = min_t(unsigned int,
834	dep->endpoint.maxburst,
835	dwc->tx_fifo_resize_max_num);
836	break;
837	case USB_SPEED_HIGH:
838	if (usb_endpoint_xfer_isoc(epd: dep->endpoint.desc)) {
839	num_fifos = min_t(unsigned int,
840	usb_endpoint_maxp_mult(dep->endpoint.desc) + 1,
841	dwc->tx_fifo_resize_max_num);
842	break;
843	}
844	fallthrough;
845	case USB_SPEED_FULL:
846	if (usb_endpoint_xfer_bulk(epd: dep->endpoint.desc))
847	num_fifos = 2;
848	break;
849	default:
850	break;
851	}
852
853	/* FIFO size for a single buffer */
854	fifo = dwc3_gadget_calc_tx_fifo_size(dwc, mult: 1);
855
856	/* Calculate the number of remaining EPs w/o any FIFO */
857	num_in_ep = dwc->max_cfg_eps;
858	num_in_ep -= dwc->num_ep_resized;
859
860	/* Reserve at least one FIFO for the number of IN EPs */
861	min_depth = num_in_ep * (fifo + 1);
862	remaining = ram_depth - min_depth - dwc->last_fifo_depth;
863	remaining = max_t(int, 0, remaining);
864	/*
865	* We've already reserved 1 FIFO per EP, so check what we can fit in
866	* addition to it. If there is not enough remaining space, allocate
867	* all the remaining space to the EP.
868	*/
869	fifo_size = (num_fifos - 1) * fifo;
870	if (remaining < fifo_size)
871	fifo_size = remaining;
872
873	fifo_size += fifo;
874	/* Last increment according to the TX FIFO size equation */
875	fifo_size++;
876
877	/* Check if TXFIFOs start at non-zero addr */
878	tmp = dwc3_readl(base: dwc->regs, DWC3_GTXFIFOSIZ(0));
879	fifo_0_start = DWC3_GTXFIFOSIZ_TXFSTADDR(tmp);
880
881	fifo_size |= (fifo_0_start + (dwc->last_fifo_depth << 16));
882	if (DWC3_IP_IS(DWC3))
883	dwc->last_fifo_depth += DWC3_GTXFIFOSIZ_TXFDEP(fifo_size);
884	else
885	dwc->last_fifo_depth += DWC31_GTXFIFOSIZ_TXFDEP(fifo_size);
886
887	/* Check fifo size allocation doesn't exceed available RAM size. */
888	if (dwc->last_fifo_depth >= ram_depth) {
889	dev_err(dwc->dev, "Fifosize(%d) > RAM size(%d) %s depth:%d\n",
890	dwc->last_fifo_depth, ram_depth,
891	dep->endpoint.name, fifo_size);
892	if (DWC3_IP_IS(DWC3))
893	fifo_size = DWC3_GTXFIFOSIZ_TXFDEP(fifo_size);
894	else
895	fifo_size = DWC31_GTXFIFOSIZ_TXFDEP(fifo_size);
896
897	dwc->last_fifo_depth -= fifo_size;
898	return -ENOMEM;
899	}
900
901	dwc3_writel(base: dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1), value: fifo_size);
902	dep->flags |= DWC3_EP_TXFIFO_RESIZED;
903	dwc->num_ep_resized++;
904
905	return 0;
906	}
907
908	/**
909	* __dwc3_gadget_ep_enable - initializes a hw endpoint
910	* @dep: endpoint to be initialized
911	* @action: one of INIT, MODIFY or RESTORE
912	*
913	* Caller should take care of locking. Execute all necessary commands to
914	* initialize a HW endpoint so it can be used by a gadget driver.
915	*/
916	static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, unsigned int action)
917	{
918	const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
919	struct dwc3 *dwc = dep->dwc;
920
921	u32 reg;
922	int ret;
923
924	if (!(dep->flags & DWC3_EP_ENABLED)) {
925	ret = dwc3_gadget_resize_tx_fifos(dep);
926	if (ret)
927	return ret;
928	}
929
930	ret = dwc3_gadget_set_ep_config(dep, action);
931	if (ret)
932	return ret;
933
934	ret = dwc3_gadget_set_xfer_resource(dep);
935	if (ret)
936	return ret;
937
938	if (!(dep->flags & DWC3_EP_ENABLED)) {
939	struct dwc3_trb *trb_st_hw;
940	struct dwc3_trb *trb_link;
941
942	dep->type = usb_endpoint_type(epd: desc);
943	dep->flags |= DWC3_EP_ENABLED;
944
945	reg = dwc3_readl(base: dwc->regs, DWC3_DALEPENA);
946	reg |= DWC3_DALEPENA_EP(dep->number);
947	dwc3_writel(base: dwc->regs, DWC3_DALEPENA, value: reg);
948
949	dep->trb_dequeue = 0;
950	dep->trb_enqueue = 0;
951
952	if (usb_endpoint_xfer_control(epd: desc))
953	goto out;
954
955	/* Initialize the TRB ring */
956	memset(dep->trb_pool, 0,
957	sizeof(struct dwc3_trb) * DWC3_TRB_NUM);
958
959	/* Link TRB. The HWO bit is never reset */
960	trb_st_hw = &dep->trb_pool[0];
961
962	trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
963	trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
964	trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
965	trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;
966	trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
967	}
968
969	/*
970	* Issue StartTransfer here with no-op TRB so we can always rely on No
971	* Response Update Transfer command.
972	*/
973	if (usb_endpoint_xfer_bulk(epd: desc) ||
974	usb_endpoint_xfer_int(epd: desc)) {
975	struct dwc3_gadget_ep_cmd_params params;
976	struct dwc3_trb *trb;
977	dma_addr_t trb_dma;
978	u32 cmd;
979
980	memset(&params, 0, sizeof(params));
981	trb = &dep->trb_pool[0];
982	trb_dma = dwc3_trb_dma_offset(dep, trb);
983
984	params.param0 = upper_32_bits(trb_dma);
985	params.param1 = lower_32_bits(trb_dma);
986
987	cmd = DWC3_DEPCMD_STARTTRANSFER;
988
989	ret = dwc3_send_gadget_ep_cmd(dep, cmd, params: &params);
990	if (ret < 0)
991	return ret;
992
993	if (dep->stream_capable) {
994	/*
995	* For streams, at start, there maybe a race where the
996	* host primes the endpoint before the function driver
997	* queues a request to initiate a stream. In that case,
998	* the controller will not see the prime to generate the
999	* ERDY and start stream. To workaround this, issue a
1000	* no-op TRB as normal, but end it immediately. As a
1001	* result, when the function driver queues the request,
1002	* the next START_TRANSFER command will cause the
1003	* controller to generate an ERDY to initiate the
1004	* stream.
1005	*/
1006	dwc3_stop_active_transfer(dep, force: true, interrupt: true);
1007
1008	/*
1009	* All stream eps will reinitiate stream on NoStream
1010	* rejection.
1011	*
1012	* However, if the controller is capable of
1013	* TXF_FLUSH_BYPASS, then IN direction endpoints will
1014	* automatically restart the stream without the driver
1015	* initiation.
1016	*/
1017	if (!dep->direction ||
1018	!(dwc->hwparams.hwparams9 &
1019	DWC3_GHWPARAMS9_DEV_TXF_FLUSH_BYPASS))
1020	dep->flags |= DWC3_EP_FORCE_RESTART_STREAM;
1021	}
1022	}
1023
1024	out:
1025	trace_dwc3_gadget_ep_enable(dep);
1026
1027	return 0;
1028	}
1029
1030	void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep, int status)
1031	{
1032	struct dwc3_request *req;
1033
1034	dwc3_stop_active_transfer(dep, force: true, interrupt: false);
1035
1036	/* If endxfer is delayed, avoid unmapping requests */
1037	if (dep->flags & DWC3_EP_DELAY_STOP)
1038	return;
1039
1040	/* - giveback all requests to gadget driver */
1041	while (!list_empty(head: &dep->started_list)) {
1042	req = next_request(list: &dep->started_list);
1043
1044	dwc3_gadget_giveback(dep, req, status);
1045	}
1046
1047	while (!list_empty(head: &dep->pending_list)) {
1048	req = next_request(list: &dep->pending_list);
1049
1050	dwc3_gadget_giveback(dep, req, status);
1051	}
1052
1053	while (!list_empty(head: &dep->cancelled_list)) {
1054	req = next_request(list: &dep->cancelled_list);
1055
1056	dwc3_gadget_giveback(dep, req, status);
1057	}
1058	}
1059
1060	/**
1061	* __dwc3_gadget_ep_disable - disables a hw endpoint
1062	* @dep: the endpoint to disable
1063	*
1064	* This function undoes what __dwc3_gadget_ep_enable did and also removes
1065	* requests which are currently being processed by the hardware and those which
1066	* are not yet scheduled.
1067	*
1068	* Caller should take care of locking.
1069	*/
1070	static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
1071	{
1072	struct dwc3 *dwc = dep->dwc;
1073	u32 reg;
1074	u32 mask;
1075
1076	trace_dwc3_gadget_ep_disable(dep);
1077
1078	/* make sure HW endpoint isn't stalled */
1079	if (dep->flags & DWC3_EP_STALL)
1080	__dwc3_gadget_ep_set_halt(dep, value: 0, protocol: false);
1081
1082	reg = dwc3_readl(base: dwc->regs, DWC3_DALEPENA);
1083	reg &= ~DWC3_DALEPENA_EP(dep->number);
1084	dwc3_writel(base: dwc->regs, DWC3_DALEPENA, value: reg);
1085
1086	dwc3_remove_requests(dwc, dep, status: -ESHUTDOWN);
1087
1088	dep->stream_capable = false;
1089	dep->type = 0;
1090	mask = DWC3_EP_TXFIFO_RESIZED | DWC3_EP_RESOURCE_ALLOCATED;
1091	/*
1092	* dwc3_remove_requests() can exit early if DWC3 EP delayed stop is
1093	* set. Do not clear DEP flags, so that the end transfer command will
1094	* be reattempted during the next SETUP stage.
1095	*/
1096	if (dep->flags & DWC3_EP_DELAY_STOP)
1097	mask |= (DWC3_EP_DELAY_STOP | DWC3_EP_TRANSFER_STARTED);
1098	dep->flags &= mask;
1099
1100	/* Clear out the ep descriptors for non-ep0 */
1101	if (dep->number > 1) {
1102	dep->endpoint.comp_desc = NULL;
1103	dep->endpoint.desc = NULL;
1104	}
1105
1106	return 0;
1107	}
1108
1109	/* -------------------------------------------------------------------------- */
1110
1111	static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
1112	const struct usb_endpoint_descriptor *desc)
1113	{
1114	return -EINVAL;
1115	}
1116
1117	static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
1118	{
1119	return -EINVAL;
1120	}
1121
1122	/* -------------------------------------------------------------------------- */
1123
1124	static int dwc3_gadget_ep_enable(struct usb_ep *ep,
1125	const struct usb_endpoint_descriptor *desc)
1126	{
1127	struct dwc3_ep *dep;
1128	struct dwc3 *dwc;
1129	unsigned long flags;
1130	int ret;
1131
1132	if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
1133	pr_debug("dwc3: invalid parameters\n");
1134	return -EINVAL;
1135	}
1136
1137	if (!desc->wMaxPacketSize) {
1138	pr_debug("dwc3: missing wMaxPacketSize\n");
1139	return -EINVAL;
1140	}
1141
1142	dep = to_dwc3_ep(ep);
1143	dwc = dep->dwc;
1144
1145	if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED,
1146	"%s is already enabled\n",
1147	dep->name))
1148	return 0;
1149
1150	spin_lock_irqsave(&dwc->lock, flags);
1151	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
1152	spin_unlock_irqrestore(lock: &dwc->lock, flags);
1153
1154	return ret;
1155	}
1156
1157	static int dwc3_gadget_ep_disable(struct usb_ep *ep)
1158	{
1159	struct dwc3_ep *dep;
1160	struct dwc3 *dwc;
1161	unsigned long flags;
1162	int ret;
1163
1164	if (!ep) {
1165	pr_debug("dwc3: invalid parameters\n");
1166	return -EINVAL;
1167	}
1168
1169	dep = to_dwc3_ep(ep);
1170	dwc = dep->dwc;
1171
1172	if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED),
1173	"%s is already disabled\n",
1174	dep->name))
1175	return 0;
1176
1177	spin_lock_irqsave(&dwc->lock, flags);
1178	ret = __dwc3_gadget_ep_disable(dep);
1179	spin_unlock_irqrestore(lock: &dwc->lock, flags);
1180
1181	return ret;
1182	}
1183
1184	static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
1185	gfp_t gfp_flags)
1186	{
1187	struct dwc3_request *req;
1188	struct dwc3_ep *dep = to_dwc3_ep(ep);
1189
1190	req = kzalloc(sizeof(*req), gfp_flags);
1191	if (!req)
1192	return NULL;
1193
1194	req->direction = dep->direction;
1195	req->epnum = dep->number;
1196	req->dep = dep;
1197	req->status = DWC3_REQUEST_STATUS_UNKNOWN;
1198
1199	trace_dwc3_alloc_request(req);
1200
1201	return &req->request;
1202	}
1203
1204	static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
1205	struct usb_request *request)
1206	{
1207	struct dwc3_request *req = to_dwc3_request(request);
1208
1209	trace_dwc3_free_request(req);
1210	kfree(objp: req);
1211	}
1212
1213	/**
1214	* dwc3_ep_prev_trb - returns the previous TRB in the ring
1215	* @dep: The endpoint with the TRB ring
1216	* @index: The index of the current TRB in the ring
1217	*
1218	* Returns the TRB prior to the one pointed to by the index. If the
1219	* index is 0, we will wrap backwards, skip the link TRB, and return
1220	* the one just before that.
1221	*/
1222	static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index)
1223	{
1224	u8 tmp = index;
1225
1226	if (!tmp)
1227	tmp = DWC3_TRB_NUM - 1;
1228
1229	return &dep->trb_pool[tmp - 1];
1230	}
1231
1232	static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep)
1233	{
1234	u8 trbs_left;
1235
1236	/*
1237	* If the enqueue & dequeue are equal then the TRB ring is either full
1238	* or empty. It's considered full when there are DWC3_TRB_NUM-1 of TRBs
1239	* pending to be processed by the driver.
1240	*/
1241	if (dep->trb_enqueue == dep->trb_dequeue) {
1242	struct dwc3_request *req;
1243
1244	/*
1245	* If there is any request remained in the started_list with
1246	* active TRBs at this point, then there is no TRB available.
1247	*/
1248	req = next_request(list: &dep->started_list);
1249	if (req && req->num_trbs)
1250	return 0;
1251
1252	return DWC3_TRB_NUM - 1;
1253	}
1254
1255	trbs_left = dep->trb_dequeue - dep->trb_enqueue;
1256	trbs_left &= (DWC3_TRB_NUM - 1);
1257
1258	if (dep->trb_dequeue < dep->trb_enqueue)
1259	trbs_left--;
1260
1261	return trbs_left;
1262	}
1263
1264	/**
1265	* dwc3_prepare_one_trb - setup one TRB from one request
1266	* @dep: endpoint for which this request is prepared
1267	* @req: dwc3_request pointer
1268	* @trb_length: buffer size of the TRB
1269	* @chain: should this TRB be chained to the next?
1270	* @node: only for isochronous endpoints. First TRB needs different type.
1271	* @use_bounce_buffer: set to use bounce buffer
1272	* @must_interrupt: set to interrupt on TRB completion
1273	*/
1274	static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
1275	struct dwc3_request *req, unsigned int trb_length,
1276	unsigned int chain, unsigned int node, bool use_bounce_buffer,
1277	bool must_interrupt)
1278	{
1279	struct dwc3_trb *trb;
1280	dma_addr_t dma;
1281	unsigned int stream_id = req->request.stream_id;
1282	unsigned int short_not_ok = req->request.short_not_ok;
1283	unsigned int no_interrupt = req->request.no_interrupt;
1284	unsigned int is_last = req->request.is_last;
1285	struct dwc3 *dwc = dep->dwc;
1286	struct usb_gadget *gadget = dwc->gadget;
1287	enum usb_device_speed speed = gadget->speed;
1288
1289	if (use_bounce_buffer)
1290	dma = dep->dwc->bounce_addr;
1291	else if (req->request.num_sgs > 0)
1292	dma = sg_dma_address(req->start_sg);
1293	else
1294	dma = req->request.dma;
1295
1296	trb = &dep->trb_pool[dep->trb_enqueue];
1297
1298	if (!req->trb) {
1299	dwc3_gadget_move_started_request(req);
1300	req->trb = trb;
1301	req->trb_dma = dwc3_trb_dma_offset(dep, trb);
1302	}
1303
1304	req->num_trbs++;
1305
1306	trb->size = DWC3_TRB_SIZE_LENGTH(trb_length);
1307	trb->bpl = lower_32_bits(dma);
1308	trb->bph = upper_32_bits(dma);
1309
1310	switch (usb_endpoint_type(epd: dep->endpoint.desc)) {
1311	case USB_ENDPOINT_XFER_CONTROL:
1312	trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;
1313	break;
1314
1315	case USB_ENDPOINT_XFER_ISOC:
1316	if (!node) {
1317	trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
1318
1319	/*
1320	* USB Specification 2.0 Section 5.9.2 states that: "If
1321	* there is only a single transaction in the microframe,
1322	* only a DATA0 data packet PID is used. If there are
1323	* two transactions per microframe, DATA1 is used for
1324	* the first transaction data packet and DATA0 is used
1325	* for the second transaction data packet. If there are
1326	* three transactions per microframe, DATA2 is used for
1327	* the first transaction data packet, DATA1 is used for
1328	* the second, and DATA0 is used for the third."
1329	*
1330	* IOW, we should satisfy the following cases:
1331	*
1332	* 1) length <= maxpacket
1333	* - DATA0
1334	*
1335	* 2) maxpacket < length <= (2 * maxpacket)
1336	* - DATA1, DATA0
1337	*
1338	* 3) (2 * maxpacket) < length <= (3 * maxpacket)
1339	* - DATA2, DATA1, DATA0
1340	*/
1341	if (speed == USB_SPEED_HIGH) {
1342	struct usb_ep *ep = &dep->endpoint;
1343	unsigned int mult = 2;
1344	unsigned int maxp = usb_endpoint_maxp(epd: ep->desc);
1345
1346	if (req->request.length <= (2 * maxp))
1347	mult--;
1348
1349	if (req->request.length <= maxp)
1350	mult--;
1351
1352	trb->size |= DWC3_TRB_SIZE_PCM1(mult);
1353	}
1354	} else {
1355	trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
1356	}
1357
1358	if (!no_interrupt && !chain)
1359	trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
1360	break;
1361
1362	case USB_ENDPOINT_XFER_BULK:
1363	case USB_ENDPOINT_XFER_INT:
1364	trb->ctrl = DWC3_TRBCTL_NORMAL;
1365	break;
1366	default:
1367	/*
1368	* This is only possible with faulty memory because we
1369	* checked it already :)
1370	*/
1371	dev_WARN(dwc->dev, "Unknown endpoint type %d\n",
1372	usb_endpoint_type(dep->endpoint.desc));
1373	}
1374
1375	/*
1376	* Enable Continue on Short Packet
1377	* when endpoint is not a stream capable
1378	*/
1379	if (usb_endpoint_dir_out(epd: dep->endpoint.desc)) {
1380	if (!dep->stream_capable)
1381	trb->ctrl |= DWC3_TRB_CTRL_CSP;
1382
1383	if (short_not_ok)
1384	trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
1385	}
1386
1387	/* All TRBs setup for MST must set CSP=1 when LST=0 */
1388	if (dep->stream_capable && DWC3_MST_CAPABLE(&dwc->hwparams))
1389	trb->ctrl |= DWC3_TRB_CTRL_CSP;
1390
1391	if ((!no_interrupt && !chain) || must_interrupt)
1392	trb->ctrl |= DWC3_TRB_CTRL_IOC;
1393
1394	if (chain)
1395	trb->ctrl |= DWC3_TRB_CTRL_CHN;
1396	else if (dep->stream_capable && is_last &&
1397	!DWC3_MST_CAPABLE(&dwc->hwparams))
1398	trb->ctrl |= DWC3_TRB_CTRL_LST;
1399
1400	if (usb_endpoint_xfer_bulk(epd: dep->endpoint.desc) && dep->stream_capable)
1401	trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(stream_id);
1402
1403	/*
1404	* As per data book 4.2.3.2TRB Control Bit Rules section
1405	*
1406	* The controller autonomously checks the HWO field of a TRB to determine if the
1407	* entire TRB is valid. Therefore, software must ensure that the rest of the TRB
1408	* is valid before setting the HWO field to '1'. In most systems, this means that
1409	* software must update the fourth DWORD of a TRB last.
1410	*
1411	* However there is a possibility of CPU re-ordering here which can cause
1412	* controller to observe the HWO bit set prematurely.
1413	* Add a write memory barrier to prevent CPU re-ordering.
1414	*/
1415	wmb();
1416	trb->ctrl |= DWC3_TRB_CTRL_HWO;
1417
1418	dwc3_ep_inc_enq(dep);
1419
1420	trace_dwc3_prepare_trb(dep, trb);
1421	}
1422
1423	static bool dwc3_needs_extra_trb(struct dwc3_ep *dep, struct dwc3_request *req)
1424	{
1425	unsigned int maxp = usb_endpoint_maxp(epd: dep->endpoint.desc);
1426	unsigned int rem = req->request.length % maxp;
1427
1428	if ((req->request.length && req->request.zero && !rem &&
1429	!usb_endpoint_xfer_isoc(epd: dep->endpoint.desc)) ||
1430	(!req->direction && rem))
1431	return true;
1432
1433	return false;
1434	}
1435
1436	/**
1437	* dwc3_prepare_last_sg - prepare TRBs for the last SG entry
1438	* @dep: The endpoint that the request belongs to
1439	* @req: The request to prepare
1440	* @entry_length: The last SG entry size
1441	* @node: Indicates whether this is not the first entry (for isoc only)
1442	*
1443	* Return the number of TRBs prepared.
1444	*/
1445	static int dwc3_prepare_last_sg(struct dwc3_ep *dep,
1446	struct dwc3_request *req, unsigned int entry_length,
1447	unsigned int node)
1448	{
1449	unsigned int maxp = usb_endpoint_maxp(epd: dep->endpoint.desc);
1450	unsigned int rem = req->request.length % maxp;
1451	unsigned int num_trbs = 1;
1452	bool needs_extra_trb;
1453
1454	if (dwc3_needs_extra_trb(dep, req))
1455	num_trbs++;
1456
1457	if (dwc3_calc_trbs_left(dep) < num_trbs)
1458	return 0;
1459
1460	needs_extra_trb = num_trbs > 1;
1461
1462	/* Prepare a normal TRB */
1463	if (req->direction || req->request.length)
1464	dwc3_prepare_one_trb(dep, req, trb_length: entry_length,
1465	chain: needs_extra_trb, node, use_bounce_buffer: false, must_interrupt: false);
1466
1467	/* Prepare extra TRBs for ZLP and MPS OUT transfer alignment */
1468	if ((!req->direction && !req->request.length) || needs_extra_trb)
1469	dwc3_prepare_one_trb(dep, req,
1470	trb_length: req->direction ? 0 : maxp - rem,
1471	chain: false, node: 1, use_bounce_buffer: true, must_interrupt: false);
1472
1473	return num_trbs;
1474	}
1475
1476	static int dwc3_prepare_trbs_sg(struct dwc3_ep *dep,
1477	struct dwc3_request *req)
1478	{
1479	struct scatterlist *sg = req->start_sg;
1480	struct scatterlist *s;
1481	int i;
1482	unsigned int length = req->request.length;
1483	unsigned int remaining = req->num_pending_sgs;
1484	unsigned int num_queued_sgs = req->request.num_mapped_sgs - remaining;
1485	unsigned int num_trbs = req->num_trbs;
1486	bool needs_extra_trb = dwc3_needs_extra_trb(dep, req);
1487
1488	/*
1489	* If we resume preparing the request, then get the remaining length of
1490	* the request and resume where we left off.
1491	*/
1492	for_each_sg(req->request.sg, s, num_queued_sgs, i)
1493	length -= sg_dma_len(s);
1494
1495	for_each_sg(sg, s, remaining, i) {
1496	unsigned int num_trbs_left = dwc3_calc_trbs_left(dep);
1497	unsigned int trb_length;
1498	bool must_interrupt = false;
1499	bool last_sg = false;
1500
1501	trb_length = min_t(unsigned int, length, sg_dma_len(s));
1502
1503	length -= trb_length;
1504
1505	/*
1506	* IOMMU driver is coalescing the list of sgs which shares a
1507	* page boundary into one and giving it to USB driver. With
1508	* this the number of sgs mapped is not equal to the number of
1509	* sgs passed. So mark the chain bit to false if it isthe last
1510	* mapped sg.
1511	*/
1512	if ((i == remaining - 1) || !length)
1513	last_sg = true;
1514
1515	if (!num_trbs_left)
1516	break;
1517
1518	if (last_sg) {
1519	if (!dwc3_prepare_last_sg(dep, req, entry_length: trb_length, node: i))
1520	break;
1521	} else {
1522	/*
1523	* Look ahead to check if we have enough TRBs for the
1524	* next SG entry. If not, set interrupt on this TRB to
1525	* resume preparing the next SG entry when more TRBs are
1526	* free.
1527	*/
1528	if (num_trbs_left == 1 || (needs_extra_trb &&
1529	num_trbs_left <= 2 &&
1530	sg_dma_len(sg_next(s)) >= length)) {
1531	struct dwc3_request *r;
1532
1533	/* Check if previous requests already set IOC */
1534	list_for_each_entry(r, &dep->started_list, list) {
1535	if (r != req && !r->request.no_interrupt)
1536	break;
1537
1538	if (r == req)
1539	must_interrupt = true;
1540	}
1541	}
1542
1543	dwc3_prepare_one_trb(dep, req, trb_length, chain: 1, node: i, use_bounce_buffer: false,
1544	must_interrupt);
1545	}
1546
1547	/*
1548	* There can be a situation where all sgs in sglist are not
1549	* queued because of insufficient trb number. To handle this
1550	* case, update start_sg to next sg to be queued, so that
1551	* we have free trbs we can continue queuing from where we
1552	* previously stopped
1553	*/
1554	if (!last_sg)
1555	req->start_sg = sg_next(sg: s);
1556
1557	req->num_pending_sgs--;
1558
1559	/*
1560	* The number of pending SG entries may not correspond to the
1561	* number of mapped SG entries. If all the data are queued, then
1562	* don't include unused SG entries.
1563	*/
1564	if (length == 0) {
1565	req->num_pending_sgs = 0;
1566	break;
1567	}
1568
1569	if (must_interrupt)
1570	break;
1571	}
1572
1573	return req->num_trbs - num_trbs;
1574	}
1575
1576	static int dwc3_prepare_trbs_linear(struct dwc3_ep *dep,
1577	struct dwc3_request *req)
1578	{
1579	return dwc3_prepare_last_sg(dep, req, entry_length: req->request.length, node: 0);
1580	}
1581
1582	/*
1583	* dwc3_prepare_trbs - setup TRBs from requests
1584	* @dep: endpoint for which requests are being prepared
1585	*
1586	* The function goes through the requests list and sets up TRBs for the
1587	* transfers. The function returns once there are no more TRBs available or
1588	* it runs out of requests.
1589	*
1590	* Returns the number of TRBs prepared or negative errno.
1591	*/
1592	static int dwc3_prepare_trbs(struct dwc3_ep *dep)
1593	{
1594	struct dwc3_request *req, *n;
1595	int ret = 0;
1596
1597	BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
1598
1599	/*
1600	* We can get in a situation where there's a request in the started list
1601	* but there weren't enough TRBs to fully kick it in the first time
1602	* around, so it has been waiting for more TRBs to be freed up.
1603	*
1604	* In that case, we should check if we have a request with pending_sgs
1605	* in the started list and prepare TRBs for that request first,
1606	* otherwise we will prepare TRBs completely out of order and that will
1607	* break things.
1608	*/
1609	list_for_each_entry(req, &dep->started_list, list) {
1610	if (req->num_pending_sgs > 0) {
1611	ret = dwc3_prepare_trbs_sg(dep, req);
1612	if (!ret || req->num_pending_sgs)
1613	return ret;
1614	}
1615
1616	if (!dwc3_calc_trbs_left(dep))
1617	return ret;
1618
1619	/*
1620	* Don't prepare beyond a transfer. In DWC_usb32, its transfer
1621	* burst capability may try to read and use TRBs beyond the
1622	* active transfer instead of stopping.
1623	*/
1624	if (dep->stream_capable && req->request.is_last &&
1625	!DWC3_MST_CAPABLE(&dep->dwc->hwparams))
1626	return ret;
1627	}
1628
1629	list_for_each_entry_safe(req, n, &dep->pending_list, list) {
1630	struct dwc3 *dwc = dep->dwc;
1631
1632	ret = usb_gadget_map_request_by_dev(dev: dwc->sysdev, req: &req->request,
1633	is_in: dep->direction);
1634	if (ret)
1635	return ret;
1636
1637	req->start_sg = req->request.sg;
1638	req->num_pending_sgs = req->request.num_mapped_sgs;
1639
1640	if (req->num_pending_sgs > 0) {
1641	ret = dwc3_prepare_trbs_sg(dep, req);
1642	if (req->num_pending_sgs)
1643	return ret;
1644	} else {
1645	ret = dwc3_prepare_trbs_linear(dep, req);
1646	}
1647
1648	if (!ret || !dwc3_calc_trbs_left(dep))
1649	return ret;
1650
1651	/*
1652	* Don't prepare beyond a transfer. In DWC_usb32, its transfer
1653	* burst capability may try to read and use TRBs beyond the
1654	* active transfer instead of stopping.
1655	*/
1656	if (dep->stream_capable && req->request.is_last &&
1657	!DWC3_MST_CAPABLE(&dwc->hwparams))
1658	return ret;
1659	}
1660
1661	return ret;
1662	}
1663
1664	static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep);
1665
1666	static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep)
1667	{
1668	struct dwc3_gadget_ep_cmd_params params;
1669	struct dwc3_request *req;
1670	int starting;
1671	int ret;
1672	u32 cmd;
1673
1674	/*
1675	* Note that it's normal to have no new TRBs prepared (i.e. ret == 0).
1676	* This happens when we need to stop and restart a transfer such as in
1677	* the case of reinitiating a stream or retrying an isoc transfer.
1678	*/
1679	ret = dwc3_prepare_trbs(dep);
1680	if (ret < 0)
1681	return ret;
1682
1683	starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED);
1684
1685	/*
1686	* If there's no new TRB prepared and we don't need to restart a
1687	* transfer, there's no need to update the transfer.
1688	*/
1689	if (!ret && !starting)
1690	return ret;
1691
1692	req = next_request(list: &dep->started_list);
1693	if (!req) {
1694	dep->flags |= DWC3_EP_PENDING_REQUEST;
1695	return 0;
1696	}
1697
1698	memset(&params, 0, sizeof(params));
1699
1700	if (starting) {
1701	params.param0 = upper_32_bits(req->trb_dma);
1702	params.param1 = lower_32_bits(req->trb_dma);
1703	cmd = DWC3_DEPCMD_STARTTRANSFER;
1704
1705	if (dep->stream_capable)
1706	cmd |= DWC3_DEPCMD_PARAM(req->request.stream_id);
1707
1708	if (usb_endpoint_xfer_isoc(epd: dep->endpoint.desc))
1709	cmd |= DWC3_DEPCMD_PARAM(dep->frame_number);
1710	} else {
1711	cmd = DWC3_DEPCMD_UPDATETRANSFER |
1712	DWC3_DEPCMD_PARAM(dep->resource_index);
1713	}
1714
1715	ret = dwc3_send_gadget_ep_cmd(dep, cmd, params: &params);
1716	if (ret < 0) {
1717	struct dwc3_request *tmp;
1718
1719	if (ret == -EAGAIN)
1720	return ret;
1721
1722	dwc3_stop_active_transfer(dep, force: true, interrupt: true);
1723
1724	list_for_each_entry_safe(req, tmp, &dep->started_list, list)
1725	dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_DEQUEUED);
1726
1727	/* If ep isn't started, then there's no end transfer pending */
1728	if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
1729	dwc3_gadget_ep_cleanup_cancelled_requests(dep);
1730
1731	return ret;
1732	}
1733
1734	if (dep->stream_capable && req->request.is_last &&
1735	!DWC3_MST_CAPABLE(&dep->dwc->hwparams))
1736	dep->flags |= DWC3_EP_WAIT_TRANSFER_COMPLETE;
1737
1738	return 0;
1739	}
1740
1741	static int __dwc3_gadget_get_frame(struct dwc3 *dwc)
1742	{
1743	u32 reg;
1744
1745	reg = dwc3_readl(base: dwc->regs, DWC3_DSTS);
1746	return DWC3_DSTS_SOFFN(reg);
1747	}
1748
1749	/**
1750	* __dwc3_stop_active_transfer - stop the current active transfer
1751	* @dep: isoc endpoint
1752	* @force: set forcerm bit in the command
1753	* @interrupt: command complete interrupt after End Transfer command
1754	*
1755	* When setting force, the ForceRM bit will be set. In that case
1756	* the controller won't update the TRB progress on command
1757	* completion. It also won't clear the HWO bit in the TRB.
1758	* The command will also not complete immediately in that case.
1759	*/
1760	static int __dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, bool interrupt)
1761	{
1762	struct dwc3_gadget_ep_cmd_params params;
1763	u32 cmd;
1764	int ret;
1765
1766	cmd = DWC3_DEPCMD_ENDTRANSFER;
1767	cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0;
1768	cmd |= interrupt ? DWC3_DEPCMD_CMDIOC : 0;
1769	cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
1770	memset(&params, 0, sizeof(params));
1771	ret = dwc3_send_gadget_ep_cmd(dep, cmd, params: &params);
1772	/*
1773	* If the End Transfer command was timed out while the device is
1774	* not in SETUP phase, it's possible that an incoming Setup packet
1775	* may prevent the command's completion. Let's retry when the
1776	* ep0state returns to EP0_SETUP_PHASE.
1777	*/
1778	if (ret == -ETIMEDOUT && dep->dwc->ep0state != EP0_SETUP_PHASE) {
1779	dep->flags |= DWC3_EP_DELAY_STOP;
1780	return 0;
1781	}
1782
1783	if (ret)
1784	dev_err_ratelimited(dep->dwc->dev,
1785	"end transfer failed: %d\n", ret);
1786
1787	dep->resource_index = 0;
1788
1789	if (!interrupt)
1790	dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
1791	else if (!ret)
1792	dep->flags |= DWC3_EP_END_TRANSFER_PENDING;
1793
1794	dep->flags &= ~DWC3_EP_DELAY_STOP;
1795	return ret;
1796	}
1797
1798	/**
1799	* dwc3_gadget_start_isoc_quirk - workaround invalid frame number
1800	* @dep: isoc endpoint
1801	*
1802	* This function tests for the correct combination of BIT[15:14] from the 16-bit
1803	* microframe number reported by the XferNotReady event for the future frame
1804	* number to start the isoc transfer.
1805	*
1806	* In DWC_usb31 version 1.70a-ea06 and prior, for highspeed and fullspeed
1807	* isochronous IN, BIT[15:14] of the 16-bit microframe number reported by the
1808	* XferNotReady event are invalid. The driver uses this number to schedule the
1809	* isochronous transfer and passes it to the START TRANSFER command. Because
1810	* this number is invalid, the command may fail. If BIT[15:14] matches the
1811	* internal 16-bit microframe, the START TRANSFER command will pass and the
1812	* transfer will start at the scheduled time, if it is off by 1, the command
1813	* will still pass, but the transfer will start 2 seconds in the future. For all
1814	* other conditions, the START TRANSFER command will fail with bus-expiry.
1815	*
1816	* In order to workaround this issue, we can test for the correct combination of
1817	* BIT[15:14] by sending START TRANSFER commands with different values of
1818	* BIT[15:14]: 'b00, 'b01, 'b10, and 'b11. Each combination is 2^14 uframe apart
1819	* (or 2 seconds). 4 seconds into the future will result in a bus-expiry status.
1820	* As the result, within the 4 possible combinations for BIT[15:14], there will
1821	* be 2 successful and 2 failure START COMMAND status. One of the 2 successful
1822	* command status will result in a 2-second delay start. The smaller BIT[15:14]
1823	* value is the correct combination.
1824	*
1825	* Since there are only 4 outcomes and the results are ordered, we can simply
1826	* test 2 START TRANSFER commands with BIT[15:14] combinations 'b00 and 'b01 to
1827	* deduce the smaller successful combination.
1828	*
1829	* Let test0 = test status for combination 'b00 and test1 = test status for 'b01
1830	* of BIT[15:14]. The correct combination is as follow:
1831	*
1832	* if test0 fails and test1 passes, BIT[15:14] is 'b01
1833	* if test0 fails and test1 fails, BIT[15:14] is 'b10
1834	* if test0 passes and test1 fails, BIT[15:14] is 'b11
1835	* if test0 passes and test1 passes, BIT[15:14] is 'b00
1836	*
1837	* Synopsys STAR 9001202023: Wrong microframe number for isochronous IN
1838	* endpoints.
1839	*/
1840	static int dwc3_gadget_start_isoc_quirk(struct dwc3_ep *dep)
1841	{
1842	int cmd_status = 0;
1843	bool test0;
1844	bool test1;
1845
1846	while (dep->combo_num < 2) {
1847	struct dwc3_gadget_ep_cmd_params params;
1848	u32 test_frame_number;
1849	u32 cmd;
1850
1851	/*
1852	* Check if we can start isoc transfer on the next interval or
1853	* 4 uframes in the future with BIT[15:14] as dep->combo_num
1854	*/
1855	test_frame_number = dep->frame_number & DWC3_FRNUMBER_MASK;
1856	test_frame_number |= dep->combo_num << 14;
1857	test_frame_number += max_t(u32, 4, dep->interval);
1858
1859	params.param0 = upper_32_bits(dep->dwc->bounce_addr);
1860	params.param1 = lower_32_bits(dep->dwc->bounce_addr);
1861
1862	cmd = DWC3_DEPCMD_STARTTRANSFER;
1863	cmd |= DWC3_DEPCMD_PARAM(test_frame_number);
1864	cmd_status = dwc3_send_gadget_ep_cmd(dep, cmd, params: &params);
1865
1866	/* Redo if some other failure beside bus-expiry is received */
1867	if (cmd_status && cmd_status != -EAGAIN) {
1868	dep->start_cmd_status = 0;
1869	dep->combo_num = 0;
1870	return 0;
1871	}
1872
1873	/* Store the first test status */
1874	if (dep->combo_num == 0)
1875	dep->start_cmd_status = cmd_status;
1876
1877	dep->combo_num++;
1878
1879	/*
1880	* End the transfer if the START_TRANSFER command is successful
1881	* to wait for the next XferNotReady to test the command again
1882	*/
1883	if (cmd_status == 0) {
1884	dwc3_stop_active_transfer(dep, force: true, interrupt: true);
1885	return 0;
1886	}
1887	}
1888
1889	/* test0 and test1 are both completed at this point */
1890	test0 = (dep->start_cmd_status == 0);
1891	test1 = (cmd_status == 0);
1892
1893	if (!test0 && test1)
1894	dep->combo_num = 1;
1895	else if (!test0 && !test1)
1896	dep->combo_num = 2;
1897	else if (test0 && !test1)
1898	dep->combo_num = 3;
1899	else if (test0 && test1)
1900	dep->combo_num = 0;
1901
1902	dep->frame_number &= DWC3_FRNUMBER_MASK;
1903	dep->frame_number |= dep->combo_num << 14;
1904	dep->frame_number += max_t(u32, 4, dep->interval);
1905
1906	/* Reinitialize test variables */
1907	dep->start_cmd_status = 0;
1908	dep->combo_num = 0;
1909
1910	return __dwc3_gadget_kick_transfer(dep);
1911	}
1912
1913	static int __dwc3_gadget_start_isoc(struct dwc3_ep *dep)
1914	{
1915	const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
1916	struct dwc3 *dwc = dep->dwc;
1917	int ret;
1918	int i;
1919
1920	if (list_empty(head: &dep->pending_list) &&
1921	list_empty(head: &dep->started_list)) {
1922	dep->flags |= DWC3_EP_PENDING_REQUEST;
1923	return -EAGAIN;
1924	}
1925
1926	if (!dwc->dis_start_transfer_quirk &&
1927	(DWC3_VER_IS_PRIOR(DWC31, 170A) ||
1928	DWC3_VER_TYPE_IS_WITHIN(DWC31, 170A, EA01, EA06))) {
1929	if (dwc->gadget->speed <= USB_SPEED_HIGH && dep->direction)
1930	return dwc3_gadget_start_isoc_quirk(dep);
1931	}
1932
1933	if (desc->bInterval <= 14 &&
1934	dwc->gadget->speed >= USB_SPEED_HIGH) {
1935	u32 frame = __dwc3_gadget_get_frame(dwc);
1936	bool rollover = frame <
1937	(dep->frame_number & DWC3_FRNUMBER_MASK);
1938
1939	/*
1940	* frame_number is set from XferNotReady and may be already
1941	* out of date. DSTS only provides the lower 14 bit of the
1942	* current frame number. So add the upper two bits of
1943	* frame_number and handle a possible rollover.
1944	* This will provide the correct frame_number unless more than
1945	* rollover has happened since XferNotReady.
1946	*/
1947
1948	dep->frame_number = (dep->frame_number & ~DWC3_FRNUMBER_MASK) |
1949	frame;
1950	if (rollover)
1951	dep->frame_number += BIT(14);
1952	}
1953
1954	for (i = 0; i < DWC3_ISOC_MAX_RETRIES; i++) {
1955	int future_interval = i + 1;
1956
1957	/* Give the controller at least 500us to schedule transfers */
1958	if (desc->bInterval < 3)
1959	future_interval += 3 - desc->bInterval;
1960
1961	dep->frame_number = DWC3_ALIGN_FRAME(dep, future_interval);
1962
1963	ret = __dwc3_gadget_kick_transfer(dep);
1964	if (ret != -EAGAIN)
1965	break;
1966	}
1967
1968	/*
1969	* After a number of unsuccessful start attempts due to bus-expiry
1970	* status, issue END_TRANSFER command and retry on the next XferNotReady
1971	* event.
1972	*/
1973	if (ret == -EAGAIN)
1974	ret = __dwc3_stop_active_transfer(dep, force: false, interrupt: true);
1975
1976	return ret;
1977	}
1978
1979	static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
1980	{
1981	struct dwc3 *dwc = dep->dwc;
1982
1983	if (!dep->endpoint.desc || !dwc->pullups_connected || !dwc->connected) {
1984	dev_dbg(dwc->dev, "%s: can't queue to disabled endpoint\n",
1985	dep->name);
1986	return -ESHUTDOWN;
1987	}
1988
1989	if (WARN(req->dep != dep, "request %p belongs to '%s'\n",
1990	&req->request, req->dep->name))
1991	return -EINVAL;
1992
1993	if (WARN(req->status < DWC3_REQUEST_STATUS_COMPLETED,
1994	"%s: request %p already in flight\n",
1995	dep->name, &req->request))
1996	return -EINVAL;
1997
1998	pm_runtime_get(dev: dwc->dev);
1999
2000	req->request.actual = 0;
2001	req->request.status = -EINPROGRESS;
2002
2003	trace_dwc3_ep_queue(req);
2004
2005	list_add_tail(new: &req->list, head: &dep->pending_list);
2006	req->status = DWC3_REQUEST_STATUS_QUEUED;
2007
2008	if (dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE)
2009	return 0;
2010
2011	/*
2012	* Start the transfer only after the END_TRANSFER is completed
2013	* and endpoint STALL is cleared.
2014	*/
2015	if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) ||
2016	(dep->flags & DWC3_EP_WEDGE) ||
2017	(dep->flags & DWC3_EP_DELAY_STOP) ||
2018	(dep->flags & DWC3_EP_STALL)) {
2019	dep->flags |= DWC3_EP_DELAY_START;
2020	return 0;
2021	}
2022
2023	/*
2024	* NOTICE: Isochronous endpoints should NEVER be prestarted. We must
2025	* wait for a XferNotReady event so we will know what's the current
2026	* (micro-)frame number.
2027	*
2028	* Without this trick, we are very, very likely gonna get Bus Expiry
2029	* errors which will force us issue EndTransfer command.
2030	*/
2031	if (usb_endpoint_xfer_isoc(epd: dep->endpoint.desc)) {
2032	if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) {
2033	if ((dep->flags & DWC3_EP_PENDING_REQUEST))
2034	return __dwc3_gadget_start_isoc(dep);
2035
2036	return 0;
2037	}
2038	}
2039
2040	__dwc3_gadget_kick_transfer(dep);
2041
2042	return 0;
2043	}
2044
2045	static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
2046	gfp_t gfp_flags)
2047	{
2048	struct dwc3_request *req = to_dwc3_request(request);
2049	struct dwc3_ep *dep = to_dwc3_ep(ep);
2050	struct dwc3 *dwc = dep->dwc;
2051
2052	unsigned long flags;
2053
2054	int ret;
2055
2056	spin_lock_irqsave(&dwc->lock, flags);
2057	ret = __dwc3_gadget_ep_queue(dep, req);
2058	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2059
2060	return ret;
2061	}
2062
2063	static void dwc3_gadget_ep_skip_trbs(struct dwc3_ep *dep, struct dwc3_request *req)
2064	{
2065	int i;
2066
2067	/* If req->trb is not set, then the request has not started */
2068	if (!req->trb)
2069	return;
2070
2071	/*
2072	* If request was already started, this means we had to
2073	* stop the transfer. With that we also need to ignore
2074	* all TRBs used by the request, however TRBs can only
2075	* be modified after completion of END_TRANSFER
2076	* command. So what we do here is that we wait for
2077	* END_TRANSFER completion and only after that, we jump
2078	* over TRBs by clearing HWO and incrementing dequeue
2079	* pointer.
2080	*/
2081	for (i = 0; i < req->num_trbs; i++) {
2082	struct dwc3_trb *trb;
2083
2084	trb = &dep->trb_pool[dep->trb_dequeue];
2085	trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
2086	dwc3_ep_inc_deq(dep);
2087	}
2088
2089	req->num_trbs = 0;
2090	}
2091
2092	static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep)
2093	{
2094	struct dwc3_request *req;
2095	struct dwc3 *dwc = dep->dwc;
2096
2097	while (!list_empty(head: &dep->cancelled_list)) {
2098	req = next_request(list: &dep->cancelled_list);
2099	dwc3_gadget_ep_skip_trbs(dep, req);
2100	switch (req->status) {
2101	case DWC3_REQUEST_STATUS_DISCONNECTED:
2102	dwc3_gadget_giveback(dep, req, status: -ESHUTDOWN);
2103	break;
2104	case DWC3_REQUEST_STATUS_DEQUEUED:
2105	dwc3_gadget_giveback(dep, req, status: -ECONNRESET);
2106	break;
2107	case DWC3_REQUEST_STATUS_STALLED:
2108	dwc3_gadget_giveback(dep, req, status: -EPIPE);
2109	break;
2110	default:
2111	dev_err(dwc->dev, "request cancelled with wrong reason:%d\n", req->status);
2112	dwc3_gadget_giveback(dep, req, status: -ECONNRESET);
2113	break;
2114	}
2115	/*
2116	* The endpoint is disabled, let the dwc3_remove_requests()
2117	* handle the cleanup.
2118	*/
2119	if (!dep->endpoint.desc)
2120	break;
2121	}
2122	}
2123
2124	static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
2125	struct usb_request *request)
2126	{
2127	struct dwc3_request *req = to_dwc3_request(request);
2128	struct dwc3_request *r = NULL;
2129
2130	struct dwc3_ep *dep = to_dwc3_ep(ep);
2131	struct dwc3 *dwc = dep->dwc;
2132
2133	unsigned long flags;
2134	int ret = 0;
2135
2136	trace_dwc3_ep_dequeue(req);
2137
2138	spin_lock_irqsave(&dwc->lock, flags);
2139
2140	list_for_each_entry(r, &dep->cancelled_list, list) {
2141	if (r == req)
2142	goto out;
2143	}
2144
2145	list_for_each_entry(r, &dep->pending_list, list) {
2146	if (r == req) {
2147	/*
2148	* Explicitly check for EP0/1 as dequeue for those
2149	* EPs need to be handled differently. Control EP
2150	* only deals with one USB req, and giveback will
2151	* occur during dwc3_ep0_stall_and_restart(). EP0
2152	* requests are never added to started_list.
2153	*/
2154	if (dep->number > 1)
2155	dwc3_gadget_giveback(dep, req, status: -ECONNRESET);
2156	else
2157	dwc3_ep0_reset_state(dwc);
2158	goto out;
2159	}
2160	}
2161
2162	list_for_each_entry(r, &dep->started_list, list) {
2163	if (r == req) {
2164	struct dwc3_request *t;
2165
2166	/* wait until it is processed */
2167	dwc3_stop_active_transfer(dep, force: true, interrupt: true);
2168
2169	/*
2170	* Remove any started request if the transfer is
2171	* cancelled.
2172	*/
2173	list_for_each_entry_safe(r, t, &dep->started_list, list)
2174	dwc3_gadget_move_cancelled_request(req: r,
2175	DWC3_REQUEST_STATUS_DEQUEUED);
2176
2177	dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE;
2178
2179	goto out;
2180	}
2181	}
2182
2183	dev_err(dwc->dev, "request %p was not queued to %s\n",
2184	request, ep->name);
2185	ret = -EINVAL;
2186	out:
2187	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2188
2189	return ret;
2190	}
2191
2192	int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
2193	{
2194	struct dwc3_gadget_ep_cmd_params params;
2195	struct dwc3 *dwc = dep->dwc;
2196	struct dwc3_request *req;
2197	struct dwc3_request *tmp;
2198	int ret;
2199
2200	if (usb_endpoint_xfer_isoc(epd: dep->endpoint.desc)) {
2201	dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
2202	return -EINVAL;
2203	}
2204
2205	memset(&params, 0x00, sizeof(params));
2206
2207	if (value) {
2208	struct dwc3_trb *trb;
2209
2210	unsigned int transfer_in_flight;
2211	unsigned int started;
2212
2213	if (dep->number > 1)
2214	trb = dwc3_ep_prev_trb(dep, index: dep->trb_enqueue);
2215	else
2216	trb = &dwc->ep0_trb[dep->trb_enqueue];
2217
2218	transfer_in_flight = trb->ctrl & DWC3_TRB_CTRL_HWO;
2219	started = !list_empty(head: &dep->started_list);
2220
2221	if (!protocol && ((dep->direction && transfer_in_flight) ||
2222	(!dep->direction && started))) {
2223	return -EAGAIN;
2224	}
2225
2226	ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETSTALL,
2227	params: &params);
2228	if (ret)
2229	dev_err(dwc->dev, "failed to set STALL on %s\n",
2230	dep->name);
2231	else
2232	dep->flags |= DWC3_EP_STALL;
2233	} else {
2234	/*
2235	* Don't issue CLEAR_STALL command to control endpoints. The
2236	* controller automatically clears the STALL when it receives
2237	* the SETUP token.
2238	*/
2239	if (dep->number <= 1) {
2240	dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
2241	return 0;
2242	}
2243
2244	dwc3_stop_active_transfer(dep, force: true, interrupt: true);
2245
2246	list_for_each_entry_safe(req, tmp, &dep->started_list, list)
2247	dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_STALLED);
2248
2249	if (dep->flags & DWC3_EP_END_TRANSFER_PENDING ||
2250	(dep->flags & DWC3_EP_DELAY_STOP)) {
2251	dep->flags |= DWC3_EP_PENDING_CLEAR_STALL;
2252	if (protocol)
2253	dwc->clear_stall_protocol = dep->number;
2254
2255	return 0;
2256	}
2257
2258	dwc3_gadget_ep_cleanup_cancelled_requests(dep);
2259
2260	ret = dwc3_send_clear_stall_ep_cmd(dep);
2261	if (ret) {
2262	dev_err(dwc->dev, "failed to clear STALL on %s\n",
2263	dep->name);
2264	return ret;
2265	}
2266
2267	dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
2268
2269	if ((dep->flags & DWC3_EP_DELAY_START) &&
2270	!usb_endpoint_xfer_isoc(epd: dep->endpoint.desc))
2271	__dwc3_gadget_kick_transfer(dep);
2272
2273	dep->flags &= ~DWC3_EP_DELAY_START;
2274	}
2275
2276	return ret;
2277	}
2278
2279	static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
2280	{
2281	struct dwc3_ep *dep = to_dwc3_ep(ep);
2282	struct dwc3 *dwc = dep->dwc;
2283
2284	unsigned long flags;
2285
2286	int ret;
2287
2288	spin_lock_irqsave(&dwc->lock, flags);
2289	ret = __dwc3_gadget_ep_set_halt(dep, value, protocol: false);
2290	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2291
2292	return ret;
2293	}
2294
2295	static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
2296	{
2297	struct dwc3_ep *dep = to_dwc3_ep(ep);
2298	struct dwc3 *dwc = dep->dwc;
2299	unsigned long flags;
2300	int ret;
2301
2302	spin_lock_irqsave(&dwc->lock, flags);
2303	dep->flags |= DWC3_EP_WEDGE;
2304
2305	if (dep->number == 0 || dep->number == 1)
2306	ret = __dwc3_gadget_ep0_set_halt(ep, value: 1);
2307	else
2308	ret = __dwc3_gadget_ep_set_halt(dep, value: 1, protocol: false);
2309	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2310
2311	return ret;
2312	}
2313
2314	/* -------------------------------------------------------------------------- */
2315
2316	static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
2317	.bLength = USB_DT_ENDPOINT_SIZE,
2318	.bDescriptorType = USB_DT_ENDPOINT,
2319	.bmAttributes = USB_ENDPOINT_XFER_CONTROL,
2320	};
2321
2322	static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
2323	.enable = dwc3_gadget_ep0_enable,
2324	.disable = dwc3_gadget_ep0_disable,
2325	.alloc_request = dwc3_gadget_ep_alloc_request,
2326	.free_request = dwc3_gadget_ep_free_request,
2327	.queue = dwc3_gadget_ep0_queue,
2328	.dequeue = dwc3_gadget_ep_dequeue,
2329	.set_halt = dwc3_gadget_ep0_set_halt,
2330	.set_wedge = dwc3_gadget_ep_set_wedge,
2331	};
2332
2333	static const struct usb_ep_ops dwc3_gadget_ep_ops = {
2334	.enable = dwc3_gadget_ep_enable,
2335	.disable = dwc3_gadget_ep_disable,
2336	.alloc_request = dwc3_gadget_ep_alloc_request,
2337	.free_request = dwc3_gadget_ep_free_request,
2338	.queue = dwc3_gadget_ep_queue,
2339	.dequeue = dwc3_gadget_ep_dequeue,
2340	.set_halt = dwc3_gadget_ep_set_halt,
2341	.set_wedge = dwc3_gadget_ep_set_wedge,
2342	};
2343
2344	/* -------------------------------------------------------------------------- */
2345
2346	static void dwc3_gadget_enable_linksts_evts(struct dwc3 *dwc, bool set)
2347	{
2348	u32 reg;
2349
2350	if (DWC3_VER_IS_PRIOR(DWC3, 250A))
2351	return;
2352
2353	reg = dwc3_readl(base: dwc->regs, DWC3_DEVTEN);
2354	if (set)
2355	reg |= DWC3_DEVTEN_ULSTCNGEN;
2356	else
2357	reg &= ~DWC3_DEVTEN_ULSTCNGEN;
2358
2359	dwc3_writel(base: dwc->regs, DWC3_DEVTEN, value: reg);
2360	}
2361
2362	static int dwc3_gadget_get_frame(struct usb_gadget *g)
2363	{
2364	struct dwc3 *dwc = gadget_to_dwc(g);
2365
2366	return __dwc3_gadget_get_frame(dwc);
2367	}
2368
2369	static int __dwc3_gadget_wakeup(struct dwc3 *dwc)
2370	{
2371	int ret;
2372	u32 reg;
2373
2374	u8 link_state;
2375
2376	/*
2377	* According to the Databook Remote wakeup request should
2378	* be issued only when the device is in early suspend state.
2379	*
2380	* We can check that via USB Link State bits in DSTS register.
2381	*/
2382	reg = dwc3_readl(base: dwc->regs, DWC3_DSTS);
2383
2384	link_state = DWC3_DSTS_USBLNKST(reg);
2385
2386	switch (link_state) {
2387	case DWC3_LINK_STATE_RESET:
2388	case DWC3_LINK_STATE_RX_DET: /* in HS, means Early Suspend */
2389	case DWC3_LINK_STATE_U3: /* in HS, means SUSPEND */
2390	case DWC3_LINK_STATE_U2: /* in HS, means Sleep (L1) */
2391	case DWC3_LINK_STATE_U1:
2392	case DWC3_LINK_STATE_RESUME:
2393	break;
2394	default:
2395	return -EINVAL;
2396	}
2397
2398	dwc3_gadget_enable_linksts_evts(dwc, set: true);
2399
2400	ret = dwc3_gadget_set_link_state(dwc, state: DWC3_LINK_STATE_RECOV);
2401	if (ret < 0) {
2402	dev_err(dwc->dev, "failed to put link in Recovery\n");
2403	dwc3_gadget_enable_linksts_evts(dwc, set: false);
2404	return ret;
2405	}
2406
2407	/* Recent versions do this automatically */
2408	if (DWC3_VER_IS_PRIOR(DWC3, 194A)) {
2409	/* write zeroes to Link Change Request */
2410	reg = dwc3_readl(base: dwc->regs, DWC3_DCTL);
2411	reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
2412	dwc3_writel(base: dwc->regs, DWC3_DCTL, value: reg);
2413	}
2414
2415	/*
2416	* Since link status change events are enabled we will receive
2417	* an U0 event when wakeup is successful.
2418	*/
2419	return 0;
2420	}
2421
2422	static int dwc3_gadget_wakeup(struct usb_gadget *g)
2423	{
2424	struct dwc3 *dwc = gadget_to_dwc(g);
2425	unsigned long flags;
2426	int ret;
2427
2428	if (!dwc->wakeup_configured) {
2429	dev_err(dwc->dev, "remote wakeup not configured\n");
2430	return -EINVAL;
2431	}
2432
2433	spin_lock_irqsave(&dwc->lock, flags);
2434	if (!dwc->gadget->wakeup_armed) {
2435	dev_err(dwc->dev, "not armed for remote wakeup\n");
2436	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2437	return -EINVAL;
2438	}
2439	ret = __dwc3_gadget_wakeup(dwc);
2440
2441	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2442
2443	return ret;
2444	}
2445
2446	static void dwc3_resume_gadget(struct dwc3 *dwc);
2447
2448	static int dwc3_gadget_func_wakeup(struct usb_gadget *g, int intf_id)
2449	{
2450	struct dwc3 *dwc = gadget_to_dwc(g);
2451	unsigned long flags;
2452	int ret;
2453	int link_state;
2454
2455	if (!dwc->wakeup_configured) {
2456	dev_err(dwc->dev, "remote wakeup not configured\n");
2457	return -EINVAL;
2458	}
2459
2460	spin_lock_irqsave(&dwc->lock, flags);
2461	/*
2462	* If the link is in U3, signal for remote wakeup and wait for the
2463	* link to transition to U0 before sending device notification.
2464	*/
2465	link_state = dwc3_gadget_get_link_state(dwc);
2466	if (link_state == DWC3_LINK_STATE_U3) {
2467	dwc->wakeup_pending_funcs |= BIT(intf_id);
2468	ret = __dwc3_gadget_wakeup(dwc);
2469	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2470	return ret;
2471	}
2472
2473	ret = dwc3_send_gadget_generic_command(dwc, DWC3_DGCMD_DEV_NOTIFICATION,
2474	DWC3_DGCMDPAR_DN_FUNC_WAKE |
2475	DWC3_DGCMDPAR_INTF_SEL(intf_id));
2476	if (ret)
2477	dev_err(dwc->dev, "function remote wakeup failed, ret:%d\n", ret);
2478
2479	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2480
2481	return ret;
2482	}
2483
2484	static int dwc3_gadget_set_remote_wakeup(struct usb_gadget *g, int set)
2485	{
2486	struct dwc3 *dwc = gadget_to_dwc(g);
2487	unsigned long flags;
2488
2489	spin_lock_irqsave(&dwc->lock, flags);
2490	dwc->wakeup_configured = !!set;
2491	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2492
2493	return 0;
2494	}
2495
2496	static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
2497	int is_selfpowered)
2498	{
2499	struct dwc3 *dwc = gadget_to_dwc(g);
2500	unsigned long flags;
2501
2502	spin_lock_irqsave(&dwc->lock, flags);
2503	g->is_selfpowered = !!is_selfpowered;
2504	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2505
2506	return 0;
2507	}
2508
2509	static void dwc3_stop_active_transfers(struct dwc3 *dwc)
2510	{
2511	u32 epnum;
2512
2513	for (epnum = 2; epnum < dwc->num_eps; epnum++) {
2514	struct dwc3_ep *dep;
2515
2516	dep = dwc->eps[epnum];
2517	if (!dep)
2518	continue;
2519
2520	dwc3_remove_requests(dwc, dep, status: -ESHUTDOWN);
2521	}
2522	}
2523
2524	static void __dwc3_gadget_set_ssp_rate(struct dwc3 *dwc)
2525	{
2526	enum usb_ssp_rate ssp_rate = dwc->gadget_ssp_rate;
2527	u32 reg;
2528
2529	if (ssp_rate == USB_SSP_GEN_UNKNOWN)
2530	ssp_rate = dwc->max_ssp_rate;
2531
2532	reg = dwc3_readl(base: dwc->regs, DWC3_DCFG);
2533	reg &= ~DWC3_DCFG_SPEED_MASK;
2534	reg &= ~DWC3_DCFG_NUMLANES(~0);
2535
2536	if (ssp_rate == USB_SSP_GEN_1x2)
2537	reg |= DWC3_DCFG_SUPERSPEED;
2538	else if (dwc->max_ssp_rate != USB_SSP_GEN_1x2)
2539	reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2540
2541	if (ssp_rate != USB_SSP_GEN_2x1 &&
2542	dwc->max_ssp_rate != USB_SSP_GEN_2x1)
2543	reg |= DWC3_DCFG_NUMLANES(1);
2544
2545	dwc3_writel(base: dwc->regs, DWC3_DCFG, value: reg);
2546	}
2547
2548	static void __dwc3_gadget_set_speed(struct dwc3 *dwc)
2549	{
2550	enum usb_device_speed speed;
2551	u32 reg;
2552
2553	speed = dwc->gadget_max_speed;
2554	if (speed == USB_SPEED_UNKNOWN || speed > dwc->maximum_speed)
2555	speed = dwc->maximum_speed;
2556
2557	if (speed == USB_SPEED_SUPER_PLUS &&
2558	DWC3_IP_IS(DWC32)) {
2559	__dwc3_gadget_set_ssp_rate(dwc);
2560	return;
2561	}
2562
2563	reg = dwc3_readl(base: dwc->regs, DWC3_DCFG);
2564	reg &= ~(DWC3_DCFG_SPEED_MASK);
2565
2566	/*
2567	* WORKAROUND: DWC3 revision < 2.20a have an issue
2568	* which would cause metastability state on Run/Stop
2569	* bit if we try to force the IP to USB2-only mode.
2570	*
2571	* Because of that, we cannot configure the IP to any
2572	* speed other than the SuperSpeed
2573	*
2574	* Refers to:
2575	*
2576	* STAR#9000525659: Clock Domain Crossing on DCTL in
2577	* USB 2.0 Mode
2578	*/
2579	if (DWC3_VER_IS_PRIOR(DWC3, 220A) &&
2580	!dwc->dis_metastability_quirk) {
2581	reg |= DWC3_DCFG_SUPERSPEED;
2582	} else {
2583	switch (speed) {
2584	case USB_SPEED_FULL:
2585	reg |= DWC3_DCFG_FULLSPEED;
2586	break;
2587	case USB_SPEED_HIGH:
2588	reg |= DWC3_DCFG_HIGHSPEED;
2589	break;
2590	case USB_SPEED_SUPER:
2591	reg |= DWC3_DCFG_SUPERSPEED;
2592	break;
2593	case USB_SPEED_SUPER_PLUS:
2594	if (DWC3_IP_IS(DWC3))
2595	reg |= DWC3_DCFG_SUPERSPEED;
2596	else
2597	reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2598	break;
2599	default:
2600	dev_err(dwc->dev, "invalid speed (%d)\n", speed);
2601
2602	if (DWC3_IP_IS(DWC3))
2603	reg |= DWC3_DCFG_SUPERSPEED;
2604	else
2605	reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2606	}
2607	}
2608
2609	if (DWC3_IP_IS(DWC32) &&
2610	speed > USB_SPEED_UNKNOWN &&
2611	speed < USB_SPEED_SUPER_PLUS)
2612	reg &= ~DWC3_DCFG_NUMLANES(~0);
2613
2614	dwc3_writel(base: dwc->regs, DWC3_DCFG, value: reg);
2615	}
2616
2617	static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on)
2618	{
2619	u32 reg;
2620	u32 timeout = 2000;
2621	u32 saved_config = 0;
2622
2623	if (pm_runtime_suspended(dev: dwc->dev))
2624	return 0;
2625
2626	/*
2627	* When operating in USB 2.0 speeds (HS/FS), ensure that
2628	* GUSB2PHYCFG.ENBLSLPM and GUSB2PHYCFG.SUSPHY are cleared before starting
2629	* or stopping the controller. This resolves timeout issues that occur
2630	* during frequent role switches between host and device modes.
2631	*
2632	* Save and clear these settings, then restore them after completing the
2633	* controller start or stop sequence.
2634	*
2635	* This solution was discovered through experimentation as it is not
2636	* mentioned in the dwc3 programming guide. It has been tested on an
2637	* Exynos platforms.
2638	*/
2639	reg = dwc3_readl(base: dwc->regs, DWC3_GUSB2PHYCFG(0));
2640	if (reg & DWC3_GUSB2PHYCFG_SUSPHY) {
2641	saved_config |= DWC3_GUSB2PHYCFG_SUSPHY;
2642	reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
2643	}
2644
2645	if (reg & DWC3_GUSB2PHYCFG_ENBLSLPM) {
2646	saved_config |= DWC3_GUSB2PHYCFG_ENBLSLPM;
2647	reg &= ~DWC3_GUSB2PHYCFG_ENBLSLPM;
2648	}
2649
2650	if (saved_config)
2651	dwc3_writel(base: dwc->regs, DWC3_GUSB2PHYCFG(0), value: reg);
2652
2653	reg = dwc3_readl(base: dwc->regs, DWC3_DCTL);
2654	if (is_on) {
2655	if (DWC3_VER_IS_WITHIN(DWC3, ANY, 187A)) {
2656	reg &= ~DWC3_DCTL_TRGTULST_MASK;
2657	reg |= DWC3_DCTL_TRGTULST_RX_DET;
2658	}
2659
2660	if (!DWC3_VER_IS_PRIOR(DWC3, 194A))
2661	reg &= ~DWC3_DCTL_KEEP_CONNECT;
2662	reg |= DWC3_DCTL_RUN_STOP;
2663
2664	__dwc3_gadget_set_speed(dwc);
2665	dwc->pullups_connected = true;
2666	} else {
2667	reg &= ~DWC3_DCTL_RUN_STOP;
2668
2669	dwc->pullups_connected = false;
2670	}
2671
2672	dwc3_pre_run_stop(dwc, is_on);
2673	dwc3_gadget_dctl_write_safe(dwc, value: reg);
2674
2675	do {
2676	usleep_range(min: 1000, max: 2000);
2677	reg = dwc3_readl(base: dwc->regs, DWC3_DSTS);
2678	reg &= DWC3_DSTS_DEVCTRLHLT;
2679	} while (--timeout && !(!is_on ^ !reg));
2680
2681	if (saved_config) {
2682	reg = dwc3_readl(base: dwc->regs, DWC3_GUSB2PHYCFG(0));
2683	reg |= saved_config;
2684	dwc3_writel(base: dwc->regs, DWC3_GUSB2PHYCFG(0), value: reg);
2685	}
2686
2687	if (!timeout)
2688	return -ETIMEDOUT;
2689
2690	return 0;
2691	}
2692
2693	static void dwc3_gadget_disable_irq(struct dwc3 *dwc);
2694	static void __dwc3_gadget_stop(struct dwc3 *dwc);
2695	static int __dwc3_gadget_start(struct dwc3 *dwc);
2696
2697	static int dwc3_gadget_soft_disconnect(struct dwc3 *dwc)
2698	{
2699	unsigned long flags;
2700	int ret;
2701
2702	spin_lock_irqsave(&dwc->lock, flags);
2703	if (!dwc->pullups_connected) {
2704	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2705	return 0;
2706	}
2707
2708	dwc->connected = false;
2709
2710	/*
2711	* Attempt to end pending SETUP status phase, and not wait for the
2712	* function to do so.
2713	*/
2714	if (dwc->delayed_status)
2715	dwc3_ep0_send_delayed_status(dwc);
2716
2717	/*
2718	* In the Synopsys DesignWare Cores USB3 Databook Rev. 3.30a
2719	* Section 4.1.8 Table 4-7, it states that for a device-initiated
2720	* disconnect, the SW needs to ensure that it sends "a DEPENDXFER
2721	* command for any active transfers" before clearing the RunStop
2722	* bit.
2723	*/
2724	dwc3_stop_active_transfers(dwc);
2725	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2726
2727	/*
2728	* Per databook, when we want to stop the gadget, if a control transfer
2729	* is still in process, complete it and get the core into setup phase.
2730	* In case the host is unresponsive to a SETUP transaction, forcefully
2731	* stall the transfer, and move back to the SETUP phase, so that any
2732	* pending endxfers can be executed.
2733	*/
2734	if (dwc->ep0state != EP0_SETUP_PHASE) {
2735	reinit_completion(x: &dwc->ep0_in_setup);
2736
2737	ret = wait_for_completion_timeout(x: &dwc->ep0_in_setup,
2738	timeout: msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT));
2739	if (ret == 0) {
2740	dev_warn(dwc->dev, "wait for SETUP phase timed out\n");
2741	spin_lock_irqsave(&dwc->lock, flags);
2742	dwc3_ep0_reset_state(dwc);
2743	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2744	}
2745	}
2746
2747	/*
2748	* Note: if the GEVNTCOUNT indicates events in the event buffer, the
2749	* driver needs to acknowledge them before the controller can halt.
2750	* Simply let the interrupt handler acknowledges and handle the
2751	* remaining event generated by the controller while polling for
2752	* DSTS.DEVCTLHLT.
2753	*/
2754	ret = dwc3_gadget_run_stop(dwc, is_on: false);
2755
2756	/*
2757	* Stop the gadget after controller is halted, so that if needed, the
2758	* events to update EP0 state can still occur while the run/stop
2759	* routine polls for the halted state. DEVTEN is cleared as part of
2760	* gadget stop.
2761	*/
2762	spin_lock_irqsave(&dwc->lock, flags);
2763	__dwc3_gadget_stop(dwc);
2764	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2765
2766	usb_gadget_set_state(gadget: dwc->gadget, state: USB_STATE_NOTATTACHED);
2767
2768	return ret;
2769	}
2770
2771	static int dwc3_gadget_soft_connect(struct dwc3 *dwc)
2772	{
2773	int ret;
2774
2775	/*
2776	* In the Synopsys DWC_usb31 1.90a programming guide section
2777	* 4.1.9, it specifies that for a reconnect after a
2778	* device-initiated disconnect requires a core soft reset
2779	* (DCTL.CSftRst) before enabling the run/stop bit.
2780	*/
2781	ret = dwc3_core_soft_reset(dwc);
2782	if (ret)
2783	return ret;
2784
2785	dwc3_event_buffers_setup(dwc);
2786	__dwc3_gadget_start(dwc);
2787	return dwc3_gadget_run_stop(dwc, is_on: true);
2788	}
2789
2790	static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
2791	{
2792	struct dwc3 *dwc = gadget_to_dwc(g);
2793	int ret;
2794
2795	is_on = !!is_on;
2796
2797	dwc->softconnect = is_on;
2798
2799	/*
2800	* Avoid issuing a runtime resume if the device is already in the
2801	* suspended state during gadget disconnect. DWC3 gadget was already
2802	* halted/stopped during runtime suspend.
2803	*/
2804	if (!is_on) {
2805	pm_runtime_barrier(dev: dwc->dev);
2806	if (pm_runtime_suspended(dev: dwc->dev))
2807	return 0;
2808	}
2809
2810	/*
2811	* Check the return value for successful resume, or error. For a
2812	* successful resume, the DWC3 runtime PM resume routine will handle
2813	* the run stop sequence, so avoid duplicate operations here.
2814	*/
2815	ret = pm_runtime_get_sync(dev: dwc->dev);
2816	if (!ret || ret < 0) {
2817	pm_runtime_put(dev: dwc->dev);
2818	if (ret < 0)
2819	pm_runtime_set_suspended(dev: dwc->dev);
2820	return ret;
2821	}
2822
2823	if (dwc->pullups_connected == is_on) {
2824	pm_runtime_put(dev: dwc->dev);
2825	return 0;
2826	}
2827
2828	synchronize_irq(irq: dwc->irq_gadget);
2829
2830	if (!is_on)
2831	ret = dwc3_gadget_soft_disconnect(dwc);
2832	else
2833	ret = dwc3_gadget_soft_connect(dwc);
2834
2835	pm_runtime_put(dev: dwc->dev);
2836
2837	return ret;
2838	}
2839
2840	static void dwc3_gadget_enable_irq(struct dwc3 *dwc)
2841	{
2842	u32 reg;
2843
2844	/* Enable all but Start and End of Frame IRQs */
2845	reg = (DWC3_DEVTEN_EVNTOVERFLOWEN |
2846	DWC3_DEVTEN_CMDCMPLTEN |
2847	DWC3_DEVTEN_ERRTICERREN |
2848	DWC3_DEVTEN_WKUPEVTEN |
2849	DWC3_DEVTEN_CONNECTDONEEN |
2850	DWC3_DEVTEN_USBRSTEN |
2851	DWC3_DEVTEN_DISCONNEVTEN);
2852
2853	if (DWC3_VER_IS_PRIOR(DWC3, 250A))
2854	reg |= DWC3_DEVTEN_ULSTCNGEN;
2855
2856	/* On 2.30a and above this bit enables U3/L2-L1 Suspend Events */
2857	if (!DWC3_VER_IS_PRIOR(DWC3, 230A))
2858	reg |= DWC3_DEVTEN_U3L2L1SUSPEN;
2859
2860	dwc3_writel(base: dwc->regs, DWC3_DEVTEN, value: reg);
2861	}
2862
2863	static void dwc3_gadget_disable_irq(struct dwc3 *dwc)
2864	{
2865	/* mask all interrupts */
2866	dwc3_writel(base: dwc->regs, DWC3_DEVTEN, value: 0x00);
2867	}
2868
2869	static irqreturn_t dwc3_interrupt(int irq, void *_dwc);
2870	static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc);
2871
2872	/**
2873	* dwc3_gadget_setup_nump - calculate and initialize NUMP field of %DWC3_DCFG
2874	* @dwc: pointer to our context structure
2875	*
2876	* The following looks like complex but it's actually very simple. In order to
2877	* calculate the number of packets we can burst at once on OUT transfers, we're
2878	* gonna use RxFIFO size.
2879	*
2880	* To calculate RxFIFO size we need two numbers:
2881	* MDWIDTH = size, in bits, of the internal memory bus
2882	* RAM2_DEPTH = depth, in MDWIDTH, of internal RAM2 (where RxFIFO sits)
2883	*
2884	* Given these two numbers, the formula is simple:
2885	*
2886	* RxFIFO Size = (RAM2_DEPTH * MDWIDTH / 8) - 24 - 16;
2887	*
2888	* 24 bytes is for 3x SETUP packets
2889	* 16 bytes is a clock domain crossing tolerance
2890	*
2891	* Given RxFIFO Size, NUMP = RxFIFOSize / 1024;
2892	*/
2893	static void dwc3_gadget_setup_nump(struct dwc3 *dwc)
2894	{
2895	u32 ram2_depth;
2896	u32 mdwidth;
2897	u32 nump;
2898	u32 reg;
2899
2900	ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7);
2901	mdwidth = dwc3_mdwidth(dwc);
2902
2903	nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024;
2904	nump = min_t(u32, nump, 16);
2905
2906	/* update NumP */
2907	reg = dwc3_readl(base: dwc->regs, DWC3_DCFG);
2908	reg &= ~DWC3_DCFG_NUMP_MASK;
2909	reg |= nump << DWC3_DCFG_NUMP_SHIFT;
2910	dwc3_writel(base: dwc->regs, DWC3_DCFG, value: reg);
2911	}
2912
2913	static int __dwc3_gadget_start(struct dwc3 *dwc)
2914	{
2915	struct dwc3_ep *dep;
2916	int ret = 0;
2917	u32 reg;
2918
2919	/*
2920	* Use IMOD if enabled via dwc->imod_interval. Otherwise, if
2921	* the core supports IMOD, disable it.
2922	*/
2923	if (dwc->imod_interval) {
2924	dwc3_writel(base: dwc->regs, DWC3_DEV_IMOD(0), value: dwc->imod_interval);
2925	dwc3_writel(base: dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
2926	} else if (dwc3_has_imod(dwc)) {
2927	dwc3_writel(base: dwc->regs, DWC3_DEV_IMOD(0), value: 0);
2928	}
2929
2930	/*
2931	* We are telling dwc3 that we want to use DCFG.NUMP as ACK TP's NUMP
2932	* field instead of letting dwc3 itself calculate that automatically.
2933	*
2934	* This way, we maximize the chances that we'll be able to get several
2935	* bursts of data without going through any sort of endpoint throttling.
2936	*/
2937	reg = dwc3_readl(base: dwc->regs, DWC3_GRXTHRCFG);
2938	if (DWC3_IP_IS(DWC3))
2939	reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL;
2940	else
2941	reg &= ~DWC31_GRXTHRCFG_PKTCNTSEL;
2942
2943	dwc3_writel(base: dwc->regs, DWC3_GRXTHRCFG, value: reg);
2944
2945	dwc3_gadget_setup_nump(dwc);
2946
2947	/*
2948	* Currently the controller handles single stream only. So, Ignore
2949	* Packet Pending bit for stream selection and don't search for another
2950	* stream if the host sends Data Packet with PP=0 (for OUT direction) or
2951	* ACK with NumP=0 and PP=0 (for IN direction). This slightly improves
2952	* the stream performance.
2953	*/
2954	reg = dwc3_readl(base: dwc->regs, DWC3_DCFG);
2955	reg |= DWC3_DCFG_IGNSTRMPP;
2956	dwc3_writel(base: dwc->regs, DWC3_DCFG, value: reg);
2957
2958	/* Enable MST by default if the device is capable of MST */
2959	if (DWC3_MST_CAPABLE(&dwc->hwparams)) {
2960	reg = dwc3_readl(base: dwc->regs, DWC3_DCFG1);
2961	reg &= ~DWC3_DCFG1_DIS_MST_ENH;
2962	dwc3_writel(base: dwc->regs, DWC3_DCFG1, value: reg);
2963	}
2964
2965	/* Start with SuperSpeed Default */
2966	dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2967
2968	ret = dwc3_gadget_start_config(dwc, resource_index: 0);
2969	if (ret) {
2970	dev_err(dwc->dev, "failed to config endpoints\n");
2971	return ret;
2972	}
2973
2974	dep = dwc->eps[0];
2975	dep->flags = 0;
2976	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
2977	if (ret) {
2978	dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2979	goto err0;
2980	}
2981
2982	dep = dwc->eps[1];
2983	dep->flags = 0;
2984	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
2985	if (ret) {
2986	dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2987	goto err1;
2988	}
2989
2990	/* begin to receive SETUP packets */
2991	dwc->ep0state = EP0_SETUP_PHASE;
2992	dwc->ep0_bounced = false;
2993	dwc->link_state = DWC3_LINK_STATE_SS_DIS;
2994	dwc->delayed_status = false;
2995	dwc3_ep0_out_start(dwc);
2996
2997	dwc3_gadget_enable_irq(dwc);
2998	dwc3_enable_susphy(dwc, enable: true);
2999
3000	return 0;
3001
3002	err1:
3003	__dwc3_gadget_ep_disable(dep: dwc->eps[0]);
3004
3005	err0:
3006	return ret;
3007	}
3008
3009	static int dwc3_gadget_start(struct usb_gadget *g,
3010	struct usb_gadget_driver *driver)
3011	{
3012	struct dwc3 *dwc = gadget_to_dwc(g);
3013	unsigned long flags;
3014	int ret;
3015	int irq;
3016
3017	irq = dwc->irq_gadget;
3018	ret = request_threaded_irq(irq, handler: dwc3_interrupt, thread_fn: dwc3_thread_interrupt,
3019	IRQF_SHARED, name: "dwc3", dev: dwc->ev_buf);
3020	if (ret) {
3021	dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
3022	irq, ret);
3023	return ret;
3024	}
3025
3026	spin_lock_irqsave(&dwc->lock, flags);
3027	dwc->gadget_driver = driver;
3028	spin_unlock_irqrestore(lock: &dwc->lock, flags);
3029
3030	if (dwc->sys_wakeup)
3031	device_wakeup_enable(dev: dwc->sysdev);
3032
3033	return 0;
3034	}
3035
3036	static void __dwc3_gadget_stop(struct dwc3 *dwc)
3037	{
3038	dwc3_gadget_disable_irq(dwc);
3039	__dwc3_gadget_ep_disable(dep: dwc->eps[0]);
3040	__dwc3_gadget_ep_disable(dep: dwc->eps[1]);
3041	}
3042
3043	static int dwc3_gadget_stop(struct usb_gadget *g)
3044	{
3045	struct dwc3 *dwc = gadget_to_dwc(g);
3046	unsigned long flags;
3047
3048	if (dwc->sys_wakeup)
3049	device_wakeup_disable(dev: dwc->sysdev);
3050
3051	spin_lock_irqsave(&dwc->lock, flags);
3052	dwc->gadget_driver = NULL;
3053	dwc->max_cfg_eps = 0;
3054	spin_unlock_irqrestore(lock: &dwc->lock, flags);
3055
3056	free_irq(dwc->irq_gadget, dwc->ev_buf);
3057
3058	return 0;
3059	}
3060
3061	static void dwc3_gadget_config_params(struct usb_gadget *g,
3062	struct usb_dcd_config_params *params)
3063	{
3064	struct dwc3 *dwc = gadget_to_dwc(g);
3065
3066	params->besl_baseline = USB_DEFAULT_BESL_UNSPECIFIED;
3067	params->besl_deep = USB_DEFAULT_BESL_UNSPECIFIED;
3068
3069	/* Recommended BESL */
3070	if (!dwc->dis_enblslpm_quirk) {
3071	/*
3072	* If the recommended BESL baseline is 0 or if the BESL deep is
3073	* less than 2, Microsoft's Windows 10 host usb stack will issue
3074	* a usb reset immediately after it receives the extended BOS
3075	* descriptor and the enumeration will fail. To maintain
3076	* compatibility with the Windows' usb stack, let's set the
3077	* recommended BESL baseline to 1 and clamp the BESL deep to be
3078	* within 2 to 15.
3079	*/
3080	params->besl_baseline = 1;
3081	if (dwc->is_utmi_l1_suspend)
3082	params->besl_deep =
3083	clamp_t(u8, dwc->hird_threshold, 2, 15);
3084	}
3085
3086	/* U1 Device exit Latency */
3087	if (dwc->dis_u1_entry_quirk)
3088	params->bU1devExitLat = 0;
3089	else
3090	params->bU1devExitLat = DWC3_DEFAULT_U1_DEV_EXIT_LAT;
3091
3092	/* U2 Device exit Latency */
3093	if (dwc->dis_u2_entry_quirk)
3094	params->bU2DevExitLat = 0;
3095	else
3096	params->bU2DevExitLat =
3097	cpu_to_le16(DWC3_DEFAULT_U2_DEV_EXIT_LAT);
3098	}
3099
3100	static void dwc3_gadget_set_speed(struct usb_gadget *g,
3101	enum usb_device_speed speed)
3102	{
3103	struct dwc3 *dwc = gadget_to_dwc(g);
3104	unsigned long flags;
3105
3106	spin_lock_irqsave(&dwc->lock, flags);
3107	dwc->gadget_max_speed = speed;
3108	spin_unlock_irqrestore(lock: &dwc->lock, flags);
3109	}
3110
3111	static void dwc3_gadget_set_ssp_rate(struct usb_gadget *g,
3112	enum usb_ssp_rate rate)
3113	{
3114	struct dwc3 *dwc = gadget_to_dwc(g);
3115	unsigned long flags;
3116
3117	spin_lock_irqsave(&dwc->lock, flags);
3118	dwc->gadget_max_speed = USB_SPEED_SUPER_PLUS;
3119	dwc->gadget_ssp_rate = rate;
3120	spin_unlock_irqrestore(lock: &dwc->lock, flags);
3121	}
3122
3123	static int dwc3_gadget_vbus_draw(struct usb_gadget *g, unsigned int mA)
3124	{
3125	struct dwc3 *dwc = gadget_to_dwc(g);
3126	union power_supply_propval val = {0};
3127	int ret;
3128
3129	if (dwc->usb2_phy)
3130	return usb_phy_set_power(x: dwc->usb2_phy, mA);
3131
3132	if (!dwc->usb_psy)
3133	return -EOPNOTSUPP;
3134
3135	val.intval = 1000 * mA;
3136	ret = power_supply_set_property(psy: dwc->usb_psy, psp: POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT, val: &val);
3137
3138	return ret;
3139	}
3140
3141	/**
3142	* dwc3_gadget_check_config - ensure dwc3 can support the USB configuration
3143	* @g: pointer to the USB gadget
3144	*
3145	* Used to record the maximum number of endpoints being used in a USB composite
3146	* device. (across all configurations) This is to be used in the calculation
3147	* of the TXFIFO sizes when resizing internal memory for individual endpoints.
3148	* It will help ensured that the resizing logic reserves enough space for at
3149	* least one max packet.
3150	*/
3151	static int dwc3_gadget_check_config(struct usb_gadget *g)
3152	{
3153	struct dwc3 *dwc = gadget_to_dwc(g);
3154	struct usb_ep *ep;
3155	int fifo_size = 0;
3156	int ram_depth;
3157	int ep_num = 0;
3158
3159	if (!dwc->do_fifo_resize)
3160	return 0;
3161
3162	list_for_each_entry(ep, &g->ep_list, ep_list) {
3163	/* Only interested in the IN endpoints */
3164	if (ep->claimed && (ep->address & USB_DIR_IN))
3165	ep_num++;
3166	}
3167
3168	if (ep_num <= dwc->max_cfg_eps)
3169	return 0;
3170
3171	/* Update the max number of eps in the composition */
3172	dwc->max_cfg_eps = ep_num;
3173
3174	fifo_size = dwc3_gadget_calc_tx_fifo_size(dwc, mult: dwc->max_cfg_eps);
3175	/* Based on the equation, increment by one for every ep */
3176	fifo_size += dwc->max_cfg_eps;
3177
3178	/* Check if we can fit a single fifo per endpoint */
3179	ram_depth = dwc3_gadget_calc_ram_depth(dwc);
3180	if (fifo_size > ram_depth)
3181	return -ENOMEM;
3182
3183	return 0;
3184	}
3185
3186	static void dwc3_gadget_async_callbacks(struct usb_gadget *g, bool enable)
3187	{
3188	struct dwc3 *dwc = gadget_to_dwc(g);
3189	unsigned long flags;
3190
3191	spin_lock_irqsave(&dwc->lock, flags);
3192	dwc->async_callbacks = enable;
3193	spin_unlock_irqrestore(lock: &dwc->lock, flags);
3194	}
3195
3196	static const struct usb_gadget_ops dwc3_gadget_ops = {
3197	.get_frame = dwc3_gadget_get_frame,
3198	.wakeup = dwc3_gadget_wakeup,
3199	.func_wakeup = dwc3_gadget_func_wakeup,
3200	.set_remote_wakeup = dwc3_gadget_set_remote_wakeup,
3201	.set_selfpowered = dwc3_gadget_set_selfpowered,
3202	.pullup = dwc3_gadget_pullup,
3203	.udc_start = dwc3_gadget_start,
3204	.udc_stop = dwc3_gadget_stop,
3205	.udc_set_speed = dwc3_gadget_set_speed,
3206	.udc_set_ssp_rate = dwc3_gadget_set_ssp_rate,
3207	.get_config_params = dwc3_gadget_config_params,
3208	.vbus_draw = dwc3_gadget_vbus_draw,
3209	.check_config = dwc3_gadget_check_config,
3210	.udc_async_callbacks = dwc3_gadget_async_callbacks,
3211	};
3212
3213	/* -------------------------------------------------------------------------- */
3214
3215	static int dwc3_gadget_init_control_endpoint(struct dwc3_ep *dep)
3216	{
3217	struct dwc3 *dwc = dep->dwc;
3218
3219	usb_ep_set_maxpacket_limit(ep: &dep->endpoint, maxpacket_limit: 512);
3220	dep->endpoint.maxburst = 1;
3221	dep->endpoint.ops = &dwc3_gadget_ep0_ops;
3222	if (!dep->direction)
3223	dwc->gadget->ep0 = &dep->endpoint;
3224
3225	dep->endpoint.caps.type_control = true;
3226
3227	return 0;
3228	}
3229
3230	static int dwc3_gadget_init_in_endpoint(struct dwc3_ep *dep)
3231	{
3232	struct dwc3 *dwc = dep->dwc;
3233	u32 mdwidth;
3234	int size;
3235	int maxpacket;
3236
3237	mdwidth = dwc3_mdwidth(dwc);
3238
3239	/* MDWIDTH is represented in bits, we need it in bytes */
3240	mdwidth /= 8;
3241
3242	size = dwc3_readl(base: dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1));
3243	if (DWC3_IP_IS(DWC3))
3244	size = DWC3_GTXFIFOSIZ_TXFDEP(size);
3245	else
3246	size = DWC31_GTXFIFOSIZ_TXFDEP(size);
3247
3248	/*
3249	* maxpacket size is determined as part of the following, after assuming
3250	* a mult value of one maxpacket:
3251	* DWC3 revision 280A and prior:
3252	* fifo_size = mult * (max_packet / mdwidth) + 1;
3253	* maxpacket = mdwidth * (fifo_size - 1);
3254	*
3255	* DWC3 revision 290A and onwards:
3256	* fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1
3257	* maxpacket = mdwidth * ((fifo_size - 1) - 1) - mdwidth;
3258	*/
3259	if (DWC3_VER_IS_PRIOR(DWC3, 290A))
3260	maxpacket = mdwidth * (size - 1);
3261	else
3262	maxpacket = mdwidth * ((size - 1) - 1) - mdwidth;
3263
3264	/* Functionally, space for one max packet is sufficient */
3265	size = min_t(int, maxpacket, 1024);
3266	usb_ep_set_maxpacket_limit(ep: &dep->endpoint, maxpacket_limit: size);
3267
3268	dep->endpoint.max_streams = 16;
3269	dep->endpoint.ops = &dwc3_gadget_ep_ops;
3270	list_add_tail(new: &dep->endpoint.ep_list,
3271	head: &dwc->gadget->ep_list);
3272	dep->endpoint.caps.type_iso = true;
3273	dep->endpoint.caps.type_bulk = true;
3274	dep->endpoint.caps.type_int = true;
3275
3276	return dwc3_alloc_trb_pool(dep);
3277	}
3278
3279	static int dwc3_gadget_init_out_endpoint(struct dwc3_ep *dep)
3280	{
3281	struct dwc3 *dwc = dep->dwc;
3282	u32 mdwidth;
3283	int size;
3284
3285	mdwidth = dwc3_mdwidth(dwc);
3286
3287	/* MDWIDTH is represented in bits, convert to bytes */
3288	mdwidth /= 8;
3289
3290	/* All OUT endpoints share a single RxFIFO space */
3291	size = dwc3_readl(base: dwc->regs, DWC3_GRXFIFOSIZ(0));
3292	if (DWC3_IP_IS(DWC3))
3293	size = DWC3_GRXFIFOSIZ_RXFDEP(size);
3294	else
3295	size = DWC31_GRXFIFOSIZ_RXFDEP(size);
3296
3297	/* FIFO depth is in MDWDITH bytes */
3298	size *= mdwidth;
3299
3300	/*
3301	* To meet performance requirement, a minimum recommended RxFIFO size
3302	* is defined as follow:
3303	* RxFIFO size >= (3 x MaxPacketSize) +
3304	* (3 x 8 bytes setup packets size) + (16 bytes clock crossing margin)
3305	*
3306	* Then calculate the max packet limit as below.
3307	*/
3308	size -= (3 * 8) + 16;
3309	if (size < 0)
3310	size = 0;
3311	else
3312	size /= 3;
3313
3314	usb_ep_set_maxpacket_limit(ep: &dep->endpoint, maxpacket_limit: size);
3315	dep->endpoint.max_streams = 16;
3316	dep->endpoint.ops = &dwc3_gadget_ep_ops;
3317	list_add_tail(new: &dep->endpoint.ep_list,
3318	head: &dwc->gadget->ep_list);
3319	dep->endpoint.caps.type_iso = true;
3320	dep->endpoint.caps.type_bulk = true;
3321	dep->endpoint.caps.type_int = true;
3322
3323	return dwc3_alloc_trb_pool(dep);
3324	}
3325
3326	#define nostream_work_to_dep(w) (container_of(to_delayed_work(w), struct dwc3_ep, nostream_work))
3327	static void dwc3_nostream_work(struct work_struct *work)
3328	{
3329	struct dwc3_ep *dep = nostream_work_to_dep(work);
3330	struct dwc3 *dwc = dep->dwc;
3331	unsigned long flags;
3332
3333	spin_lock_irqsave(&dwc->lock, flags);
3334	if (dep->flags & DWC3_EP_STREAM_PRIMED)
3335	goto out;
3336
3337	if ((dep->flags & DWC3_EP_IGNORE_NEXT_NOSTREAM) ||
3338	(!DWC3_MST_CAPABLE(&dwc->hwparams) &&
3339	!(dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE)))
3340	goto out;
3341	/*
3342	* If the host rejects a stream due to no active stream, by the
3343	* USB and xHCI spec, the endpoint will be put back to idle
3344	* state. When the host is ready (buffer added/updated), it will
3345	* prime the endpoint to inform the usb device controller. This
3346	* triggers the device controller to issue ERDY to restart the
3347	* stream. However, some hosts don't follow this and keep the
3348	* endpoint in the idle state. No prime will come despite host
3349	* streams are updated, and the device controller will not be
3350	* triggered to generate ERDY to move the next stream data. To
3351	* workaround this and maintain compatibility with various
3352	* hosts, force to reinitiate the stream until the host is ready
3353	* instead of waiting for the host to prime the endpoint.
3354	*/
3355	if (DWC3_VER_IS_WITHIN(DWC32, 100A, ANY)) {
3356	unsigned int cmd = DWC3_DGCMD_SET_ENDPOINT_PRIME;
3357
3358	dwc3_send_gadget_generic_command(dwc, cmd, param: dep->number);
3359	} else {
3360	dep->flags |= DWC3_EP_DELAY_START;
3361	dwc3_stop_active_transfer(dep, force: true, interrupt: true);
3362	spin_unlock_irqrestore(lock: &dwc->lock, flags);
3363	return;
3364	}
3365	out:
3366	dep->flags &= ~DWC3_EP_IGNORE_NEXT_NOSTREAM;
3367	spin_unlock_irqrestore(lock: &dwc->lock, flags);
3368	}
3369
3370	static int dwc3_gadget_init_endpoint(struct dwc3 *dwc, u8 epnum)
3371	{
3372	struct dwc3_ep *dep;
3373	bool direction = epnum & 1;
3374	int ret;
3375	u8 num = epnum >> 1;
3376
3377	dep = kzalloc(sizeof(*dep), GFP_KERNEL);
3378	if (!dep)
3379	return -ENOMEM;
3380
3381	dep->dwc = dwc;
3382	dep->number = epnum;
3383	dep->direction = direction;
3384	dep->regs = dwc->regs + DWC3_DEP_BASE(epnum);
3385	dwc->eps[epnum] = dep;
3386	dep->combo_num = 0;
3387	dep->start_cmd_status = 0;
3388
3389	snprintf(buf: dep->name, size: sizeof(dep->name), fmt: "ep%u%s", num,
3390	direction ? "in" : "out");
3391
3392	dep->endpoint.name = dep->name;
3393
3394	if (!(dep->number > 1)) {
3395	dep->endpoint.desc = &dwc3_gadget_ep0_desc;
3396	dep->endpoint.comp_desc = NULL;
3397	}
3398
3399	if (num == 0)
3400	ret = dwc3_gadget_init_control_endpoint(dep);
3401	else if (direction)
3402	ret = dwc3_gadget_init_in_endpoint(dep);
3403	else
3404	ret = dwc3_gadget_init_out_endpoint(dep);
3405
3406	if (ret)
3407	return ret;
3408
3409	dep->endpoint.caps.dir_in = direction;
3410	dep->endpoint.caps.dir_out = !direction;
3411
3412	INIT_LIST_HEAD(list: &dep->pending_list);
3413	INIT_LIST_HEAD(list: &dep->started_list);
3414	INIT_LIST_HEAD(list: &dep->cancelled_list);
3415	INIT_DELAYED_WORK(&dep->nostream_work, dwc3_nostream_work);
3416
3417	dwc3_debugfs_create_endpoint_dir(dep);
3418
3419	return 0;
3420	}
3421
3422	static int dwc3_gadget_get_reserved_endpoints(struct dwc3 *dwc, const char *propname,
3423	u8 *eps, u8 num)
3424	{
3425	u8 count;
3426	int ret;
3427
3428	if (!device_property_present(dev: dwc->dev, propname))
3429	return 0;
3430
3431	ret = device_property_count_u8(dev: dwc->dev, propname);
3432	if (ret < 0)
3433	return ret;
3434	count = ret;
3435
3436	ret = device_property_read_u8_array(dev: dwc->dev, propname, val: eps, min(num, count));
3437	if (ret)
3438	return ret;
3439
3440	return count;
3441	}
3442
3443	static int dwc3_gadget_init_endpoints(struct dwc3 *dwc, u8 total)
3444	{
3445	const char *propname = "snps,reserved-endpoints";
3446	u8 epnum;
3447	u8 reserved_eps[DWC3_ENDPOINTS_NUM];
3448	u8 count;
3449	u8 num;
3450	int ret;
3451
3452	INIT_LIST_HEAD(list: &dwc->gadget->ep_list);
3453
3454	ret = dwc3_gadget_get_reserved_endpoints(dwc, propname,
3455	eps: reserved_eps, ARRAY_SIZE(reserved_eps));
3456	if (ret < 0) {
3457	dev_err(dwc->dev, "failed to read %s\n", propname);
3458	return ret;
3459	}
3460	count = ret;
3461
3462	for (epnum = 0; epnum < total; epnum++) {
3463	for (num = 0; num < count; num++) {
3464	if (epnum == reserved_eps[num])
3465	break;
3466	}
3467	if (num < count)
3468	continue;
3469
3470	ret = dwc3_gadget_init_endpoint(dwc, epnum);
3471	if (ret)
3472	return ret;
3473	}
3474
3475	return 0;
3476	}
3477
3478	static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
3479	{
3480	struct dwc3_ep *dep;
3481	u8 epnum;
3482
3483	for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
3484	dep = dwc->eps[epnum];
3485	if (!dep)
3486	continue;
3487	/*
3488	* Physical endpoints 0 and 1 are special; they form the
3489	* bi-directional USB endpoint 0.
3490	*
3491	* For those two physical endpoints, we don't allocate a TRB
3492	* pool nor do we add them the endpoints list. Due to that, we
3493	* shouldn't do these two operations otherwise we would end up
3494	* with all sorts of bugs when removing dwc3.ko.
3495	*/
3496	if (epnum != 0 && epnum != 1) {
3497	dwc3_free_trb_pool(dep);
3498	list_del(entry: &dep->endpoint.ep_list);
3499	}
3500
3501	dwc3_debugfs_remove_endpoint_dir(dep);
3502	kfree(objp: dep);
3503	}
3504	}
3505
3506	/* -------------------------------------------------------------------------- */
3507
3508	static int dwc3_gadget_ep_reclaim_completed_trb(struct dwc3_ep *dep,
3509	struct dwc3_request *req, struct dwc3_trb *trb,
3510	const struct dwc3_event_depevt *event, int status)
3511	{
3512	unsigned int count;
3513
3514	dwc3_ep_inc_deq(dep);
3515
3516	trace_dwc3_complete_trb(dep, trb);
3517	req->num_trbs--;
3518
3519	/*
3520	* If we're in the middle of series of chained TRBs and we
3521	* receive a short transfer along the way, DWC3 will skip
3522	* through all TRBs including the last TRB in the chain (the
3523	* where CHN bit is zero. DWC3 will also avoid clearing HWO
3524	* bit and SW has to do it manually.
3525	*
3526	* We're going to do that here to avoid problems of HW trying
3527	* to use bogus TRBs for transfers.
3528	*/
3529	if (trb->ctrl & DWC3_TRB_CTRL_HWO)
3530	trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
3531
3532	/*
3533	* For isochronous transfers, the first TRB in a service interval must
3534	* have the Isoc-First type. Track and report its interval frame number.
3535	*/
3536	if (usb_endpoint_xfer_isoc(epd: dep->endpoint.desc) &&
3537	(trb->ctrl & DWC3_TRBCTL_ISOCHRONOUS_FIRST)) {
3538	unsigned int frame_number;
3539
3540	frame_number = DWC3_TRB_CTRL_GET_SID_SOFN(trb->ctrl);
3541	frame_number &= ~(dep->interval - 1);
3542	req->request.frame_number = frame_number;
3543	}
3544
3545	/*
3546	* We use bounce buffer for requests that needs extra TRB or OUT ZLP. If
3547	* this TRB points to the bounce buffer address, it's a MPS alignment
3548	* TRB. Don't add it to req->remaining calculation.
3549	*/
3550	if (trb->bpl == lower_32_bits(dep->dwc->bounce_addr) &&
3551	trb->bph == upper_32_bits(dep->dwc->bounce_addr)) {
3552	trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
3553	return 1;
3554	}
3555
3556	count = trb->size & DWC3_TRB_SIZE_MASK;
3557	req->remaining += count;
3558
3559	if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
3560	return 1;
3561
3562	if (event->status & DEPEVT_STATUS_SHORT &&
3563	!(trb->ctrl & DWC3_TRB_CTRL_CHN))
3564	return 1;
3565
3566	if ((trb->ctrl & DWC3_TRB_CTRL_ISP_IMI) &&
3567	DWC3_TRB_SIZE_TRBSTS(trb->size) == DWC3_TRBSTS_MISSED_ISOC)
3568	return 1;
3569
3570	if ((trb->ctrl & DWC3_TRB_CTRL_IOC) ||
3571	(trb->ctrl & DWC3_TRB_CTRL_LST))
3572	return 1;
3573
3574	return 0;
3575	}
3576
3577	static int dwc3_gadget_ep_reclaim_trb_sg(struct dwc3_ep *dep,
3578	struct dwc3_request *req, const struct dwc3_event_depevt *event,
3579	int status)
3580	{
3581	struct dwc3_trb *trb;
3582	unsigned int num_completed_trbs = req->num_trbs;
3583	unsigned int i;
3584	int ret = 0;
3585
3586	for (i = 0; i < num_completed_trbs; i++) {
3587	trb = &dep->trb_pool[dep->trb_dequeue];
3588
3589	ret = dwc3_gadget_ep_reclaim_completed_trb(dep, req,
3590	trb, event, status);
3591	if (ret)
3592	break;
3593	}
3594
3595	return ret;
3596	}
3597
3598	static bool dwc3_gadget_ep_request_completed(struct dwc3_request *req)
3599	{
3600	return req->num_pending_sgs == 0 && req->num_trbs == 0;
3601	}
3602
3603	static int dwc3_gadget_ep_cleanup_completed_request(struct dwc3_ep *dep,
3604	const struct dwc3_event_depevt *event,
3605	struct dwc3_request *req, int status)
3606	{
3607	int request_status;
3608	int ret;
3609
3610	ret = dwc3_gadget_ep_reclaim_trb_sg(dep, req, event, status);
3611
3612	req->request.actual = req->request.length - req->remaining;
3613
3614	if (!dwc3_gadget_ep_request_completed(req))
3615	goto out;
3616
3617	/*
3618	* The event status only reflects the status of the TRB with IOC set.
3619	* For the requests that don't set interrupt on completion, the driver
3620	* needs to check and return the status of the completed TRBs associated
3621	* with the request. Use the status of the last TRB of the request.
3622	*/
3623	if (req->request.no_interrupt) {
3624	struct dwc3_trb *trb;
3625
3626	trb = dwc3_ep_prev_trb(dep, index: dep->trb_dequeue);
3627	switch (DWC3_TRB_SIZE_TRBSTS(trb->size)) {
3628	case DWC3_TRBSTS_MISSED_ISOC:
3629	/* Isoc endpoint only */
3630	request_status = -EXDEV;
3631	break;
3632	case DWC3_TRB_STS_XFER_IN_PROG:
3633	/* Applicable when End Transfer with ForceRM=0 */
3634	case DWC3_TRBSTS_SETUP_PENDING:
3635	/* Control endpoint only */
3636	case DWC3_TRBSTS_OK:
3637	default:
3638	request_status = 0;
3639	break;
3640	}
3641	} else {
3642	request_status = status;
3643	}
3644
3645	dwc3_gadget_giveback(dep, req, status: request_status);
3646
3647	out:
3648	return ret;
3649	}
3650
3651	static void dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep *dep,
3652	const struct dwc3_event_depevt *event, int status)
3653	{
3654	struct dwc3_request *req;
3655
3656	while (!list_empty(head: &dep->started_list)) {
3657	int ret;
3658
3659	req = next_request(list: &dep->started_list);
3660	ret = dwc3_gadget_ep_cleanup_completed_request(dep, event,
3661	req, status);
3662	if (ret)
3663	break;
3664	/*
3665	* The endpoint is disabled, let the dwc3_remove_requests()
3666	* handle the cleanup.
3667	*/
3668	if (!dep->endpoint.desc)
3669	break;
3670	}
3671	}
3672
3673	static bool dwc3_gadget_ep_should_continue(struct dwc3_ep *dep)
3674	{
3675	struct dwc3_request *req;
3676	struct dwc3 *dwc = dep->dwc;
3677
3678	if (!dep->endpoint.desc || !dwc->pullups_connected ||
3679	!dwc->connected)
3680	return false;
3681
3682	if (!list_empty(head: &dep->pending_list))
3683	return true;
3684
3685	/*
3686	* We only need to check the first entry of the started list. We can
3687	* assume the completed requests are removed from the started list.
3688	*/
3689	req = next_request(list: &dep->started_list);
3690	if (!req)
3691	return false;
3692
3693	return !dwc3_gadget_ep_request_completed(req);
3694	}
3695
3696	static void dwc3_gadget_endpoint_frame_from_event(struct dwc3_ep *dep,
3697	const struct dwc3_event_depevt *event)
3698	{
3699	dep->frame_number = event->parameters;
3700	}
3701
3702	static bool dwc3_gadget_endpoint_trbs_complete(struct dwc3_ep *dep,
3703	const struct dwc3_event_depevt *event, int status)
3704	{
3705	struct dwc3 *dwc = dep->dwc;
3706	bool no_started_trb = true;
3707
3708	dwc3_gadget_ep_cleanup_completed_requests(dep, event, status);
3709
3710	if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)
3711	goto out;
3712
3713	if (!dep->endpoint.desc)
3714	return no_started_trb;
3715
3716	if (usb_endpoint_xfer_isoc(epd: dep->endpoint.desc) &&
3717	list_empty(head: &dep->started_list) &&
3718	(list_empty(head: &dep->pending_list) || status == -EXDEV))
3719	dwc3_stop_active_transfer(dep, force: true, interrupt: true);
3720	else if (dwc3_gadget_ep_should_continue(dep))
3721	if (__dwc3_gadget_kick_transfer(dep) == 0)
3722	no_started_trb = false;
3723
3724	out:
3725	/*
3726	* WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.
3727	* See dwc3_gadget_linksts_change_interrupt() for 1st half.
3728	*/
3729	if (DWC3_VER_IS_PRIOR(DWC3, 183A)) {
3730	u32 reg;
3731	int i;
3732
3733	for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
3734	dep = dwc->eps[i];
3735	if (!dep)
3736	continue;
3737
3738	if (!(dep->flags & DWC3_EP_ENABLED))
3739	continue;
3740
3741	if (!list_empty(head: &dep->started_list))
3742	return no_started_trb;
3743	}
3744
3745	reg = dwc3_readl(base: dwc->regs, DWC3_DCTL);
3746	reg |= dwc->u1u2;
3747	dwc3_writel(base: dwc->regs, DWC3_DCTL, value: reg);
3748
3749	dwc->u1u2 = 0;
3750	}
3751
3752	return no_started_trb;
3753	}
3754
3755	static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep,
3756	const struct dwc3_event_depevt *event)
3757	{
3758	int status = 0;
3759
3760	if (!dep->endpoint.desc)
3761	return;
3762
3763	if (usb_endpoint_xfer_isoc(epd: dep->endpoint.desc))
3764	dwc3_gadget_endpoint_frame_from_event(dep, event);
3765
3766	if (event->status & DEPEVT_STATUS_BUSERR)
3767	status = -ECONNRESET;
3768
3769	if (event->status & DEPEVT_STATUS_MISSED_ISOC)
3770	status = -EXDEV;
3771
3772	dwc3_gadget_endpoint_trbs_complete(dep, event, status);
3773	}
3774
3775	static void dwc3_gadget_endpoint_transfer_complete(struct dwc3_ep *dep,
3776	const struct dwc3_event_depevt *event)
3777	{
3778	int status = 0;
3779
3780	dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
3781
3782	if (event->status & DEPEVT_STATUS_BUSERR)
3783	status = -ECONNRESET;
3784
3785	if (dwc3_gadget_endpoint_trbs_complete(dep, event, status))
3786	dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE;
3787	}
3788
3789	static void dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep *dep,
3790	const struct dwc3_event_depevt *event)
3791	{
3792	/*
3793	* During a device-initiated disconnect, a late xferNotReady event can
3794	* be generated after the End Transfer command resets the event filter,
3795	* but before the controller is halted. Ignore it to prevent a new
3796	* transfer from starting.
3797	*/
3798	if (!dep->dwc->connected)
3799	return;
3800
3801	dwc3_gadget_endpoint_frame_from_event(dep, event);
3802
3803	/*
3804	* The XferNotReady event is generated only once before the endpoint
3805	* starts. It will be generated again when END_TRANSFER command is
3806	* issued. For some controller versions, the XferNotReady event may be
3807	* generated while the END_TRANSFER command is still in process. Ignore
3808	* it and wait for the next XferNotReady event after the command is
3809	* completed.
3810	*/
3811	if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)
3812	return;
3813
3814	(void) __dwc3_gadget_start_isoc(dep);
3815	}
3816
3817	static void dwc3_gadget_endpoint_command_complete(struct dwc3_ep *dep,
3818	const struct dwc3_event_depevt *event)
3819	{
3820	u8 cmd = DEPEVT_PARAMETER_CMD(event->parameters);
3821
3822	if (cmd != DWC3_DEPCMD_ENDTRANSFER)
3823	return;
3824
3825	/*
3826	* The END_TRANSFER command will cause the controller to generate a
3827	* NoStream Event, and it's not due to the host DP NoStream rejection.
3828	* Ignore the next NoStream event.
3829	*/
3830	if (dep->stream_capable)
3831	dep->flags |= DWC3_EP_IGNORE_NEXT_NOSTREAM;
3832
3833	dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
3834	dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
3835	dwc3_gadget_ep_cleanup_cancelled_requests(dep);
3836
3837	if (dep->flags & DWC3_EP_PENDING_CLEAR_STALL) {
3838	struct dwc3 *dwc = dep->dwc;
3839
3840	dep->flags &= ~DWC3_EP_PENDING_CLEAR_STALL;
3841	if (dwc3_send_clear_stall_ep_cmd(dep)) {
3842	struct usb_ep *ep0 = &dwc->eps[0]->endpoint;
3843
3844	dev_err(dwc->dev, "failed to clear STALL on %s\n", dep->name);
3845	if (dwc->delayed_status)
3846	__dwc3_gadget_ep0_set_halt(ep: ep0, value: 1);
3847	return;
3848	}
3849
3850	dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
3851	if (dwc->clear_stall_protocol == dep->number)
3852	dwc3_ep0_send_delayed_status(dwc);
3853	}
3854
3855	if ((dep->flags & DWC3_EP_DELAY_START) &&
3856	!usb_endpoint_xfer_isoc(epd: dep->endpoint.desc))
3857	__dwc3_gadget_kick_transfer(dep);
3858
3859	dep->flags &= ~DWC3_EP_DELAY_START;
3860	}
3861
3862	static void dwc3_gadget_endpoint_stream_event(struct dwc3_ep *dep,
3863	const struct dwc3_event_depevt *event)
3864	{
3865	if (event->status == DEPEVT_STREAMEVT_FOUND) {
3866	cancel_delayed_work(dwork: &dep->nostream_work);
3867	dep->flags |= DWC3_EP_STREAM_PRIMED;
3868	dep->flags &= ~DWC3_EP_IGNORE_NEXT_NOSTREAM;
3869	return;
3870	}
3871
3872	/* Note: NoStream rejection event param value is 0 and not 0xFFFF */
3873	switch (event->parameters) {
3874	case DEPEVT_STREAM_PRIME:
3875	cancel_delayed_work(dwork: &dep->nostream_work);
3876	dep->flags |= DWC3_EP_STREAM_PRIMED;
3877	dep->flags &= ~DWC3_EP_IGNORE_NEXT_NOSTREAM;
3878	break;
3879	case DEPEVT_STREAM_NOSTREAM:
3880	dep->flags &= ~DWC3_EP_STREAM_PRIMED;
3881	if (dep->flags & DWC3_EP_FORCE_RESTART_STREAM)
3882	queue_delayed_work(wq: system_percpu_wq, dwork: &dep->nostream_work,
3883	delay: msecs_to_jiffies(m: 100));
3884	break;
3885	}
3886	}
3887
3888	static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
3889	const struct dwc3_event_depevt *event)
3890	{
3891	struct dwc3_ep *dep;
3892	u8 epnum = event->endpoint_number;
3893
3894	dep = dwc->eps[epnum];
3895	if (!dep) {
3896	dev_warn(dwc->dev, "spurious event, endpoint %u is not allocated\n", epnum);
3897	return;
3898	}
3899
3900	if (!(dep->flags & DWC3_EP_ENABLED)) {
3901	if ((epnum > 1) && !(dep->flags & DWC3_EP_TRANSFER_STARTED))
3902	return;
3903
3904	/* Handle only EPCMDCMPLT when EP disabled */
3905	if ((event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT) &&
3906	!(epnum <= 1 && event->endpoint_event == DWC3_DEPEVT_XFERCOMPLETE))
3907	return;
3908	}
3909
3910	if (epnum == 0 || epnum == 1) {
3911	dwc3_ep0_interrupt(dwc, event);
3912	return;
3913	}
3914
3915	switch (event->endpoint_event) {
3916	case DWC3_DEPEVT_XFERINPROGRESS:
3917	dwc3_gadget_endpoint_transfer_in_progress(dep, event);
3918	break;
3919	case DWC3_DEPEVT_XFERNOTREADY:
3920	dwc3_gadget_endpoint_transfer_not_ready(dep, event);
3921	break;
3922	case DWC3_DEPEVT_EPCMDCMPLT:
3923	dwc3_gadget_endpoint_command_complete(dep, event);
3924	break;
3925	case DWC3_DEPEVT_XFERCOMPLETE:
3926	dwc3_gadget_endpoint_transfer_complete(dep, event);
3927	break;
3928	case DWC3_DEPEVT_STREAMEVT:
3929	dwc3_gadget_endpoint_stream_event(dep, event);
3930	break;
3931	case DWC3_DEPEVT_RXTXFIFOEVT:
3932	break;
3933	default:
3934	dev_err(dwc->dev, "unknown endpoint event %d\n", event->endpoint_event);
3935	break;
3936	}
3937	}
3938
3939	static void dwc3_disconnect_gadget(struct dwc3 *dwc)
3940	{
3941	if (dwc->async_callbacks && dwc->gadget_driver->disconnect) {
3942	spin_unlock(lock: &dwc->lock);
3943	dwc->gadget_driver->disconnect(dwc->gadget);
3944	spin_lock(lock: &dwc->lock);
3945	}
3946	}
3947
3948	static void dwc3_suspend_gadget(struct dwc3 *dwc)
3949	{
3950	if (dwc->async_callbacks && dwc->gadget_driver->suspend) {
3951	spin_unlock(lock: &dwc->lock);
3952	dwc->gadget_driver->suspend(dwc->gadget);
3953	spin_lock(lock: &dwc->lock);
3954	}
3955	}
3956
3957	static void dwc3_resume_gadget(struct dwc3 *dwc)
3958	{
3959	if (dwc->async_callbacks && dwc->gadget_driver->resume) {
3960	spin_unlock(lock: &dwc->lock);
3961	dwc->gadget_driver->resume(dwc->gadget);
3962	spin_lock(lock: &dwc->lock);
3963	}
3964	}
3965
3966	static void dwc3_reset_gadget(struct dwc3 *dwc)
3967	{
3968	if (!dwc->gadget_driver)
3969	return;
3970
3971	if (dwc->async_callbacks && dwc->gadget->speed != USB_SPEED_UNKNOWN) {
3972	spin_unlock(lock: &dwc->lock);
3973	usb_gadget_udc_reset(gadget: dwc->gadget, driver: dwc->gadget_driver);
3974	spin_lock(lock: &dwc->lock);
3975	}
3976	}
3977
3978	void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force,
3979	bool interrupt)
3980	{
3981	struct dwc3 *dwc = dep->dwc;
3982
3983	/*
3984	* Only issue End Transfer command to the control endpoint of a started
3985	* Data Phase. Typically we should only do so in error cases such as
3986	* invalid/unexpected direction as described in the control transfer
3987	* flow of the programming guide.
3988	*/
3989	if (dep->number <= 1 && dwc->ep0state != EP0_DATA_PHASE)
3990	return;
3991
3992	if (interrupt && (dep->flags & DWC3_EP_DELAY_STOP))
3993	return;
3994
3995	if (!(dep->flags & DWC3_EP_TRANSFER_STARTED) ||
3996	(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
3997	return;
3998
3999	/*
4000	* If a Setup packet is received but yet to DMA out, the controller will
4001	* not process the End Transfer command of any endpoint. Polling of its
4002	* DEPCMD.CmdAct may block setting up TRB for Setup packet, causing a
4003	* timeout. Delay issuing the End Transfer command until the Setup TRB is
4004	* prepared.
4005	*/
4006	if (dwc->ep0state != EP0_SETUP_PHASE && !dwc->delayed_status) {
4007	dep->flags |= DWC3_EP_DELAY_STOP;
4008	return;
4009	}
4010
4011	/*
4012	* NOTICE: We are violating what the Databook says about the
4013	* EndTransfer command. Ideally we would _always_ wait for the
4014	* EndTransfer Command Completion IRQ, but that's causing too
4015	* much trouble synchronizing between us and gadget driver.
4016	*
4017	* We have discussed this with the IP Provider and it was
4018	* suggested to giveback all requests here.
4019	*
4020	* Note also that a similar handling was tested by Synopsys
4021	* (thanks a lot Paul) and nothing bad has come out of it.
4022	* In short, what we're doing is issuing EndTransfer with
4023	* CMDIOC bit set and delay kicking transfer until the
4024	* EndTransfer command had completed.
4025	*
4026	* As of IP version 3.10a of the DWC_usb3 IP, the controller
4027	* supports a mode to work around the above limitation. The
4028	* software can poll the CMDACT bit in the DEPCMD register
4029	* after issuing a EndTransfer command. This mode is enabled
4030	* by writing GUCTL2[14]. This polling is already done in the
4031	* dwc3_send_gadget_ep_cmd() function so if the mode is
4032	* enabled, the EndTransfer command will have completed upon
4033	* returning from this function.
4034	*
4035	* This mode is NOT available on the DWC_usb31 IP. In this
4036	* case, if the IOC bit is not set, then delay by 1ms
4037	* after issuing the EndTransfer command. This allows for the
4038	* controller to handle the command completely before DWC3
4039	* remove requests attempts to unmap USB request buffers.
4040	*/
4041
4042	__dwc3_stop_active_transfer(dep, force, interrupt);
4043	}
4044
4045	static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
4046	{
4047	u32 epnum;
4048
4049	for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
4050	struct dwc3_ep *dep;
4051	int ret;
4052
4053	dep = dwc->eps[epnum];
4054	if (!dep)
4055	continue;
4056
4057	if (!(dep->flags & DWC3_EP_STALL))
4058	continue;
4059
4060	dep->flags &= ~DWC3_EP_STALL;
4061
4062	ret = dwc3_send_clear_stall_ep_cmd(dep);
4063	if (ret)
4064	dev_err_ratelimited(dwc->dev,
4065	"failed to clear STALL on %s\n", dep->name);
4066	}
4067	}
4068
4069	static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
4070	{
4071	int reg;
4072
4073	dwc->suspended = false;
4074
4075	dwc3_gadget_set_link_state(dwc, state: DWC3_LINK_STATE_RX_DET);
4076
4077	reg = dwc3_readl(base: dwc->regs, DWC3_DCTL);
4078	reg &= ~DWC3_DCTL_INITU1ENA;
4079	reg &= ~DWC3_DCTL_INITU2ENA;
4080	dwc3_gadget_dctl_write_safe(dwc, value: reg);
4081
4082	dwc->connected = false;
4083
4084	dwc3_disconnect_gadget(dwc);
4085
4086	dwc->gadget->speed = USB_SPEED_UNKNOWN;
4087	dwc->setup_packet_pending = false;
4088	dwc->gadget->wakeup_armed = false;
4089	dwc3_gadget_enable_linksts_evts(dwc, set: false);
4090	usb_gadget_set_state(gadget: dwc->gadget, state: USB_STATE_NOTATTACHED);
4091
4092	dwc3_ep0_reset_state(dwc);
4093
4094	/*
4095	* Request PM idle to address condition where usage count is
4096	* already decremented to zero, but waiting for the disconnect
4097	* interrupt to set dwc->connected to FALSE.
4098	*/
4099	pm_request_idle(dev: dwc->dev);
4100	}
4101
4102	static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
4103	{
4104	u32 reg;
4105
4106	dwc->suspended = false;
4107
4108	/*
4109	* Ideally, dwc3_reset_gadget() would trigger the function
4110	* drivers to stop any active transfers through ep disable.
4111	* However, for functions which defer ep disable, such as mass
4112	* storage, we will need to rely on the call to stop active
4113	* transfers here, and avoid allowing of request queuing.
4114	*/
4115	dwc->connected = false;
4116
4117	/*
4118	* WORKAROUND: DWC3 revisions <1.88a have an issue which
4119	* would cause a missing Disconnect Event if there's a
4120	* pending Setup Packet in the FIFO.
4121	*
4122	* There's no suggested workaround on the official Bug
4123	* report, which states that "unless the driver/application
4124	* is doing any special handling of a disconnect event,
4125	* there is no functional issue".
4126	*
4127	* Unfortunately, it turns out that we _do_ some special
4128	* handling of a disconnect event, namely complete all
4129	* pending transfers, notify gadget driver of the
4130	* disconnection, and so on.
4131	*
4132	* Our suggested workaround is to follow the Disconnect
4133	* Event steps here, instead, based on a setup_packet_pending
4134	* flag. Such flag gets set whenever we have a SETUP_PENDING
4135	* status for EP0 TRBs and gets cleared on XferComplete for the
4136	* same endpoint.
4137	*
4138	* Refers to:
4139	*
4140	* STAR#9000466709: RTL: Device : Disconnect event not
4141	* generated if setup packet pending in FIFO
4142	*/
4143	if (DWC3_VER_IS_PRIOR(DWC3, 188A)) {
4144	if (dwc->setup_packet_pending)
4145	dwc3_gadget_disconnect_interrupt(dwc);
4146	}
4147
4148	dwc3_reset_gadget(dwc);
4149
4150	/*
4151	* From SNPS databook section 8.1.2, the EP0 should be in setup
4152	* phase. So ensure that EP0 is in setup phase by issuing a stall
4153	* and restart if EP0 is not in setup phase.
4154	*/
4155	dwc3_ep0_reset_state(dwc);
4156
4157	/*
4158	* In the Synopsis DesignWare Cores USB3 Databook Rev. 3.30a
4159	* Section 4.1.2 Table 4-2, it states that during a USB reset, the SW
4160	* needs to ensure that it sends "a DEPENDXFER command for any active
4161	* transfers."
4162	*/
4163	dwc3_stop_active_transfers(dwc);
4164	dwc->connected = true;
4165
4166	reg = dwc3_readl(base: dwc->regs, DWC3_DCTL);
4167	reg &= ~DWC3_DCTL_TSTCTRL_MASK;
4168	dwc3_gadget_dctl_write_safe(dwc, value: reg);
4169	dwc->test_mode = false;
4170	dwc->gadget->wakeup_armed = false;
4171	dwc3_gadget_enable_linksts_evts(dwc, set: false);
4172	dwc3_clear_stall_all_ep(dwc);
4173
4174	/* Reset device address to zero */
4175	reg = dwc3_readl(base: dwc->regs, DWC3_DCFG);
4176	reg &= ~(DWC3_DCFG_DEVADDR_MASK);
4177	dwc3_writel(base: dwc->regs, DWC3_DCFG, value: reg);
4178	}
4179
4180	static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
4181	{
4182	struct dwc3_ep *dep;
4183	int ret;
4184	u32 reg;
4185	u8 lanes = 1;
4186	u8 speed;
4187
4188	if (!dwc->softconnect)
4189	return;
4190
4191	reg = dwc3_readl(base: dwc->regs, DWC3_DSTS);
4192	speed = reg & DWC3_DSTS_CONNECTSPD;
4193	dwc->speed = speed;
4194
4195	if (DWC3_IP_IS(DWC32))
4196	lanes = DWC3_DSTS_CONNLANES(reg) + 1;
4197
4198	dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN;
4199
4200	/*
4201	* RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
4202	* each time on Connect Done.
4203	*
4204	* Currently we always use the reset value. If any platform
4205	* wants to set this to a different value, we need to add a
4206	* setting and update GCTL.RAMCLKSEL here.
4207	*/
4208
4209	switch (speed) {
4210	case DWC3_DSTS_SUPERSPEED_PLUS:
4211	dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
4212	dwc->gadget->ep0->maxpacket = 512;
4213	dwc->gadget->speed = USB_SPEED_SUPER_PLUS;
4214
4215	if (lanes > 1)
4216	dwc->gadget->ssp_rate = USB_SSP_GEN_2x2;
4217	else
4218	dwc->gadget->ssp_rate = USB_SSP_GEN_2x1;
4219	break;
4220	case DWC3_DSTS_SUPERSPEED:
4221	/*
4222	* WORKAROUND: DWC3 revisions <1.90a have an issue which
4223	* would cause a missing USB3 Reset event.
4224	*
4225	* In such situations, we should force a USB3 Reset
4226	* event by calling our dwc3_gadget_reset_interrupt()
4227	* routine.
4228	*
4229	* Refers to:
4230	*
4231	* STAR#9000483510: RTL: SS : USB3 reset event may
4232	* not be generated always when the link enters poll
4233	*/
4234	if (DWC3_VER_IS_PRIOR(DWC3, 190A))
4235	dwc3_gadget_reset_interrupt(dwc);
4236
4237	dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
4238	dwc->gadget->ep0->maxpacket = 512;
4239	dwc->gadget->speed = USB_SPEED_SUPER;
4240
4241	if (lanes > 1) {
4242	dwc->gadget->speed = USB_SPEED_SUPER_PLUS;
4243	dwc->gadget->ssp_rate = USB_SSP_GEN_1x2;
4244	}
4245	break;
4246	case DWC3_DSTS_HIGHSPEED:
4247	dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
4248	dwc->gadget->ep0->maxpacket = 64;
4249	dwc->gadget->speed = USB_SPEED_HIGH;
4250	break;
4251	case DWC3_DSTS_FULLSPEED:
4252	dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
4253	dwc->gadget->ep0->maxpacket = 64;
4254	dwc->gadget->speed = USB_SPEED_FULL;
4255	break;
4256	}
4257
4258	dwc->eps[1]->endpoint.maxpacket = dwc->gadget->ep0->maxpacket;
4259
4260	/* Enable USB2 LPM Capability */
4261
4262	if (!DWC3_VER_IS_WITHIN(DWC3, ANY, 194A) &&
4263	!dwc->usb2_gadget_lpm_disable &&
4264	(speed != DWC3_DSTS_SUPERSPEED) &&
4265	(speed != DWC3_DSTS_SUPERSPEED_PLUS)) {
4266	reg = dwc3_readl(base: dwc->regs, DWC3_DCFG);
4267	reg |= DWC3_DCFG_LPM_CAP;
4268	dwc3_writel(base: dwc->regs, DWC3_DCFG, value: reg);
4269
4270	reg = dwc3_readl(base: dwc->regs, DWC3_DCTL);
4271	reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN);
4272
4273	reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold |
4274	(dwc->is_utmi_l1_suspend << 4));
4275
4276	/*
4277	* When dwc3 revisions >= 2.40a, LPM Erratum is enabled and
4278	* DCFG.LPMCap is set, core responses with an ACK and the
4279	* BESL value in the LPM token is less than or equal to LPM
4280	* NYET threshold.
4281	*/
4282	WARN_ONCE(DWC3_VER_IS_PRIOR(DWC3, 240A) && dwc->has_lpm_erratum,
4283	"LPM Erratum not available on dwc3 revisions < 2.40a\n");
4284
4285	if (dwc->has_lpm_erratum && !DWC3_VER_IS_PRIOR(DWC3, 240A)) {
4286	reg &= ~DWC3_DCTL_NYET_THRES_MASK;
4287	reg |= DWC3_DCTL_NYET_THRES(dwc->lpm_nyet_threshold);
4288	}
4289
4290	dwc3_gadget_dctl_write_safe(dwc, value: reg);
4291	} else {
4292	if (dwc->usb2_gadget_lpm_disable) {
4293	reg = dwc3_readl(base: dwc->regs, DWC3_DCFG);
4294	reg &= ~DWC3_DCFG_LPM_CAP;
4295	dwc3_writel(base: dwc->regs, DWC3_DCFG, value: reg);
4296	}
4297
4298	reg = dwc3_readl(base: dwc->regs, DWC3_DCTL);
4299	reg &= ~DWC3_DCTL_HIRD_THRES_MASK;
4300	dwc3_gadget_dctl_write_safe(dwc, value: reg);
4301	}
4302
4303	dep = dwc->eps[0];
4304	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
4305	if (ret) {
4306	dev_err(dwc->dev, "failed to enable %s\n", dep->name);
4307	return;
4308	}
4309
4310	dep = dwc->eps[1];
4311	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
4312	if (ret) {
4313	dev_err(dwc->dev, "failed to enable %s\n", dep->name);
4314	return;
4315	}
4316
4317	/*
4318	* Configure PHY via GUSB3PIPECTLn if required.
4319	*
4320	* Update GTXFIFOSIZn
4321	*
4322	* In both cases reset values should be sufficient.
4323	*/
4324	}
4325
4326	static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc, unsigned int evtinfo)
4327	{
4328	dwc->suspended = false;
4329
4330	/*
4331	* TODO take core out of low power mode when that's
4332	* implemented.
4333	*/
4334
4335	if (dwc->async_callbacks && dwc->gadget_driver->resume) {
4336	spin_unlock(lock: &dwc->lock);
4337	dwc->gadget_driver->resume(dwc->gadget);
4338	spin_lock(lock: &dwc->lock);
4339	}
4340
4341	dwc->link_state = evtinfo & DWC3_LINK_STATE_MASK;
4342	}
4343
4344	static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
4345	unsigned int evtinfo)
4346	{
4347	enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
4348	unsigned int pwropt;
4349	int ret;
4350	int intf_id;
4351
4352	/*
4353	* WORKAROUND: DWC3 < 2.50a have an issue when configured without
4354	* Hibernation mode enabled which would show up when device detects
4355	* host-initiated U3 exit.
4356	*
4357	* In that case, device will generate a Link State Change Interrupt
4358	* from U3 to RESUME which is only necessary if Hibernation is
4359	* configured in.
4360	*
4361	* There are no functional changes due to such spurious event and we
4362	* just need to ignore it.
4363	*
4364	* Refers to:
4365	*
4366	* STAR#9000570034 RTL: SS Resume event generated in non-Hibernation
4367	* operational mode
4368	*/
4369	pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
4370	if (DWC3_VER_IS_PRIOR(DWC3, 250A) &&
4371	(pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
4372	if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
4373	(next == DWC3_LINK_STATE_RESUME)) {
4374	return;
4375	}
4376	}
4377
4378	/*
4379	* WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending
4380	* on the link partner, the USB session might do multiple entry/exit
4381	* of low power states before a transfer takes place.
4382	*
4383	* Due to this problem, we might experience lower throughput. The
4384	* suggested workaround is to disable DCTL[12:9] bits if we're
4385	* transitioning from U1/U2 to U0 and enable those bits again
4386	* after a transfer completes and there are no pending transfers
4387	* on any of the enabled endpoints.
4388	*
4389	* This is the first half of that workaround.
4390	*
4391	* Refers to:
4392	*
4393	* STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us
4394	* core send LGO_Ux entering U0
4395	*/
4396	if (DWC3_VER_IS_PRIOR(DWC3, 183A)) {
4397	if (next == DWC3_LINK_STATE_U0) {
4398	u32 u1u2;
4399	u32 reg;
4400
4401	switch (dwc->link_state) {
4402	case DWC3_LINK_STATE_U1:
4403	case DWC3_LINK_STATE_U2:
4404	reg = dwc3_readl(base: dwc->regs, DWC3_DCTL);
4405	u1u2 = reg & (DWC3_DCTL_INITU2ENA
4406	| DWC3_DCTL_ACCEPTU2ENA
4407	| DWC3_DCTL_INITU1ENA
4408	| DWC3_DCTL_ACCEPTU1ENA);
4409
4410	if (!dwc->u1u2)
4411	dwc->u1u2 = reg & u1u2;
4412
4413	reg &= ~u1u2;
4414
4415	dwc3_gadget_dctl_write_safe(dwc, value: reg);
4416	break;
4417	default:
4418	/* do nothing */
4419	break;
4420	}
4421	}
4422	}
4423
4424	switch (next) {
4425	case DWC3_LINK_STATE_U0:
4426	if (dwc->gadget->wakeup_armed || dwc->wakeup_pending_funcs) {
4427	dwc3_gadget_enable_linksts_evts(dwc, set: false);
4428	dwc3_resume_gadget(dwc);
4429	dwc->suspended = false;
4430	}
4431	break;
4432	case DWC3_LINK_STATE_U1:
4433	if (dwc->speed == USB_SPEED_SUPER)
4434	dwc3_suspend_gadget(dwc);
4435	break;
4436	case DWC3_LINK_STATE_U2:
4437	case DWC3_LINK_STATE_U3:
4438	dwc3_suspend_gadget(dwc);
4439	break;
4440	case DWC3_LINK_STATE_RESUME:
4441	dwc3_resume_gadget(dwc);
4442	break;
4443	default:
4444	/* do nothing */
4445	break;
4446	}
4447
4448	dwc->link_state = next;
4449
4450	/* Proceed with func wakeup if any interfaces that has requested */
4451	while (dwc->wakeup_pending_funcs && (next == DWC3_LINK_STATE_U0)) {
4452	intf_id = ffs(dwc->wakeup_pending_funcs) - 1;
4453	ret = dwc3_send_gadget_generic_command(dwc, DWC3_DGCMD_DEV_NOTIFICATION,
4454	DWC3_DGCMDPAR_DN_FUNC_WAKE |
4455	DWC3_DGCMDPAR_INTF_SEL(intf_id));
4456	if (ret)
4457	dev_err(dwc->dev, "Failed to send DN wake for intf %d\n", intf_id);
4458
4459	dwc->wakeup_pending_funcs &= ~BIT(intf_id);
4460	}
4461	}
4462
4463	static void dwc3_gadget_suspend_interrupt(struct dwc3 *dwc,
4464	unsigned int evtinfo)
4465	{
4466	enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
4467
4468	if (!dwc->suspended && next == DWC3_LINK_STATE_U3) {
4469	dwc->suspended = true;
4470	dwc3_suspend_gadget(dwc);
4471	}
4472
4473	dwc->link_state = next;
4474	}
4475
4476	static void dwc3_gadget_interrupt(struct dwc3 *dwc,
4477	const struct dwc3_event_devt *event)
4478	{
4479	switch (event->type) {
4480	case DWC3_DEVICE_EVENT_DISCONNECT:
4481	dwc3_gadget_disconnect_interrupt(dwc);
4482	break;
4483	case DWC3_DEVICE_EVENT_RESET:
4484	dwc3_gadget_reset_interrupt(dwc);
4485	break;
4486	case DWC3_DEVICE_EVENT_CONNECT_DONE:
4487	dwc3_gadget_conndone_interrupt(dwc);
4488	break;
4489	case DWC3_DEVICE_EVENT_WAKEUP:
4490	dwc3_gadget_wakeup_interrupt(dwc, evtinfo: event->event_info);
4491	break;
4492	case DWC3_DEVICE_EVENT_HIBER_REQ:
4493	dev_WARN_ONCE(dwc->dev, true, "unexpected hibernation event\n");
4494	break;
4495	case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
4496	dwc3_gadget_linksts_change_interrupt(dwc, evtinfo: event->event_info);
4497	break;
4498	case DWC3_DEVICE_EVENT_SUSPEND:
4499	/* It changed to be suspend event for version 2.30a and above */
4500	if (!DWC3_VER_IS_PRIOR(DWC3, 230A))
4501	dwc3_gadget_suspend_interrupt(dwc, evtinfo: event->event_info);
4502	break;
4503	case DWC3_DEVICE_EVENT_SOF:
4504	case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
4505	case DWC3_DEVICE_EVENT_CMD_CMPL:
4506	case DWC3_DEVICE_EVENT_OVERFLOW:
4507	break;
4508	default:
4509	dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
4510	}
4511	}
4512
4513	static void dwc3_process_event_entry(struct dwc3 *dwc,
4514	const union dwc3_event *event)
4515	{
4516	trace_dwc3_event(event: event->raw, dwc);
4517
4518	if (!event->type.is_devspec)
4519	dwc3_endpoint_interrupt(dwc, event: &event->depevt);
4520	else if (event->type.type == DWC3_EVENT_TYPE_DEV)
4521	dwc3_gadget_interrupt(dwc, event: &event->devt);
4522	else
4523	dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
4524	}
4525
4526	static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt)
4527	{
4528	struct dwc3 *dwc = evt->dwc;
4529	irqreturn_t ret = IRQ_NONE;
4530	int left;
4531
4532	left = evt->count;
4533
4534	if (!(evt->flags & DWC3_EVENT_PENDING))
4535	return IRQ_NONE;
4536
4537	while (left > 0) {
4538	union dwc3_event event;
4539
4540	event.raw = *(u32 *) (evt->cache + evt->lpos);
4541
4542	dwc3_process_event_entry(dwc, event: &event);
4543
4544	/*
4545	* FIXME we wrap around correctly to the next entry as
4546	* almost all entries are 4 bytes in size. There is one
4547	* entry which has 12 bytes which is a regular entry
4548	* followed by 8 bytes data. ATM I don't know how
4549	* things are organized if we get next to the a
4550	* boundary so I worry about that once we try to handle
4551	* that.
4552	*/
4553	evt->lpos = (evt->lpos + 4) % evt->length;
4554	left -= 4;
4555	}
4556
4557	evt->count = 0;
4558	ret = IRQ_HANDLED;
4559
4560	/* Unmask interrupt */
4561	dwc3_writel(base: dwc->regs, DWC3_GEVNTSIZ(0),
4562	DWC3_GEVNTSIZ_SIZE(evt->length));
4563
4564	evt->flags &= ~DWC3_EVENT_PENDING;
4565	/*
4566	* Add an explicit write memory barrier to make sure that the update of
4567	* clearing DWC3_EVENT_PENDING is observed in dwc3_check_event_buf()
4568	*/
4569	wmb();
4570
4571	if (dwc->imod_interval) {
4572	dwc3_writel(base: dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
4573	dwc3_writel(base: dwc->regs, DWC3_DEV_IMOD(0), value: dwc->imod_interval);
4574	}
4575
4576	return ret;
4577	}
4578
4579	static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt)
4580	{
4581	struct dwc3_event_buffer *evt = _evt;
4582	struct dwc3 *dwc = evt->dwc;
4583	unsigned long flags;
4584	irqreturn_t ret = IRQ_NONE;
4585
4586	local_bh_disable();
4587	spin_lock_irqsave(&dwc->lock, flags);
4588	ret = dwc3_process_event_buf(evt);
4589	spin_unlock_irqrestore(lock: &dwc->lock, flags);
4590	local_bh_enable();
4591
4592	return ret;
4593	}
4594
4595	static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt)
4596	{
4597	struct dwc3 *dwc = evt->dwc;
4598	u32 amount;
4599	u32 count;
4600
4601	if (pm_runtime_suspended(dev: dwc->dev)) {
4602	dwc->pending_events = true;
4603	/*
4604	* Trigger runtime resume. The get() function will be balanced
4605	* after processing the pending events in dwc3_process_pending
4606	* events().
4607	*/
4608	pm_runtime_get(dev: dwc->dev);
4609	disable_irq_nosync(irq: dwc->irq_gadget);
4610	return IRQ_HANDLED;
4611	}
4612
4613	/*
4614	* With PCIe legacy interrupt, test shows that top-half irq handler can
4615	* be called again after HW interrupt deassertion. Check if bottom-half
4616	* irq event handler completes before caching new event to prevent
4617	* losing events.
4618	*/
4619	if (evt->flags & DWC3_EVENT_PENDING)
4620	return IRQ_HANDLED;
4621
4622	count = dwc3_readl(base: dwc->regs, DWC3_GEVNTCOUNT(0));
4623	count &= DWC3_GEVNTCOUNT_MASK;
4624	if (!count)
4625	return IRQ_NONE;
4626
4627	if (count > evt->length) {
4628	dev_err_ratelimited(dwc->dev, "invalid count(%u) > evt->length(%u)\n",
4629	count, evt->length);
4630	return IRQ_NONE;
4631	}
4632
4633	evt->count = count;
4634	evt->flags |= DWC3_EVENT_PENDING;
4635
4636	/* Mask interrupt */
4637	dwc3_writel(base: dwc->regs, DWC3_GEVNTSIZ(0),
4638	DWC3_GEVNTSIZ_INTMASK | DWC3_GEVNTSIZ_SIZE(evt->length));
4639
4640	amount = min(count, evt->length - evt->lpos);
4641	memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount);
4642
4643	if (amount < count)
4644	memcpy(evt->cache, evt->buf, count - amount);
4645
4646	dwc3_writel(base: dwc->regs, DWC3_GEVNTCOUNT(0), value: count);
4647
4648	return IRQ_WAKE_THREAD;
4649	}
4650
4651	static irqreturn_t dwc3_interrupt(int irq, void *_evt)
4652	{
4653	struct dwc3_event_buffer *evt = _evt;
4654
4655	return dwc3_check_event_buf(evt);
4656	}
4657
4658	static int dwc3_gadget_get_irq(struct dwc3 *dwc)
4659	{
4660	struct platform_device *dwc3_pdev = to_platform_device(dwc->dev);
4661	int irq;
4662
4663	irq = platform_get_irq_byname_optional(dev: dwc3_pdev, name: "peripheral");
4664	if (irq > 0)
4665	goto out;
4666
4667	if (irq == -EPROBE_DEFER)
4668	goto out;
4669
4670	irq = platform_get_irq_byname_optional(dev: dwc3_pdev, name: "dwc_usb3");
4671	if (irq > 0)
4672	goto out;
4673
4674	if (irq == -EPROBE_DEFER)
4675	goto out;
4676
4677	irq = platform_get_irq(dwc3_pdev, 0);
4678
4679	out:
4680	return irq;
4681	}
4682
4683	static void dwc_gadget_release(struct device *dev)
4684	{
4685	struct usb_gadget *gadget = container_of(dev, struct usb_gadget, dev);
4686
4687	kfree(objp: gadget);
4688	}
4689
4690	/**
4691	* dwc3_gadget_init - initializes gadget related registers
4692	* @dwc: pointer to our controller context structure
4693	*
4694	* Returns 0 on success otherwise negative errno.
4695	*/
4696	int dwc3_gadget_init(struct dwc3 *dwc)
4697	{
4698	int ret;
4699	int irq;
4700	struct device *dev;
4701
4702	irq = dwc3_gadget_get_irq(dwc);
4703	if (irq < 0) {
4704	ret = irq;
4705	goto err0;
4706	}
4707
4708	dwc->irq_gadget = irq;
4709
4710	dwc->ep0_trb = dma_alloc_coherent(dev: dwc->sysdev,
4711	size: sizeof(*dwc->ep0_trb) * 2,
4712	dma_handle: &dwc->ep0_trb_addr, GFP_KERNEL);
4713	if (!dwc->ep0_trb) {
4714	dev_err(dwc->dev, "failed to allocate ep0 trb\n");
4715	ret = -ENOMEM;
4716	goto err0;
4717	}
4718
4719	dwc->setup_buf = kzalloc(DWC3_EP0_SETUP_SIZE, GFP_KERNEL);
4720	if (!dwc->setup_buf) {
4721	ret = -ENOMEM;
4722	goto err1;
4723	}
4724
4725	dwc->bounce = dma_alloc_coherent(dev: dwc->sysdev, DWC3_BOUNCE_SIZE,
4726	dma_handle: &dwc->bounce_addr, GFP_KERNEL);
4727	if (!dwc->bounce) {
4728	ret = -ENOMEM;
4729	goto err2;
4730	}
4731
4732	init_completion(x: &dwc->ep0_in_setup);
4733	dwc->gadget = kzalloc(sizeof(struct usb_gadget), GFP_KERNEL);
4734	if (!dwc->gadget) {
4735	ret = -ENOMEM;
4736	goto err3;
4737	}
4738
4739
4740	usb_initialize_gadget(parent: dwc->dev, gadget: dwc->gadget, release: dwc_gadget_release);
4741	dev = &dwc->gadget->dev;
4742	dev->platform_data = dwc;
4743	dwc->gadget->ops = &dwc3_gadget_ops;
4744	dwc->gadget->speed = USB_SPEED_UNKNOWN;
4745	dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN;
4746	dwc->gadget->sg_supported = true;
4747	dwc->gadget->name = "dwc3-gadget";
4748	dwc->gadget->lpm_capable = !dwc->usb2_gadget_lpm_disable;
4749	dwc->gadget->wakeup_capable = true;
4750
4751	/*
4752	* FIXME We might be setting max_speed to <SUPER, however versions
4753	* <2.20a of dwc3 have an issue with metastability (documented
4754	* elsewhere in this driver) which tells us we can't set max speed to
4755	* anything lower than SUPER.
4756	*
4757	* Because gadget.max_speed is only used by composite.c and function
4758	* drivers (i.e. it won't go into dwc3's registers) we are allowing this
4759	* to happen so we avoid sending SuperSpeed Capability descriptor
4760	* together with our BOS descriptor as that could confuse host into
4761	* thinking we can handle super speed.
4762	*
4763	* Note that, in fact, we won't even support GetBOS requests when speed
4764	* is less than super speed because we don't have means, yet, to tell
4765	* composite.c that we are USB 2.0 + LPM ECN.
4766	*/
4767	if (DWC3_VER_IS_PRIOR(DWC3, 220A) &&
4768	!dwc->dis_metastability_quirk)
4769	dev_info(dwc->dev, "changing max_speed on rev %08x\n",
4770	dwc->revision);
4771
4772	dwc->gadget->max_speed = dwc->maximum_speed;
4773	dwc->gadget->max_ssp_rate = dwc->max_ssp_rate;
4774
4775	/*
4776	* REVISIT: Here we should clear all pending IRQs to be
4777	* sure we're starting from a well known location.
4778	*/
4779
4780	ret = dwc3_gadget_init_endpoints(dwc, total: dwc->num_eps);
4781	if (ret)
4782	goto err4;
4783
4784	ret = usb_add_gadget(gadget: dwc->gadget);
4785	if (ret) {
4786	dev_err(dwc->dev, "failed to add gadget\n");
4787	goto err5;
4788	}
4789
4790	if (DWC3_IP_IS(DWC32) && dwc->maximum_speed == USB_SPEED_SUPER_PLUS)
4791	dwc3_gadget_set_ssp_rate(g: dwc->gadget, rate: dwc->max_ssp_rate);
4792	else
4793	dwc3_gadget_set_speed(g: dwc->gadget, speed: dwc->maximum_speed);
4794
4795	/* No system wakeup if no gadget driver bound */
4796	if (dwc->sys_wakeup)
4797	device_wakeup_disable(dev: dwc->sysdev);
4798
4799	return 0;
4800
4801	err5:
4802	dwc3_gadget_free_endpoints(dwc);
4803	err4:
4804	usb_put_gadget(gadget: dwc->gadget);
4805	dwc->gadget = NULL;
4806	err3:
4807	dma_free_coherent(dev: dwc->sysdev, DWC3_BOUNCE_SIZE, cpu_addr: dwc->bounce,
4808	dma_handle: dwc->bounce_addr);
4809
4810	err2:
4811	kfree(objp: dwc->setup_buf);
4812
4813	err1:
4814	dma_free_coherent(dev: dwc->sysdev, size: sizeof(*dwc->ep0_trb) * 2,
4815	cpu_addr: dwc->ep0_trb, dma_handle: dwc->ep0_trb_addr);
4816
4817	err0:
4818	return ret;
4819	}
4820	EXPORT_SYMBOL_GPL(dwc3_gadget_init);
4821
4822	/* -------------------------------------------------------------------------- */
4823
4824	void dwc3_gadget_exit(struct dwc3 *dwc)
4825	{
4826	if (!dwc->gadget)
4827	return;
4828
4829	dwc3_enable_susphy(dwc, enable: true);
4830	usb_del_gadget(gadget: dwc->gadget);
4831	dwc3_gadget_free_endpoints(dwc);
4832	usb_put_gadget(gadget: dwc->gadget);
4833	dma_free_coherent(dev: dwc->sysdev, DWC3_BOUNCE_SIZE, cpu_addr: dwc->bounce,
4834	dma_handle: dwc->bounce_addr);
4835	kfree(objp: dwc->setup_buf);
4836	dma_free_coherent(dev: dwc->sysdev, size: sizeof(*dwc->ep0_trb) * 2,
4837	cpu_addr: dwc->ep0_trb, dma_handle: dwc->ep0_trb_addr);
4838	}
4839	EXPORT_SYMBOL_GPL(dwc3_gadget_exit);
4840
4841	int dwc3_gadget_suspend(struct dwc3 *dwc)
4842	{
4843	unsigned long flags;
4844	int ret;
4845
4846	ret = dwc3_gadget_soft_disconnect(dwc);
4847	/*
4848	* Attempt to reset the controller's state. Likely no
4849	* communication can be established until the host
4850	* performs a port reset.
4851	*/
4852	if (ret && dwc->softconnect) {
4853	dwc3_gadget_soft_connect(dwc);
4854	return -EAGAIN;
4855	}
4856
4857	spin_lock_irqsave(&dwc->lock, flags);
4858	if (dwc->gadget_driver)
4859	dwc3_disconnect_gadget(dwc);
4860	spin_unlock_irqrestore(lock: &dwc->lock, flags);
4861
4862	return 0;
4863	}
4864
4865	int dwc3_gadget_resume(struct dwc3 *dwc)
4866	{
4867	if (!dwc->gadget_driver || !dwc->softconnect)
4868	return 0;
4869
4870	return dwc3_gadget_soft_connect(dwc);
4871	}
4872