1	// SPDX-License-Identifier: GPL-2.0+
2	/*
3	* NVIDIA Tegra XUSB device mode controller
4	*
5	* Copyright (c) 2013-2022, NVIDIA CORPORATION. All rights reserved.
6	* Copyright (c) 2015, Google Inc.
7	*/
8
9	#include <linux/clk.h>
10	#include <linux/completion.h>
11	#include <linux/delay.h>
12	#include <linux/dma-mapping.h>
13	#include <linux/dmapool.h>
14	#include <linux/interrupt.h>
15	#include <linux/iopoll.h>
16	#include <linux/kernel.h>
17	#include <linux/module.h>
18	#include <linux/of.h>
19	#include <linux/phy/phy.h>
20	#include <linux/phy/tegra/xusb.h>
21	#include <linux/pm_domain.h>
22	#include <linux/platform_device.h>
23	#include <linux/pm_runtime.h>
24	#include <linux/regulator/consumer.h>
25	#include <linux/reset.h>
26	#include <linux/usb/ch9.h>
27	#include <linux/usb/gadget.h>
28	#include <linux/usb/otg.h>
29	#include <linux/usb/role.h>
30	#include <linux/usb/phy.h>
31	#include <linux/workqueue.h>
32
33	/* XUSB_DEV registers */
34	#define DB 0x004
35	#define DB_TARGET_MASK GENMASK(15, 8)
36	#define DB_TARGET(x) (((x) << 8) & DB_TARGET_MASK)
37	#define DB_STREAMID_MASK GENMASK(31, 16)
38	#define DB_STREAMID(x) (((x) << 16) & DB_STREAMID_MASK)
39	#define ERSTSZ 0x008
40	#define ERSTSZ_ERSTXSZ_SHIFT(x) ((x) * 16)
41	#define ERSTSZ_ERSTXSZ_MASK GENMASK(15, 0)
42	#define ERSTXBALO(x) (0x010 + 8 * (x))
43	#define ERSTXBAHI(x) (0x014 + 8 * (x))
44	#define ERDPLO 0x020
45	#define ERDPLO_EHB BIT(3)
46	#define ERDPHI 0x024
47	#define EREPLO 0x028
48	#define EREPLO_ECS BIT(0)
49	#define EREPLO_SEGI BIT(1)
50	#define EREPHI 0x02c
51	#define CTRL 0x030
52	#define CTRL_RUN BIT(0)
53	#define CTRL_LSE BIT(1)
54	#define CTRL_IE BIT(4)
55	#define CTRL_SMI_EVT BIT(5)
56	#define CTRL_SMI_DSE BIT(6)
57	#define CTRL_EWE BIT(7)
58	#define CTRL_DEVADDR_MASK GENMASK(30, 24)
59	#define CTRL_DEVADDR(x) (((x) << 24) & CTRL_DEVADDR_MASK)
60	#define CTRL_ENABLE BIT(31)
61	#define ST 0x034
62	#define ST_RC BIT(0)
63	#define ST_IP BIT(4)
64	#define RT_IMOD 0x038
65	#define RT_IMOD_IMODI_MASK GENMASK(15, 0)
66	#define RT_IMOD_IMODI(x) ((x) & RT_IMOD_IMODI_MASK)
67	#define RT_IMOD_IMODC_MASK GENMASK(31, 16)
68	#define RT_IMOD_IMODC(x) (((x) << 16) & RT_IMOD_IMODC_MASK)
69	#define PORTSC 0x03c
70	#define PORTSC_CCS BIT(0)
71	#define PORTSC_PED BIT(1)
72	#define PORTSC_PR BIT(4)
73	#define PORTSC_PLS_SHIFT 5
74	#define PORTSC_PLS_MASK GENMASK(8, 5)
75	#define PORTSC_PLS_U0 0x0
76	#define PORTSC_PLS_U2 0x2
77	#define PORTSC_PLS_U3 0x3
78	#define PORTSC_PLS_DISABLED 0x4
79	#define PORTSC_PLS_RXDETECT 0x5
80	#define PORTSC_PLS_INACTIVE 0x6
81	#define PORTSC_PLS_RESUME 0xf
82	#define PORTSC_PLS(x) (((x) << PORTSC_PLS_SHIFT) & PORTSC_PLS_MASK)
83	#define PORTSC_PS_SHIFT 10
84	#define PORTSC_PS_MASK GENMASK(13, 10)
85	#define PORTSC_PS_UNDEFINED 0x0
86	#define PORTSC_PS_FS 0x1
87	#define PORTSC_PS_LS 0x2
88	#define PORTSC_PS_HS 0x3
89	#define PORTSC_PS_SS 0x4
90	#define PORTSC_LWS BIT(16)
91	#define PORTSC_CSC BIT(17)
92	#define PORTSC_WRC BIT(19)
93	#define PORTSC_PRC BIT(21)
94	#define PORTSC_PLC BIT(22)
95	#define PORTSC_CEC BIT(23)
96	#define PORTSC_WPR BIT(30)
97	#define PORTSC_CHANGE_MASK (PORTSC_CSC | PORTSC_WRC | PORTSC_PRC | \
98	PORTSC_PLC | PORTSC_CEC)
99	#define ECPLO 0x040
100	#define ECPHI 0x044
101	#define MFINDEX 0x048
102	#define MFINDEX_FRAME_SHIFT 3
103	#define MFINDEX_FRAME_MASK GENMASK(13, 3)
104	#define PORTPM 0x04c
105	#define PORTPM_L1S_MASK GENMASK(1, 0)
106	#define PORTPM_L1S_DROP 0x0
107	#define PORTPM_L1S_ACCEPT 0x1
108	#define PORTPM_L1S_NYET 0x2
109	#define PORTPM_L1S_STALL 0x3
110	#define PORTPM_L1S(x) ((x) & PORTPM_L1S_MASK)
111	#define PORTPM_RWE BIT(3)
112	#define PORTPM_U2TIMEOUT_MASK GENMASK(15, 8)
113	#define PORTPM_U1TIMEOUT_MASK GENMASK(23, 16)
114	#define PORTPM_FLA BIT(24)
115	#define PORTPM_VBA BIT(25)
116	#define PORTPM_WOC BIT(26)
117	#define PORTPM_WOD BIT(27)
118	#define PORTPM_U1E BIT(28)
119	#define PORTPM_U2E BIT(29)
120	#define PORTPM_FRWE BIT(30)
121	#define PORTPM_PNG_CYA BIT(31)
122	#define EP_HALT 0x050
123	#define EP_PAUSE 0x054
124	#define EP_RELOAD 0x058
125	#define EP_STCHG 0x05c
126	#define DEVNOTIF_LO 0x064
127	#define DEVNOTIF_LO_TRIG BIT(0)
128	#define DEVNOTIF_LO_TYPE_MASK GENMASK(7, 4)
129	#define DEVNOTIF_LO_TYPE(x) (((x) << 4) & DEVNOTIF_LO_TYPE_MASK)
130	#define DEVNOTIF_LO_TYPE_FUNCTION_WAKE 0x1
131	#define DEVNOTIF_HI 0x068
132	#define PORTHALT 0x06c
133	#define PORTHALT_HALT_LTSSM BIT(0)
134	#define PORTHALT_HALT_REJECT BIT(1)
135	#define PORTHALT_STCHG_REQ BIT(20)
136	#define PORTHALT_STCHG_INTR_EN BIT(24)
137	#define PORT_TM 0x070
138	#define EP_THREAD_ACTIVE 0x074
139	#define EP_STOPPED 0x078
140	#define HSFSPI_COUNT0 0x100
141	#define HSFSPI_COUNT13 0x134
142	#define HSFSPI_COUNT13_U2_RESUME_K_DURATION_MASK GENMASK(29, 0)
143	#define HSFSPI_COUNT13_U2_RESUME_K_DURATION(x) ((x) & \
144	HSFSPI_COUNT13_U2_RESUME_K_DURATION_MASK)
145	#define BLCG 0x840
146	#define SSPX_CORE_CNT0 0x610
147	#define SSPX_CORE_CNT0_PING_TBURST_MASK GENMASK(7, 0)
148	#define SSPX_CORE_CNT0_PING_TBURST(x) ((x) & SSPX_CORE_CNT0_PING_TBURST_MASK)
149	#define SSPX_CORE_CNT30 0x688
150	#define SSPX_CORE_CNT30_LMPITP_TIMER_MASK GENMASK(19, 0)
151	#define SSPX_CORE_CNT30_LMPITP_TIMER(x) ((x) & \
152	SSPX_CORE_CNT30_LMPITP_TIMER_MASK)
153	#define SSPX_CORE_CNT32 0x690
154	#define SSPX_CORE_CNT32_POLL_TBURST_MAX_MASK GENMASK(7, 0)
155	#define SSPX_CORE_CNT32_POLL_TBURST_MAX(x) ((x) & \
156	SSPX_CORE_CNT32_POLL_TBURST_MAX_MASK)
157	#define SSPX_CORE_CNT56 0x6fc
158	#define SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX_MASK GENMASK(19, 0)
159	#define SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX(x) ((x) & \
160	SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX_MASK)
161	#define SSPX_CORE_CNT57 0x700
162	#define SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX_MASK GENMASK(19, 0)
163	#define SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX(x) ((x) & \
164	SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX_MASK)
165	#define SSPX_CORE_CNT65 0x720
166	#define SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID_MASK GENMASK(19, 0)
167	#define SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID(x) ((x) & \
168	SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID_MASK)
169	#define SSPX_CORE_CNT66 0x724
170	#define SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID_MASK GENMASK(19, 0)
171	#define SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID(x) ((x) & \
172	SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID_MASK)
173	#define SSPX_CORE_CNT67 0x728
174	#define SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID_MASK GENMASK(19, 0)
175	#define SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID(x) ((x) & \
176	SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID_MASK)
177	#define SSPX_CORE_CNT72 0x73c
178	#define SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT_MASK GENMASK(19, 0)
179	#define SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT(x) ((x) & \
180	SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT_MASK)
181	#define SSPX_CORE_PADCTL4 0x750
182	#define SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3_MASK GENMASK(19, 0)
183	#define SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3(x) ((x) & \
184	SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3_MASK)
185	#define BLCG_DFPCI BIT(0)
186	#define BLCG_UFPCI BIT(1)
187	#define BLCG_FE BIT(2)
188	#define BLCG_COREPLL_PWRDN BIT(8)
189	#define BLCG_IOPLL_0_PWRDN BIT(9)
190	#define BLCG_IOPLL_1_PWRDN BIT(10)
191	#define BLCG_IOPLL_2_PWRDN BIT(11)
192	#define BLCG_ALL 0x1ff
193	#define CFG_DEV_SSPI_XFER 0x858
194	#define CFG_DEV_SSPI_XFER_ACKTIMEOUT_MASK GENMASK(31, 0)
195	#define CFG_DEV_SSPI_XFER_ACKTIMEOUT(x) ((x) & \
196	CFG_DEV_SSPI_XFER_ACKTIMEOUT_MASK)
197	#define CFG_DEV_FE 0x85c
198	#define CFG_DEV_FE_PORTREGSEL_MASK GENMASK(1, 0)
199	#define CFG_DEV_FE_PORTREGSEL_SS_PI 1
200	#define CFG_DEV_FE_PORTREGSEL_HSFS_PI 2
201	#define CFG_DEV_FE_PORTREGSEL(x) ((x) & CFG_DEV_FE_PORTREGSEL_MASK)
202	#define CFG_DEV_FE_INFINITE_SS_RETRY BIT(29)
203
204	/* FPCI registers */
205	#define XUSB_DEV_CFG_1 0x004
206	#define XUSB_DEV_CFG_1_IO_SPACE_EN BIT(0)
207	#define XUSB_DEV_CFG_1_MEMORY_SPACE_EN BIT(1)
208	#define XUSB_DEV_CFG_1_BUS_MASTER_EN BIT(2)
209	#define XUSB_DEV_CFG_4 0x010
210	#define XUSB_DEV_CFG_4_BASE_ADDR_MASK GENMASK(31, 15)
211	#define XUSB_DEV_CFG_5 0x014
212
213	/* IPFS registers */
214	#define XUSB_DEV_CONFIGURATION_0 0x180
215	#define XUSB_DEV_CONFIGURATION_0_EN_FPCI BIT(0)
216	#define XUSB_DEV_INTR_MASK_0 0x188
217	#define XUSB_DEV_INTR_MASK_0_IP_INT_MASK BIT(16)
218
219	struct tegra_xudc_ep_context {
220	__le32 info0;
221	__le32 info1;
222	__le32 deq_lo;
223	__le32 deq_hi;
224	__le32 tx_info;
225	__le32 rsvd[11];
226	};
227
228	#define EP_STATE_DISABLED 0
229	#define EP_STATE_RUNNING 1
230	#define EP_STATE_HALTED 2
231	#define EP_STATE_STOPPED 3
232	#define EP_STATE_ERROR 4
233
234	#define EP_TYPE_INVALID 0
235	#define EP_TYPE_ISOCH_OUT 1
236	#define EP_TYPE_BULK_OUT 2
237	#define EP_TYPE_INTERRUPT_OUT 3
238	#define EP_TYPE_CONTROL 4
239	#define EP_TYPE_ISCOH_IN 5
240	#define EP_TYPE_BULK_IN 6
241	#define EP_TYPE_INTERRUPT_IN 7
242
243	#define BUILD_EP_CONTEXT_RW(name, member, shift, mask) \
244	static inline u32 ep_ctx_read_##name(struct tegra_xudc_ep_context *ctx) \
245	{ \
246	return (le32_to_cpu(ctx->member) >> (shift)) & (mask); \
247	} \
248	static inline void \
249	ep_ctx_write_##name(struct tegra_xudc_ep_context *ctx, u32 val) \
250	{ \
251	u32 tmp; \
252	\
253	tmp = le32_to_cpu(ctx->member) & ~((mask) << (shift)); \
254	tmp |= (val & (mask)) << (shift); \
255	ctx->member = cpu_to_le32(tmp); \
256	}
257
258	BUILD_EP_CONTEXT_RW(state, info0, 0, 0x7)
259	BUILD_EP_CONTEXT_RW(mult, info0, 8, 0x3)
260	BUILD_EP_CONTEXT_RW(max_pstreams, info0, 10, 0x1f)
261	BUILD_EP_CONTEXT_RW(lsa, info0, 15, 0x1)
262	BUILD_EP_CONTEXT_RW(interval, info0, 16, 0xff)
263	BUILD_EP_CONTEXT_RW(cerr, info1, 1, 0x3)
264	BUILD_EP_CONTEXT_RW(type, info1, 3, 0x7)
265	BUILD_EP_CONTEXT_RW(hid, info1, 7, 0x1)
266	BUILD_EP_CONTEXT_RW(max_burst_size, info1, 8, 0xff)
267	BUILD_EP_CONTEXT_RW(max_packet_size, info1, 16, 0xffff)
268	BUILD_EP_CONTEXT_RW(dcs, deq_lo, 0, 0x1)
269	BUILD_EP_CONTEXT_RW(deq_lo, deq_lo, 4, 0xfffffff)
270	BUILD_EP_CONTEXT_RW(deq_hi, deq_hi, 0, 0xffffffff)
271	BUILD_EP_CONTEXT_RW(avg_trb_len, tx_info, 0, 0xffff)
272	BUILD_EP_CONTEXT_RW(max_esit_payload, tx_info, 16, 0xffff)
273	BUILD_EP_CONTEXT_RW(edtla, rsvd[0], 0, 0xffffff)
274	BUILD_EP_CONTEXT_RW(rsvd, rsvd[0], 24, 0x1)
275	BUILD_EP_CONTEXT_RW(partial_td, rsvd[0], 25, 0x1)
276	BUILD_EP_CONTEXT_RW(splitxstate, rsvd[0], 26, 0x1)
277	BUILD_EP_CONTEXT_RW(seq_num, rsvd[0], 27, 0x1f)
278	BUILD_EP_CONTEXT_RW(cerrcnt, rsvd[1], 18, 0x3)
279	BUILD_EP_CONTEXT_RW(data_offset, rsvd[2], 0, 0x1ffff)
280	BUILD_EP_CONTEXT_RW(numtrbs, rsvd[2], 22, 0x1f)
281	BUILD_EP_CONTEXT_RW(devaddr, rsvd[6], 0, 0x7f)
282
283	static inline u64 ep_ctx_read_deq_ptr(struct tegra_xudc_ep_context *ctx)
284	{
285	return ((u64)ep_ctx_read_deq_hi(ctx) << 32) |
286	(ep_ctx_read_deq_lo(ctx) << 4);
287	}
288
289	static inline void
290	ep_ctx_write_deq_ptr(struct tegra_xudc_ep_context *ctx, u64 addr)
291	{
292	ep_ctx_write_deq_lo(ctx, lower_32_bits(addr) >> 4);
293	ep_ctx_write_deq_hi(ctx, upper_32_bits(addr));
294	}
295
296	struct tegra_xudc_trb {
297	__le32 data_lo;
298	__le32 data_hi;
299	__le32 status;
300	__le32 control;
301	};
302
303	#define TRB_TYPE_RSVD 0
304	#define TRB_TYPE_NORMAL 1
305	#define TRB_TYPE_SETUP_STAGE 2
306	#define TRB_TYPE_DATA_STAGE 3
307	#define TRB_TYPE_STATUS_STAGE 4
308	#define TRB_TYPE_ISOCH 5
309	#define TRB_TYPE_LINK 6
310	#define TRB_TYPE_TRANSFER_EVENT 32
311	#define TRB_TYPE_PORT_STATUS_CHANGE_EVENT 34
312	#define TRB_TYPE_STREAM 48
313	#define TRB_TYPE_SETUP_PACKET_EVENT 63
314
315	#define TRB_CMPL_CODE_INVALID 0
316	#define TRB_CMPL_CODE_SUCCESS 1
317	#define TRB_CMPL_CODE_DATA_BUFFER_ERR 2
318	#define TRB_CMPL_CODE_BABBLE_DETECTED_ERR 3
319	#define TRB_CMPL_CODE_USB_TRANS_ERR 4
320	#define TRB_CMPL_CODE_TRB_ERR 5
321	#define TRB_CMPL_CODE_STALL 6
322	#define TRB_CMPL_CODE_INVALID_STREAM_TYPE_ERR 10
323	#define TRB_CMPL_CODE_SHORT_PACKET 13
324	#define TRB_CMPL_CODE_RING_UNDERRUN 14
325	#define TRB_CMPL_CODE_RING_OVERRUN 15
326	#define TRB_CMPL_CODE_EVENT_RING_FULL_ERR 21
327	#define TRB_CMPL_CODE_STOPPED 26
328	#define TRB_CMPL_CODE_ISOCH_BUFFER_OVERRUN 31
329	#define TRB_CMPL_CODE_STREAM_NUMP_ERROR 219
330	#define TRB_CMPL_CODE_PRIME_PIPE_RECEIVED 220
331	#define TRB_CMPL_CODE_HOST_REJECTED 221
332	#define TRB_CMPL_CODE_CTRL_DIR_ERR 222
333	#define TRB_CMPL_CODE_CTRL_SEQNUM_ERR 223
334
335	#define BUILD_TRB_RW(name, member, shift, mask) \
336	static inline u32 trb_read_##name(struct tegra_xudc_trb *trb) \
337	{ \
338	return (le32_to_cpu(trb->member) >> (shift)) & (mask); \
339	} \
340	static inline void \
341	trb_write_##name(struct tegra_xudc_trb *trb, u32 val) \
342	{ \
343	u32 tmp; \
344	\
345	tmp = le32_to_cpu(trb->member) & ~((mask) << (shift)); \
346	tmp |= (val & (mask)) << (shift); \
347	trb->member = cpu_to_le32(tmp); \
348	}
349
350	BUILD_TRB_RW(data_lo, data_lo, 0, 0xffffffff)
351	BUILD_TRB_RW(data_hi, data_hi, 0, 0xffffffff)
352	BUILD_TRB_RW(seq_num, status, 0, 0xffff)
353	BUILD_TRB_RW(transfer_len, status, 0, 0xffffff)
354	BUILD_TRB_RW(td_size, status, 17, 0x1f)
355	BUILD_TRB_RW(cmpl_code, status, 24, 0xff)
356	BUILD_TRB_RW(cycle, control, 0, 0x1)
357	BUILD_TRB_RW(toggle_cycle, control, 1, 0x1)
358	BUILD_TRB_RW(isp, control, 2, 0x1)
359	BUILD_TRB_RW(chain, control, 4, 0x1)
360	BUILD_TRB_RW(ioc, control, 5, 0x1)
361	BUILD_TRB_RW(type, control, 10, 0x3f)
362	BUILD_TRB_RW(stream_id, control, 16, 0xffff)
363	BUILD_TRB_RW(endpoint_id, control, 16, 0x1f)
364	BUILD_TRB_RW(tlbpc, control, 16, 0xf)
365	BUILD_TRB_RW(data_stage_dir, control, 16, 0x1)
366	BUILD_TRB_RW(frame_id, control, 20, 0x7ff)
367	BUILD_TRB_RW(sia, control, 31, 0x1)
368
369	static inline u64 trb_read_data_ptr(struct tegra_xudc_trb *trb)
370	{
371	return ((u64)trb_read_data_hi(trb) << 32) |
372	trb_read_data_lo(trb);
373	}
374
375	static inline void trb_write_data_ptr(struct tegra_xudc_trb *trb, u64 addr)
376	{
377	trb_write_data_lo(trb, lower_32_bits(addr));
378	trb_write_data_hi(trb, upper_32_bits(addr));
379	}
380
381	struct tegra_xudc_request {
382	struct usb_request usb_req;
383
384	size_t buf_queued;
385	unsigned int trbs_queued;
386	unsigned int trbs_needed;
387	bool need_zlp;
388
389	struct tegra_xudc_trb *first_trb;
390	struct tegra_xudc_trb *last_trb;
391
392	struct list_head list;
393	};
394
395	struct tegra_xudc_ep {
396	struct tegra_xudc *xudc;
397	struct usb_ep usb_ep;
398	unsigned int index;
399	char name[8];
400
401	struct tegra_xudc_ep_context *context;
402
403	#define XUDC_TRANSFER_RING_SIZE 64
404	struct tegra_xudc_trb *transfer_ring;
405	dma_addr_t transfer_ring_phys;
406
407	unsigned int enq_ptr;
408	unsigned int deq_ptr;
409	bool pcs;
410	bool ring_full;
411	bool stream_rejected;
412
413	struct list_head queue;
414	const struct usb_endpoint_descriptor *desc;
415	const struct usb_ss_ep_comp_descriptor *comp_desc;
416	};
417
418	struct tegra_xudc_sel_timing {
419	__u8 u1sel;
420	__u8 u1pel;
421	__le16 u2sel;
422	__le16 u2pel;
423	};
424
425	enum tegra_xudc_setup_state {
426	WAIT_FOR_SETUP,
427	DATA_STAGE_XFER,
428	DATA_STAGE_RECV,
429	STATUS_STAGE_XFER,
430	STATUS_STAGE_RECV,
431	};
432
433	struct tegra_xudc_setup_packet {
434	struct usb_ctrlrequest ctrl_req;
435	unsigned int seq_num;
436	};
437
438	struct tegra_xudc_save_regs {
439	u32 ctrl;
440	u32 portpm;
441	};
442
443	struct tegra_xudc {
444	struct device *dev;
445	const struct tegra_xudc_soc *soc;
446	struct tegra_xusb_padctl *padctl;
447
448	spinlock_t lock;
449
450	struct usb_gadget gadget;
451	struct usb_gadget_driver *driver;
452
453	#define XUDC_NR_EVENT_RINGS 2
454	#define XUDC_EVENT_RING_SIZE 4096
455	struct tegra_xudc_trb *event_ring[XUDC_NR_EVENT_RINGS];
456	dma_addr_t event_ring_phys[XUDC_NR_EVENT_RINGS];
457	unsigned int event_ring_index;
458	unsigned int event_ring_deq_ptr;
459	bool ccs;
460
461	#define XUDC_NR_EPS 32
462	struct tegra_xudc_ep ep[XUDC_NR_EPS];
463	struct tegra_xudc_ep_context *ep_context;
464	dma_addr_t ep_context_phys;
465
466	struct device *genpd_dev_device;
467	struct device *genpd_dev_ss;
468	struct device_link *genpd_dl_device;
469	struct device_link *genpd_dl_ss;
470
471	struct dma_pool *transfer_ring_pool;
472
473	bool queued_setup_packet;
474	struct tegra_xudc_setup_packet setup_packet;
475	enum tegra_xudc_setup_state setup_state;
476	u16 setup_seq_num;
477
478	u16 dev_addr;
479	u16 isoch_delay;
480	struct tegra_xudc_sel_timing sel_timing;
481	u8 test_mode_pattern;
482	u16 status_buf;
483	struct tegra_xudc_request *ep0_req;
484
485	bool pullup;
486
487	unsigned int nr_enabled_eps;
488	unsigned int nr_isoch_eps;
489
490	unsigned int device_state;
491	unsigned int resume_state;
492
493	int irq;
494
495	void __iomem *base;
496	resource_size_t phys_base;
497	void __iomem *ipfs;
498	void __iomem *fpci;
499
500	struct regulator_bulk_data *supplies;
501
502	struct clk_bulk_data *clks;
503
504	bool device_mode;
505	struct work_struct usb_role_sw_work;
506
507	struct phy **usb3_phy;
508	struct phy *curr_usb3_phy;
509	struct phy **utmi_phy;
510	struct phy *curr_utmi_phy;
511
512	struct tegra_xudc_save_regs saved_regs;
513	bool suspended;
514	bool powergated;
515
516	struct usb_phy **usbphy;
517	struct usb_phy *curr_usbphy;
518	struct notifier_block vbus_nb;
519
520	struct completion disconnect_complete;
521
522	bool selfpowered;
523
524	#define TOGGLE_VBUS_WAIT_MS 100
525	struct delayed_work plc_reset_work;
526	bool wait_csc;
527
528	struct delayed_work port_reset_war_work;
529	bool wait_for_sec_prc;
530	};
531
532	#define XUDC_TRB_MAX_BUFFER_SIZE 65536
533	#define XUDC_MAX_ISOCH_EPS 4
534	#define XUDC_INTERRUPT_MODERATION_US 0
535
536	static struct usb_endpoint_descriptor tegra_xudc_ep0_desc = {
537	.bLength = USB_DT_ENDPOINT_SIZE,
538	.bDescriptorType = USB_DT_ENDPOINT,
539	.bEndpointAddress = 0,
540	.bmAttributes = USB_ENDPOINT_XFER_CONTROL,
541	.wMaxPacketSize = cpu_to_le16(64),
542	};
543
544	struct tegra_xudc_soc {
545	const char * const *supply_names;
546	unsigned int num_supplies;
547	const char * const *clock_names;
548	unsigned int num_clks;
549	unsigned int num_phys;
550	bool u1_enable;
551	bool u2_enable;
552	bool lpm_enable;
553	bool invalid_seq_num;
554	bool pls_quirk;
555	bool port_reset_quirk;
556	bool port_speed_quirk;
557	bool has_ipfs;
558	};
559
560	static inline u32 fpci_readl(struct tegra_xudc *xudc, unsigned int offset)
561	{
562	return readl(addr: xudc->fpci + offset);
563	}
564
565	static inline void fpci_writel(struct tegra_xudc *xudc, u32 val,
566	unsigned int offset)
567	{
568	writel(val, addr: xudc->fpci + offset);
569	}
570
571	static inline u32 ipfs_readl(struct tegra_xudc *xudc, unsigned int offset)
572	{
573	return readl(addr: xudc->ipfs + offset);
574	}
575
576	static inline void ipfs_writel(struct tegra_xudc *xudc, u32 val,
577	unsigned int offset)
578	{
579	writel(val, addr: xudc->ipfs + offset);
580	}
581
582	static inline u32 xudc_readl(struct tegra_xudc *xudc, unsigned int offset)
583	{
584	return readl(addr: xudc->base + offset);
585	}
586
587	static inline void xudc_writel(struct tegra_xudc *xudc, u32 val,
588	unsigned int offset)
589	{
590	writel(val, addr: xudc->base + offset);
591	}
592
593	static inline int xudc_readl_poll(struct tegra_xudc *xudc,
594	unsigned int offset, u32 mask, u32 val)
595	{
596	u32 regval;
597
598	return readl_poll_timeout_atomic(xudc->base + offset, regval,
599	(regval & mask) == val, 1, 100);
600	}
601
602	static inline struct tegra_xudc *to_xudc(struct usb_gadget *gadget)
603	{
604	return container_of(gadget, struct tegra_xudc, gadget);
605	}
606
607	static inline struct tegra_xudc_ep *to_xudc_ep(struct usb_ep *ep)
608	{
609	return container_of(ep, struct tegra_xudc_ep, usb_ep);
610	}
611
612	static inline struct tegra_xudc_request *to_xudc_req(struct usb_request *req)
613	{
614	return container_of(req, struct tegra_xudc_request, usb_req);
615	}
616
617	static inline void dump_trb(struct tegra_xudc *xudc, const char *type,
618	struct tegra_xudc_trb *trb)
619	{
620	dev_dbg(xudc->dev,
621	"%s: %p, lo = %#x, hi = %#x, status = %#x, control = %#x\n",
622	type, trb, trb->data_lo, trb->data_hi, trb->status,
623	trb->control);
624	}
625
626	static void tegra_xudc_limit_port_speed(struct tegra_xudc *xudc)
627	{
628	u32 val;
629
630	/* limit port speed to gen 1 */
631	val = xudc_readl(xudc, SSPX_CORE_CNT56);
632	val &= ~(SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX_MASK);
633	val |= SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX(0x260);
634	xudc_writel(xudc, val, SSPX_CORE_CNT56);
635
636	val = xudc_readl(xudc, SSPX_CORE_CNT57);
637	val &= ~(SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX_MASK);
638	val |= SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX(0x6D6);
639	xudc_writel(xudc, val, SSPX_CORE_CNT57);
640
641	val = xudc_readl(xudc, SSPX_CORE_CNT65);
642	val &= ~(SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID_MASK);
643	val |= SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID(0x4B0);
644	xudc_writel(xudc, val, SSPX_CORE_CNT66);
645
646	val = xudc_readl(xudc, SSPX_CORE_CNT66);
647	val &= ~(SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID_MASK);
648	val |= SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID(0x4B0);
649	xudc_writel(xudc, val, SSPX_CORE_CNT66);
650
651	val = xudc_readl(xudc, SSPX_CORE_CNT67);
652	val &= ~(SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID_MASK);
653	val |= SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID(0x4B0);
654	xudc_writel(xudc, val, SSPX_CORE_CNT67);
655
656	val = xudc_readl(xudc, SSPX_CORE_CNT72);
657	val &= ~(SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT_MASK);
658	val |= SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT(0x10);
659	xudc_writel(xudc, val, SSPX_CORE_CNT72);
660	}
661
662	static void tegra_xudc_restore_port_speed(struct tegra_xudc *xudc)
663	{
664	u32 val;
665
666	/* restore port speed to gen2 */
667	val = xudc_readl(xudc, SSPX_CORE_CNT56);
668	val &= ~(SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX_MASK);
669	val |= SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX(0x438);
670	xudc_writel(xudc, val, SSPX_CORE_CNT56);
671
672	val = xudc_readl(xudc, SSPX_CORE_CNT57);
673	val &= ~(SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX_MASK);
674	val |= SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX(0x528);
675	xudc_writel(xudc, val, SSPX_CORE_CNT57);
676
677	val = xudc_readl(xudc, SSPX_CORE_CNT65);
678	val &= ~(SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID_MASK);
679	val |= SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID(0xE10);
680	xudc_writel(xudc, val, SSPX_CORE_CNT66);
681
682	val = xudc_readl(xudc, SSPX_CORE_CNT66);
683	val &= ~(SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID_MASK);
684	val |= SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID(0x348);
685	xudc_writel(xudc, val, SSPX_CORE_CNT66);
686
687	val = xudc_readl(xudc, SSPX_CORE_CNT67);
688	val &= ~(SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID_MASK);
689	val |= SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID(0x5a0);
690	xudc_writel(xudc, val, SSPX_CORE_CNT67);
691
692	val = xudc_readl(xudc, SSPX_CORE_CNT72);
693	val &= ~(SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT_MASK);
694	val |= SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT(0x1c21);
695	xudc_writel(xudc, val, SSPX_CORE_CNT72);
696	}
697
698	static void tegra_xudc_device_mode_on(struct tegra_xudc *xudc)
699	{
700	int err;
701
702	pm_runtime_get_sync(dev: xudc->dev);
703
704	tegra_phy_xusb_utmi_pad_power_on(phy: xudc->curr_utmi_phy);
705
706	err = phy_power_on(phy: xudc->curr_utmi_phy);
707	if (err < 0)
708	dev_err(xudc->dev, "UTMI power on failed: %d\n", err);
709
710	err = phy_power_on(phy: xudc->curr_usb3_phy);
711	if (err < 0)
712	dev_err(xudc->dev, "USB3 PHY power on failed: %d\n", err);
713
714	dev_dbg(xudc->dev, "device mode on\n");
715
716	phy_set_mode_ext(phy: xudc->curr_utmi_phy, mode: PHY_MODE_USB_OTG,
717	submode: USB_ROLE_DEVICE);
718	}
719
720	static void tegra_xudc_device_mode_off(struct tegra_xudc *xudc)
721	{
722	bool connected = false;
723	u32 pls, val;
724	int err;
725
726	dev_dbg(xudc->dev, "device mode off\n");
727
728	connected = !!(xudc_readl(xudc, PORTSC) & PORTSC_CCS);
729
730	reinit_completion(x: &xudc->disconnect_complete);
731
732	if (xudc->soc->port_speed_quirk)
733	tegra_xudc_restore_port_speed(xudc);
734
735	phy_set_mode_ext(phy: xudc->curr_utmi_phy, mode: PHY_MODE_USB_OTG, submode: USB_ROLE_NONE);
736
737	pls = (xudc_readl(xudc, PORTSC) & PORTSC_PLS_MASK) >>
738	PORTSC_PLS_SHIFT;
739
740	/* Direct link to U0 if disconnected in RESUME or U2. */
741	if (xudc->soc->pls_quirk && xudc->gadget.speed == USB_SPEED_SUPER &&
742	(pls == PORTSC_PLS_RESUME || pls == PORTSC_PLS_U2)) {
743	val = xudc_readl(xudc, PORTPM);
744	val |= PORTPM_FRWE;
745	xudc_writel(xudc, val, PORTPM);
746
747	val = xudc_readl(xudc, PORTSC);
748	val &= ~(PORTSC_CHANGE_MASK | PORTSC_PLS_MASK);
749	val |= PORTSC_LWS | PORTSC_PLS(PORTSC_PLS_U0);
750	xudc_writel(xudc, val, PORTSC);
751	}
752
753	/* Wait for disconnect event. */
754	if (connected)
755	wait_for_completion(&xudc->disconnect_complete);
756
757	/* Make sure interrupt handler has completed before powergating. */
758	synchronize_irq(irq: xudc->irq);
759
760	tegra_phy_xusb_utmi_pad_power_down(phy: xudc->curr_utmi_phy);
761
762	err = phy_power_off(phy: xudc->curr_utmi_phy);
763	if (err < 0)
764	dev_err(xudc->dev, "UTMI PHY power off failed: %d\n", err);
765
766	err = phy_power_off(phy: xudc->curr_usb3_phy);
767	if (err < 0)
768	dev_err(xudc->dev, "USB3 PHY power off failed: %d\n", err);
769
770	pm_runtime_put(dev: xudc->dev);
771	}
772
773	static void tegra_xudc_usb_role_sw_work(struct work_struct *work)
774	{
775	struct tegra_xudc *xudc = container_of(work, struct tegra_xudc,
776	usb_role_sw_work);
777
778	if (xudc->device_mode)
779	tegra_xudc_device_mode_on(xudc);
780	else
781	tegra_xudc_device_mode_off(xudc);
782	}
783
784	static int tegra_xudc_get_phy_index(struct tegra_xudc *xudc,
785	struct usb_phy *usbphy)
786	{
787	unsigned int i;
788
789	for (i = 0; i < xudc->soc->num_phys; i++) {
790	if (xudc->usbphy[i] && usbphy == xudc->usbphy[i])
791	return i;
792	}
793
794	dev_info(xudc->dev, "phy index could not be found for shared USB PHY");
795	return -1;
796	}
797
798	static void tegra_xudc_update_data_role(struct tegra_xudc *xudc,
799	struct usb_phy *usbphy)
800	{
801	int phy_index;
802
803	if ((xudc->device_mode && usbphy->last_event == USB_EVENT_VBUS) ||
804	(!xudc->device_mode && usbphy->last_event != USB_EVENT_VBUS)) {
805	dev_dbg(xudc->dev, "Same role(%d) received. Ignore",
806	xudc->device_mode);
807	return;
808	}
809
810	xudc->device_mode = (usbphy->last_event == USB_EVENT_VBUS) ? true :
811	false;
812
813	phy_index = tegra_xudc_get_phy_index(xudc, usbphy);
814	dev_dbg(xudc->dev, "%s(): current phy index is %d\n", __func__,
815	phy_index);
816
817	if (!xudc->suspended && phy_index != -1) {
818	xudc->curr_utmi_phy = xudc->utmi_phy[phy_index];
819	xudc->curr_usb3_phy = xudc->usb3_phy[phy_index];
820	xudc->curr_usbphy = usbphy;
821	schedule_work(work: &xudc->usb_role_sw_work);
822	}
823	}
824
825	static int tegra_xudc_vbus_notify(struct notifier_block *nb,
826	unsigned long action, void *data)
827	{
828	struct tegra_xudc *xudc = container_of(nb, struct tegra_xudc,
829	vbus_nb);
830	struct usb_phy *usbphy = (struct usb_phy *)data;
831
832	dev_dbg(xudc->dev, "%s(): event is %d\n", __func__, usbphy->last_event);
833
834	tegra_xudc_update_data_role(xudc, usbphy);
835
836	return NOTIFY_OK;
837	}
838
839	static void tegra_xudc_plc_reset_work(struct work_struct *work)
840	{
841	struct delayed_work *dwork = to_delayed_work(work);
842	struct tegra_xudc *xudc = container_of(dwork, struct tegra_xudc,
843	plc_reset_work);
844	unsigned long flags;
845
846	spin_lock_irqsave(&xudc->lock, flags);
847
848	if (xudc->wait_csc) {
849	u32 pls = (xudc_readl(xudc, PORTSC) & PORTSC_PLS_MASK) >>
850	PORTSC_PLS_SHIFT;
851
852	if (pls == PORTSC_PLS_INACTIVE) {
853	dev_info(xudc->dev, "PLS = Inactive. Toggle VBUS\n");
854	phy_set_mode_ext(phy: xudc->curr_utmi_phy, mode: PHY_MODE_USB_OTG,
855	submode: USB_ROLE_NONE);
856	phy_set_mode_ext(phy: xudc->curr_utmi_phy, mode: PHY_MODE_USB_OTG,
857	submode: USB_ROLE_DEVICE);
858
859	xudc->wait_csc = false;
860	}
861	}
862
863	spin_unlock_irqrestore(lock: &xudc->lock, flags);
864	}
865
866	static void tegra_xudc_port_reset_war_work(struct work_struct *work)
867	{
868	struct delayed_work *dwork = to_delayed_work(work);
869	struct tegra_xudc *xudc =
870	container_of(dwork, struct tegra_xudc, port_reset_war_work);
871	unsigned long flags;
872	u32 pls;
873	int ret;
874
875	spin_lock_irqsave(&xudc->lock, flags);
876
877	if (xudc->device_mode && xudc->wait_for_sec_prc) {
878	pls = (xudc_readl(xudc, PORTSC) & PORTSC_PLS_MASK) >>
879	PORTSC_PLS_SHIFT;
880	dev_dbg(xudc->dev, "pls = %x\n", pls);
881
882	if (pls == PORTSC_PLS_DISABLED) {
883	dev_dbg(xudc->dev, "toggle vbus\n");
884	/* PRC doesn't complete in 100ms, toggle the vbus */
885	ret = tegra_phy_xusb_utmi_port_reset(
886	phy: xudc->curr_utmi_phy);
887	if (ret == 1)
888	xudc->wait_for_sec_prc = 0;
889	}
890	}
891
892	spin_unlock_irqrestore(lock: &xudc->lock, flags);
893	}
894
895	static dma_addr_t trb_virt_to_phys(struct tegra_xudc_ep *ep,
896	struct tegra_xudc_trb *trb)
897	{
898	unsigned int index;
899
900	index = trb - ep->transfer_ring;
901
902	if (WARN_ON(index >= XUDC_TRANSFER_RING_SIZE))
903	return 0;
904
905	return (ep->transfer_ring_phys + index * sizeof(*trb));
906	}
907
908	static struct tegra_xudc_trb *trb_phys_to_virt(struct tegra_xudc_ep *ep,
909	dma_addr_t addr)
910	{
911	struct tegra_xudc_trb *trb;
912	unsigned int index;
913
914	index = (addr - ep->transfer_ring_phys) / sizeof(*trb);
915
916	if (WARN_ON(index >= XUDC_TRANSFER_RING_SIZE))
917	return NULL;
918
919	trb = &ep->transfer_ring[index];
920
921	return trb;
922	}
923
924	static void ep_reload(struct tegra_xudc *xudc, unsigned int ep)
925	{
926	xudc_writel(xudc, BIT(ep), EP_RELOAD);
927	xudc_readl_poll(xudc, EP_RELOAD, BIT(ep), val: 0);
928	}
929
930	static void ep_pause(struct tegra_xudc *xudc, unsigned int ep)
931	{
932	u32 val;
933
934	val = xudc_readl(xudc, EP_PAUSE);
935	if (val & BIT(ep))
936	return;
937	val |= BIT(ep);
938
939	xudc_writel(xudc, val, EP_PAUSE);
940
941	xudc_readl_poll(xudc, EP_STCHG, BIT(ep), BIT(ep));
942
943	xudc_writel(xudc, BIT(ep), EP_STCHG);
944	}
945
946	static void ep_unpause(struct tegra_xudc *xudc, unsigned int ep)
947	{
948	u32 val;
949
950	val = xudc_readl(xudc, EP_PAUSE);
951	if (!(val & BIT(ep)))
952	return;
953	val &= ~BIT(ep);
954
955	xudc_writel(xudc, val, EP_PAUSE);
956
957	xudc_readl_poll(xudc, EP_STCHG, BIT(ep), BIT(ep));
958
959	xudc_writel(xudc, BIT(ep), EP_STCHG);
960	}
961
962	static void ep_unpause_all(struct tegra_xudc *xudc)
963	{
964	u32 val;
965
966	val = xudc_readl(xudc, EP_PAUSE);
967
968	xudc_writel(xudc, val: 0, EP_PAUSE);
969
970	xudc_readl_poll(xudc, EP_STCHG, mask: val, val);
971
972	xudc_writel(xudc, val, EP_STCHG);
973	}
974
975	static void ep_halt(struct tegra_xudc *xudc, unsigned int ep)
976	{
977	u32 val;
978
979	val = xudc_readl(xudc, EP_HALT);
980	if (val & BIT(ep))
981	return;
982	val |= BIT(ep);
983	xudc_writel(xudc, val, EP_HALT);
984
985	xudc_readl_poll(xudc, EP_STCHG, BIT(ep), BIT(ep));
986
987	xudc_writel(xudc, BIT(ep), EP_STCHG);
988	}
989
990	static void ep_unhalt(struct tegra_xudc *xudc, unsigned int ep)
991	{
992	u32 val;
993
994	val = xudc_readl(xudc, EP_HALT);
995	if (!(val & BIT(ep)))
996	return;
997	val &= ~BIT(ep);
998	xudc_writel(xudc, val, EP_HALT);
999
1000	xudc_readl_poll(xudc, EP_STCHG, BIT(ep), BIT(ep));
1001
1002	xudc_writel(xudc, BIT(ep), EP_STCHG);
1003	}
1004
1005	static void ep_unhalt_all(struct tegra_xudc *xudc)
1006	{
1007	u32 val;
1008
1009	val = xudc_readl(xudc, EP_HALT);
1010	if (!val)
1011	return;
1012	xudc_writel(xudc, val: 0, EP_HALT);
1013
1014	xudc_readl_poll(xudc, EP_STCHG, mask: val, val);
1015
1016	xudc_writel(xudc, val, EP_STCHG);
1017	}
1018
1019	static void ep_wait_for_stopped(struct tegra_xudc *xudc, unsigned int ep)
1020	{
1021	xudc_readl_poll(xudc, EP_STOPPED, BIT(ep), BIT(ep));
1022	xudc_writel(xudc, BIT(ep), EP_STOPPED);
1023	}
1024
1025	static void ep_wait_for_inactive(struct tegra_xudc *xudc, unsigned int ep)
1026	{
1027	xudc_readl_poll(xudc, EP_THREAD_ACTIVE, BIT(ep), val: 0);
1028	}
1029
1030	static void tegra_xudc_req_done(struct tegra_xudc_ep *ep,
1031	struct tegra_xudc_request *req, int status)
1032	{
1033	struct tegra_xudc *xudc = ep->xudc;
1034
1035	dev_dbg(xudc->dev, "completing request %p on EP %u with status %d\n",
1036	req, ep->index, status);
1037
1038	if (likely(req->usb_req.status == -EINPROGRESS))
1039	req->usb_req.status = status;
1040
1041	list_del_init(entry: &req->list);
1042
1043	if (usb_endpoint_xfer_control(epd: ep->desc)) {
1044	usb_gadget_unmap_request(gadget: &xudc->gadget, req: &req->usb_req,
1045	is_in: (xudc->setup_state ==
1046	DATA_STAGE_XFER));
1047	} else {
1048	usb_gadget_unmap_request(gadget: &xudc->gadget, req: &req->usb_req,
1049	is_in: usb_endpoint_dir_in(epd: ep->desc));
1050	}
1051
1052	spin_unlock(lock: &xudc->lock);
1053	usb_gadget_giveback_request(ep: &ep->usb_ep, req: &req->usb_req);
1054	spin_lock(lock: &xudc->lock);
1055	}
1056
1057	static void tegra_xudc_ep_nuke(struct tegra_xudc_ep *ep, int status)
1058	{
1059	struct tegra_xudc_request *req;
1060
1061	while (!list_empty(head: &ep->queue)) {
1062	req = list_first_entry(&ep->queue, struct tegra_xudc_request,
1063	list);
1064	tegra_xudc_req_done(ep, req, status);
1065	}
1066	}
1067
1068	static unsigned int ep_available_trbs(struct tegra_xudc_ep *ep)
1069	{
1070	if (ep->ring_full)
1071	return 0;
1072
1073	if (ep->deq_ptr > ep->enq_ptr)
1074	return ep->deq_ptr - ep->enq_ptr - 1;
1075
1076	return XUDC_TRANSFER_RING_SIZE - (ep->enq_ptr - ep->deq_ptr) - 2;
1077	}
1078
1079	static void tegra_xudc_queue_one_trb(struct tegra_xudc_ep *ep,
1080	struct tegra_xudc_request *req,
1081	struct tegra_xudc_trb *trb,
1082	bool ioc)
1083	{
1084	struct tegra_xudc *xudc = ep->xudc;
1085	dma_addr_t buf_addr;
1086	size_t len;
1087
1088	len = min_t(size_t, XUDC_TRB_MAX_BUFFER_SIZE, req->usb_req.length -
1089	req->buf_queued);
1090	if (len > 0)
1091	buf_addr = req->usb_req.dma + req->buf_queued;
1092	else
1093	buf_addr = 0;
1094
1095	trb_write_data_ptr(trb, addr: buf_addr);
1096
1097	trb_write_transfer_len(trb, val: len);
1098	trb_write_td_size(trb, val: req->trbs_needed - req->trbs_queued - 1);
1099
1100	if (req->trbs_queued == req->trbs_needed - 1 ||
1101	(req->need_zlp && req->trbs_queued == req->trbs_needed - 2))
1102	trb_write_chain(trb, val: 0);
1103	else
1104	trb_write_chain(trb, val: 1);
1105
1106	trb_write_ioc(trb, val: ioc);
1107
1108	if (usb_endpoint_dir_out(epd: ep->desc) ||
1109	(usb_endpoint_xfer_control(epd: ep->desc) &&
1110	(xudc->setup_state == DATA_STAGE_RECV)))
1111	trb_write_isp(trb, val: 1);
1112	else
1113	trb_write_isp(trb, val: 0);
1114
1115	if (usb_endpoint_xfer_control(epd: ep->desc)) {
1116	if (xudc->setup_state == DATA_STAGE_XFER ||
1117	xudc->setup_state == DATA_STAGE_RECV)
1118	trb_write_type(trb, TRB_TYPE_DATA_STAGE);
1119	else
1120	trb_write_type(trb, TRB_TYPE_STATUS_STAGE);
1121
1122	if (xudc->setup_state == DATA_STAGE_XFER ||
1123	xudc->setup_state == STATUS_STAGE_XFER)
1124	trb_write_data_stage_dir(trb, val: 1);
1125	else
1126	trb_write_data_stage_dir(trb, val: 0);
1127	} else if (usb_endpoint_xfer_isoc(epd: ep->desc)) {
1128	trb_write_type(trb, TRB_TYPE_ISOCH);
1129	trb_write_sia(trb, val: 1);
1130	trb_write_frame_id(trb, val: 0);
1131	trb_write_tlbpc(trb, val: 0);
1132	} else if (usb_ss_max_streams(comp: ep->comp_desc)) {
1133	trb_write_type(trb, TRB_TYPE_STREAM);
1134	trb_write_stream_id(trb, val: req->usb_req.stream_id);
1135	} else {
1136	trb_write_type(trb, TRB_TYPE_NORMAL);
1137	trb_write_stream_id(trb, val: 0);
1138	}
1139
1140	trb_write_cycle(trb, val: ep->pcs);
1141
1142	req->trbs_queued++;
1143	req->buf_queued += len;
1144
1145	dump_trb(xudc, type: "TRANSFER", trb);
1146	}
1147
1148	static unsigned int tegra_xudc_queue_trbs(struct tegra_xudc_ep *ep,
1149	struct tegra_xudc_request *req)
1150	{
1151	unsigned int i, count, available;
1152	bool wait_td = false;
1153
1154	available = ep_available_trbs(ep);
1155	count = req->trbs_needed - req->trbs_queued;
1156	if (available < count) {
1157	count = available;
1158	ep->ring_full = true;
1159	}
1160
1161	/*
1162	* To generate zero-length packet on USB bus, SW needs schedule a
1163	* standalone zero-length TD. According to HW's behavior, SW needs
1164	* to schedule TDs in different ways for different endpoint types.
1165	*
1166	* For control endpoint:
1167	* - Data stage TD (IOC = 1, CH = 0)
1168	* - Ring doorbell and wait transfer event
1169	* - Data stage TD for ZLP (IOC = 1, CH = 0)
1170	* - Ring doorbell
1171	*
1172	* For bulk and interrupt endpoints:
1173	* - Normal transfer TD (IOC = 0, CH = 0)
1174	* - Normal transfer TD for ZLP (IOC = 1, CH = 0)
1175	* - Ring doorbell
1176	*/
1177
1178	if (req->need_zlp && usb_endpoint_xfer_control(epd: ep->desc) && count > 1)
1179	wait_td = true;
1180
1181	if (!req->first_trb)
1182	req->first_trb = &ep->transfer_ring[ep->enq_ptr];
1183
1184	for (i = 0; i < count; i++) {
1185	struct tegra_xudc_trb *trb = &ep->transfer_ring[ep->enq_ptr];
1186	bool ioc = false;
1187
1188	if ((i == count - 1) || (wait_td && i == count - 2))
1189	ioc = true;
1190
1191	tegra_xudc_queue_one_trb(ep, req, trb, ioc);
1192	req->last_trb = trb;
1193
1194	ep->enq_ptr++;
1195	if (ep->enq_ptr == XUDC_TRANSFER_RING_SIZE - 1) {
1196	trb = &ep->transfer_ring[ep->enq_ptr];
1197	trb_write_cycle(trb, val: ep->pcs);
1198	ep->pcs = !ep->pcs;
1199	ep->enq_ptr = 0;
1200	}
1201
1202	if (ioc)
1203	break;
1204	}
1205
1206	return count;
1207	}
1208
1209	static void tegra_xudc_ep_ring_doorbell(struct tegra_xudc_ep *ep)
1210	{
1211	struct tegra_xudc *xudc = ep->xudc;
1212	u32 val;
1213
1214	if (list_empty(head: &ep->queue))
1215	return;
1216
1217	val = DB_TARGET(ep->index);
1218	if (usb_endpoint_xfer_control(epd: ep->desc)) {
1219	val |= DB_STREAMID(xudc->setup_seq_num);
1220	} else if (usb_ss_max_streams(comp: ep->comp_desc) > 0) {
1221	struct tegra_xudc_request *req;
1222
1223	/* Don't ring doorbell if the stream has been rejected. */
1224	if (ep->stream_rejected)
1225	return;
1226
1227	req = list_first_entry(&ep->queue, struct tegra_xudc_request,
1228	list);
1229	val |= DB_STREAMID(req->usb_req.stream_id);
1230	}
1231
1232	dev_dbg(xudc->dev, "ring doorbell: %#x\n", val);
1233	xudc_writel(xudc, val, DB);
1234	}
1235
1236	static void tegra_xudc_ep_kick_queue(struct tegra_xudc_ep *ep)
1237	{
1238	struct tegra_xudc_request *req;
1239	bool trbs_queued = false;
1240
1241	list_for_each_entry(req, &ep->queue, list) {
1242	if (ep->ring_full)
1243	break;
1244
1245	if (tegra_xudc_queue_trbs(ep, req) > 0)
1246	trbs_queued = true;
1247	}
1248
1249	if (trbs_queued)
1250	tegra_xudc_ep_ring_doorbell(ep);
1251	}
1252
1253	static int
1254	__tegra_xudc_ep_queue(struct tegra_xudc_ep *ep, struct tegra_xudc_request *req)
1255	{
1256	struct tegra_xudc *xudc = ep->xudc;
1257	int err;
1258
1259	if (usb_endpoint_xfer_control(epd: ep->desc) && !list_empty(head: &ep->queue)) {
1260	dev_err(xudc->dev, "control EP has pending transfers\n");
1261	return -EINVAL;
1262	}
1263
1264	if (usb_endpoint_xfer_control(epd: ep->desc)) {
1265	err = usb_gadget_map_request(gadget: &xudc->gadget, req: &req->usb_req,
1266	is_in: (xudc->setup_state ==
1267	DATA_STAGE_XFER));
1268	} else {
1269	err = usb_gadget_map_request(gadget: &xudc->gadget, req: &req->usb_req,
1270	is_in: usb_endpoint_dir_in(epd: ep->desc));
1271	}
1272
1273	if (err < 0) {
1274	dev_err(xudc->dev, "failed to map request: %d\n", err);
1275	return err;
1276	}
1277
1278	req->first_trb = NULL;
1279	req->last_trb = NULL;
1280	req->buf_queued = 0;
1281	req->trbs_queued = 0;
1282	req->need_zlp = false;
1283	req->trbs_needed = DIV_ROUND_UP(req->usb_req.length,
1284	XUDC_TRB_MAX_BUFFER_SIZE);
1285	if (req->usb_req.length == 0)
1286	req->trbs_needed++;
1287
1288	if (!usb_endpoint_xfer_isoc(epd: ep->desc) &&
1289	req->usb_req.zero && req->usb_req.length &&
1290	((req->usb_req.length % ep->usb_ep.maxpacket) == 0)) {
1291	req->trbs_needed++;
1292	req->need_zlp = true;
1293	}
1294
1295	req->usb_req.status = -EINPROGRESS;
1296	req->usb_req.actual = 0;
1297
1298	list_add_tail(new: &req->list, head: &ep->queue);
1299
1300	tegra_xudc_ep_kick_queue(ep);
1301
1302	return 0;
1303	}
1304
1305	static int
1306	tegra_xudc_ep_queue(struct usb_ep *usb_ep, struct usb_request *usb_req,
1307	gfp_t gfp)
1308	{
1309	struct tegra_xudc_request *req;
1310	struct tegra_xudc_ep *ep;
1311	struct tegra_xudc *xudc;
1312	unsigned long flags;
1313	int ret;
1314
1315	if (!usb_ep || !usb_req)
1316	return -EINVAL;
1317
1318	ep = to_xudc_ep(ep: usb_ep);
1319	req = to_xudc_req(req: usb_req);
1320	xudc = ep->xudc;
1321
1322	spin_lock_irqsave(&xudc->lock, flags);
1323	if (xudc->powergated || !ep->desc) {
1324	ret = -ESHUTDOWN;
1325	goto unlock;
1326	}
1327
1328	ret = __tegra_xudc_ep_queue(ep, req);
1329	unlock:
1330	spin_unlock_irqrestore(lock: &xudc->lock, flags);
1331
1332	return ret;
1333	}
1334
1335	static void squeeze_transfer_ring(struct tegra_xudc_ep *ep,
1336	struct tegra_xudc_request *req)
1337	{
1338	struct tegra_xudc_trb *trb = req->first_trb;
1339	bool pcs_enq = trb_read_cycle(trb);
1340	bool pcs;
1341
1342	/*
1343	* Clear out all the TRBs part of or after the cancelled request,
1344	* and must correct trb cycle bit to the last un-enqueued state.
1345	*/
1346	while (trb != &ep->transfer_ring[ep->enq_ptr]) {
1347	pcs = trb_read_cycle(trb);
1348	memset(trb, 0, sizeof(*trb));
1349	trb_write_cycle(trb, val: !pcs);
1350	trb++;
1351
1352	if (trb_read_type(trb) == TRB_TYPE_LINK)
1353	trb = ep->transfer_ring;
1354	}
1355
1356	/* Requests will be re-queued at the start of the cancelled request. */
1357	ep->enq_ptr = req->first_trb - ep->transfer_ring;
1358	/*
1359	* Retrieve the correct cycle bit state from the first trb of
1360	* the cancelled request.
1361	*/
1362	ep->pcs = pcs_enq;
1363	ep->ring_full = false;
1364	list_for_each_entry_continue(req, &ep->queue, list) {
1365	req->usb_req.status = -EINPROGRESS;
1366	req->usb_req.actual = 0;
1367
1368	req->first_trb = NULL;
1369	req->last_trb = NULL;
1370	req->buf_queued = 0;
1371	req->trbs_queued = 0;
1372	}
1373	}
1374
1375	/*
1376	* Determine if the given TRB is in the range [first trb, last trb] for the
1377	* given request.
1378	*/
1379	static bool trb_in_request(struct tegra_xudc_ep *ep,
1380	struct tegra_xudc_request *req,
1381	struct tegra_xudc_trb *trb)
1382	{
1383	dev_dbg(ep->xudc->dev, "%s: request %p -> %p; trb %p\n", __func__,
1384	req->first_trb, req->last_trb, trb);
1385
1386	if (trb >= req->first_trb && (trb <= req->last_trb ||
1387	req->last_trb < req->first_trb))
1388	return true;
1389
1390	if (trb < req->first_trb && trb <= req->last_trb &&
1391	req->last_trb < req->first_trb)
1392	return true;
1393
1394	return false;
1395	}
1396
1397	/*
1398	* Determine if the given TRB is in the range [EP enqueue pointer, first TRB)
1399	* for the given endpoint and request.
1400	*/
1401	static bool trb_before_request(struct tegra_xudc_ep *ep,
1402	struct tegra_xudc_request *req,
1403	struct tegra_xudc_trb *trb)
1404	{
1405	struct tegra_xudc_trb *enq_trb = &ep->transfer_ring[ep->enq_ptr];
1406
1407	dev_dbg(ep->xudc->dev, "%s: request %p -> %p; enq ptr: %p; trb %p\n",
1408	__func__, req->first_trb, req->last_trb, enq_trb, trb);
1409
1410	if (trb < req->first_trb && (enq_trb <= trb ||
1411	req->first_trb < enq_trb))
1412	return true;
1413
1414	if (trb > req->first_trb && req->first_trb < enq_trb && enq_trb <= trb)
1415	return true;
1416
1417	return false;
1418	}
1419
1420	static int
1421	__tegra_xudc_ep_dequeue(struct tegra_xudc_ep *ep,
1422	struct tegra_xudc_request *req)
1423	{
1424	struct tegra_xudc *xudc = ep->xudc;
1425	struct tegra_xudc_request *r = NULL, *iter;
1426	struct tegra_xudc_trb *deq_trb;
1427	bool busy, kick_queue = false;
1428	int ret = 0;
1429
1430	/* Make sure the request is actually queued to this endpoint. */
1431	list_for_each_entry(iter, &ep->queue, list) {
1432	if (iter != req)
1433	continue;
1434	r = iter;
1435	break;
1436	}
1437
1438	if (!r)
1439	return -EINVAL;
1440
1441	/* Request hasn't been queued in the transfer ring yet. */
1442	if (!req->trbs_queued) {
1443	tegra_xudc_req_done(ep, req, status: -ECONNRESET);
1444	return 0;
1445	}
1446
1447	/* Halt DMA for this endpoint. */
1448	if (ep_ctx_read_state(ctx: ep->context) == EP_STATE_RUNNING) {
1449	ep_pause(xudc, ep: ep->index);
1450	ep_wait_for_inactive(xudc, ep: ep->index);
1451	}
1452
1453	deq_trb = trb_phys_to_virt(ep, addr: ep_ctx_read_deq_ptr(ctx: ep->context));
1454	/* Is the hardware processing the TRB at the dequeue pointer? */
1455	busy = (trb_read_cycle(trb: deq_trb) == ep_ctx_read_dcs(ctx: ep->context));
1456
1457	if (trb_in_request(ep, req, trb: deq_trb) && busy) {
1458	/*
1459	* Request has been partially completed or it hasn't
1460	* started processing yet.
1461	*/
1462	dma_addr_t deq_ptr;
1463
1464	squeeze_transfer_ring(ep, req);
1465
1466	req->usb_req.actual = ep_ctx_read_edtla(ctx: ep->context);
1467	tegra_xudc_req_done(ep, req, status: -ECONNRESET);
1468	kick_queue = true;
1469
1470	/* EDTLA is > 0: request has been partially completed */
1471	if (req->usb_req.actual > 0) {
1472	/*
1473	* Abort the pending transfer and update the dequeue
1474	* pointer
1475	*/
1476	ep_ctx_write_edtla(ctx: ep->context, val: 0);
1477	ep_ctx_write_partial_td(ctx: ep->context, val: 0);
1478	ep_ctx_write_data_offset(ctx: ep->context, val: 0);
1479
1480	deq_ptr = trb_virt_to_phys(ep,
1481	trb: &ep->transfer_ring[ep->enq_ptr]);
1482
1483	if (dma_mapping_error(dev: xudc->dev, dma_addr: deq_ptr)) {
1484	ret = -EINVAL;
1485	} else {
1486	ep_ctx_write_deq_ptr(ctx: ep->context, addr: deq_ptr);
1487	ep_ctx_write_dcs(ctx: ep->context, val: ep->pcs);
1488	ep_reload(xudc, ep: ep->index);
1489	}
1490	}
1491	} else if (trb_before_request(ep, req, trb: deq_trb) && busy) {
1492	/* Request hasn't started processing yet. */
1493	squeeze_transfer_ring(ep, req);
1494
1495	tegra_xudc_req_done(ep, req, status: -ECONNRESET);
1496	kick_queue = true;
1497	} else {
1498	/*
1499	* Request has completed, but we haven't processed the
1500	* completion event yet.
1501	*/
1502	tegra_xudc_req_done(ep, req, status: -ECONNRESET);
1503	ret = -EINVAL;
1504	}
1505
1506	/* Resume the endpoint. */
1507	ep_unpause(xudc, ep: ep->index);
1508
1509	if (kick_queue)
1510	tegra_xudc_ep_kick_queue(ep);
1511
1512	return ret;
1513	}
1514
1515	static int
1516	tegra_xudc_ep_dequeue(struct usb_ep *usb_ep, struct usb_request *usb_req)
1517	{
1518	struct tegra_xudc_request *req;
1519	struct tegra_xudc_ep *ep;
1520	struct tegra_xudc *xudc;
1521	unsigned long flags;
1522	int ret;
1523
1524	if (!usb_ep || !usb_req)
1525	return -EINVAL;
1526
1527	ep = to_xudc_ep(ep: usb_ep);
1528	req = to_xudc_req(req: usb_req);
1529	xudc = ep->xudc;
1530
1531	spin_lock_irqsave(&xudc->lock, flags);
1532
1533	if (xudc->powergated || !ep->desc) {
1534	ret = -ESHUTDOWN;
1535	goto unlock;
1536	}
1537
1538	ret = __tegra_xudc_ep_dequeue(ep, req);
1539	unlock:
1540	spin_unlock_irqrestore(lock: &xudc->lock, flags);
1541
1542	return ret;
1543	}
1544
1545	static int __tegra_xudc_ep_set_halt(struct tegra_xudc_ep *ep, bool halt)
1546	{
1547	struct tegra_xudc *xudc = ep->xudc;
1548
1549	if (!ep->desc)
1550	return -EINVAL;
1551
1552	if (usb_endpoint_xfer_isoc(epd: ep->desc)) {
1553	dev_err(xudc->dev, "can't halt isochronous EP\n");
1554	return -ENOTSUPP;
1555	}
1556
1557	if (!!(xudc_readl(xudc, EP_HALT) & BIT(ep->index)) == halt) {
1558	dev_dbg(xudc->dev, "EP %u already %s\n", ep->index,
1559	halt ? "halted" : "not halted");
1560	return 0;
1561	}
1562
1563	if (halt) {
1564	ep_halt(xudc, ep: ep->index);
1565	} else {
1566	ep_ctx_write_state(ctx: ep->context, EP_STATE_DISABLED);
1567
1568	ep_reload(xudc, ep: ep->index);
1569
1570	ep_ctx_write_state(ctx: ep->context, EP_STATE_RUNNING);
1571	ep_ctx_write_rsvd(ctx: ep->context, val: 0);
1572	ep_ctx_write_partial_td(ctx: ep->context, val: 0);
1573	ep_ctx_write_splitxstate(ctx: ep->context, val: 0);
1574	ep_ctx_write_seq_num(ctx: ep->context, val: 0);
1575
1576	ep_reload(xudc, ep: ep->index);
1577	ep_unpause(xudc, ep: ep->index);
1578	ep_unhalt(xudc, ep: ep->index);
1579
1580	tegra_xudc_ep_ring_doorbell(ep);
1581	}
1582
1583	return 0;
1584	}
1585
1586	static int tegra_xudc_ep_set_halt(struct usb_ep *usb_ep, int value)
1587	{
1588	struct tegra_xudc_ep *ep;
1589	struct tegra_xudc *xudc;
1590	unsigned long flags;
1591	int ret;
1592
1593	if (!usb_ep)
1594	return -EINVAL;
1595
1596	ep = to_xudc_ep(ep: usb_ep);
1597	xudc = ep->xudc;
1598
1599	spin_lock_irqsave(&xudc->lock, flags);
1600	if (xudc->powergated) {
1601	ret = -ESHUTDOWN;
1602	goto unlock;
1603	}
1604
1605	if (value && usb_endpoint_dir_in(epd: ep->desc) &&
1606	!list_empty(head: &ep->queue)) {
1607	dev_err(xudc->dev, "can't halt EP with requests pending\n");
1608	ret = -EAGAIN;
1609	goto unlock;
1610	}
1611
1612	ret = __tegra_xudc_ep_set_halt(ep, halt: value);
1613	unlock:
1614	spin_unlock_irqrestore(lock: &xudc->lock, flags);
1615
1616	return ret;
1617	}
1618
1619	static void tegra_xudc_ep_context_setup(struct tegra_xudc_ep *ep)
1620	{
1621	const struct usb_endpoint_descriptor *desc = ep->desc;
1622	const struct usb_ss_ep_comp_descriptor *comp_desc = ep->comp_desc;
1623	struct tegra_xudc *xudc = ep->xudc;
1624	u16 maxpacket, maxburst = 0, esit = 0;
1625	u32 val;
1626
1627	maxpacket = usb_endpoint_maxp(epd: desc);
1628	if (xudc->gadget.speed == USB_SPEED_SUPER) {
1629	if (!usb_endpoint_xfer_control(epd: desc))
1630	maxburst = comp_desc->bMaxBurst;
1631
1632	if (usb_endpoint_xfer_int(epd: desc) || usb_endpoint_xfer_isoc(epd: desc))
1633	esit = le16_to_cpu(comp_desc->wBytesPerInterval);
1634	} else if ((xudc->gadget.speed < USB_SPEED_SUPER) &&
1635	(usb_endpoint_xfer_int(epd: desc) ||
1636	usb_endpoint_xfer_isoc(epd: desc))) {
1637	if (xudc->gadget.speed == USB_SPEED_HIGH) {
1638	maxburst = usb_endpoint_maxp_mult(epd: desc) - 1;
1639	if (maxburst == 0x3) {
1640	dev_warn(xudc->dev,
1641	"invalid endpoint maxburst\n");
1642	maxburst = 0x2;
1643	}
1644	}
1645	esit = maxpacket * (maxburst + 1);
1646	}
1647
1648	memset(ep->context, 0, sizeof(*ep->context));
1649
1650	ep_ctx_write_state(ctx: ep->context, EP_STATE_RUNNING);
1651	ep_ctx_write_interval(ctx: ep->context, val: desc->bInterval);
1652	if (xudc->gadget.speed == USB_SPEED_SUPER) {
1653	if (usb_endpoint_xfer_isoc(epd: desc)) {
1654	ep_ctx_write_mult(ctx: ep->context,
1655	val: comp_desc->bmAttributes & 0x3);
1656	}
1657
1658	if (usb_endpoint_xfer_bulk(epd: desc)) {
1659	ep_ctx_write_max_pstreams(ctx: ep->context,
1660	val: comp_desc->bmAttributes &
1661	0x1f);
1662	ep_ctx_write_lsa(ctx: ep->context, val: 1);
1663	}
1664	}
1665
1666	if (!usb_endpoint_xfer_control(epd: desc) && usb_endpoint_dir_out(epd: desc))
1667	val = usb_endpoint_type(epd: desc);
1668	else
1669	val = usb_endpoint_type(epd: desc) + EP_TYPE_CONTROL;
1670
1671	ep_ctx_write_type(ctx: ep->context, val);
1672	ep_ctx_write_cerr(ctx: ep->context, val: 0x3);
1673	ep_ctx_write_max_packet_size(ctx: ep->context, val: maxpacket);
1674	ep_ctx_write_max_burst_size(ctx: ep->context, val: maxburst);
1675
1676	ep_ctx_write_deq_ptr(ctx: ep->context, addr: ep->transfer_ring_phys);
1677	ep_ctx_write_dcs(ctx: ep->context, val: ep->pcs);
1678
1679	/* Select a reasonable average TRB length based on endpoint type. */
1680	switch (usb_endpoint_type(epd: desc)) {
1681	case USB_ENDPOINT_XFER_CONTROL:
1682	val = 8;
1683	break;
1684	case USB_ENDPOINT_XFER_INT:
1685	val = 1024;
1686	break;
1687	case USB_ENDPOINT_XFER_BULK:
1688	case USB_ENDPOINT_XFER_ISOC:
1689	default:
1690	val = 3072;
1691	break;
1692	}
1693
1694	ep_ctx_write_avg_trb_len(ctx: ep->context, val);
1695	ep_ctx_write_max_esit_payload(ctx: ep->context, val: esit);
1696
1697	ep_ctx_write_cerrcnt(ctx: ep->context, val: 0x3);
1698	}
1699
1700	static void setup_link_trb(struct tegra_xudc_ep *ep,
1701	struct tegra_xudc_trb *trb)
1702	{
1703	trb_write_data_ptr(trb, addr: ep->transfer_ring_phys);
1704	trb_write_type(trb, TRB_TYPE_LINK);
1705	trb_write_toggle_cycle(trb, val: 1);
1706	}
1707
1708	static int __tegra_xudc_ep_disable(struct tegra_xudc_ep *ep)
1709	{
1710	struct tegra_xudc *xudc = ep->xudc;
1711
1712	if (ep_ctx_read_state(ctx: ep->context) == EP_STATE_DISABLED) {
1713	dev_err(xudc->dev, "endpoint %u already disabled\n",
1714	ep->index);
1715	return -EINVAL;
1716	}
1717
1718	ep_ctx_write_state(ctx: ep->context, EP_STATE_DISABLED);
1719
1720	ep_reload(xudc, ep: ep->index);
1721
1722	tegra_xudc_ep_nuke(ep, status: -ESHUTDOWN);
1723
1724	xudc->nr_enabled_eps--;
1725	if (usb_endpoint_xfer_isoc(epd: ep->desc))
1726	xudc->nr_isoch_eps--;
1727
1728	ep->desc = NULL;
1729	ep->comp_desc = NULL;
1730
1731	memset(ep->context, 0, sizeof(*ep->context));
1732
1733	ep_unpause(xudc, ep: ep->index);
1734	ep_unhalt(xudc, ep: ep->index);
1735	if (xudc_readl(xudc, EP_STOPPED) & BIT(ep->index))
1736	xudc_writel(xudc, BIT(ep->index), EP_STOPPED);
1737
1738	/*
1739	* If this is the last endpoint disabled in a de-configure request,
1740	* switch back to address state.
1741	*/
1742	if ((xudc->device_state == USB_STATE_CONFIGURED) &&
1743	(xudc->nr_enabled_eps == 1)) {
1744	u32 val;
1745
1746	xudc->device_state = USB_STATE_ADDRESS;
1747	usb_gadget_set_state(gadget: &xudc->gadget, state: xudc->device_state);
1748
1749	val = xudc_readl(xudc, CTRL);
1750	val &= ~CTRL_RUN;
1751	xudc_writel(xudc, val, CTRL);
1752	}
1753
1754	dev_info(xudc->dev, "ep %u disabled\n", ep->index);
1755
1756	return 0;
1757	}
1758
1759	static int tegra_xudc_ep_disable(struct usb_ep *usb_ep)
1760	{
1761	struct tegra_xudc_ep *ep;
1762	struct tegra_xudc *xudc;
1763	unsigned long flags;
1764	int ret;
1765
1766	if (!usb_ep)
1767	return -EINVAL;
1768
1769	ep = to_xudc_ep(ep: usb_ep);
1770	xudc = ep->xudc;
1771
1772	spin_lock_irqsave(&xudc->lock, flags);
1773	if (xudc->powergated) {
1774	ret = -ESHUTDOWN;
1775	goto unlock;
1776	}
1777
1778	ret = __tegra_xudc_ep_disable(ep);
1779	unlock:
1780	spin_unlock_irqrestore(lock: &xudc->lock, flags);
1781
1782	return ret;
1783	}
1784
1785	static int __tegra_xudc_ep_enable(struct tegra_xudc_ep *ep,
1786	const struct usb_endpoint_descriptor *desc)
1787	{
1788	struct tegra_xudc *xudc = ep->xudc;
1789	unsigned int i;
1790	u32 val;
1791
1792	if (xudc->gadget.speed == USB_SPEED_SUPER &&
1793	!usb_endpoint_xfer_control(epd: desc) && !ep->usb_ep.comp_desc)
1794	return -EINVAL;
1795
1796	/* Disable the EP if it is not disabled */
1797	if (ep_ctx_read_state(ctx: ep->context) != EP_STATE_DISABLED)
1798	__tegra_xudc_ep_disable(ep);
1799
1800	ep->desc = desc;
1801	ep->comp_desc = ep->usb_ep.comp_desc;
1802
1803	if (usb_endpoint_xfer_isoc(epd: desc)) {
1804	if (xudc->nr_isoch_eps > XUDC_MAX_ISOCH_EPS) {
1805	dev_err(xudc->dev, "too many isochronous endpoints\n");
1806	return -EBUSY;
1807	}
1808	xudc->nr_isoch_eps++;
1809	}
1810
1811	memset(ep->transfer_ring, 0, XUDC_TRANSFER_RING_SIZE *
1812	sizeof(*ep->transfer_ring));
1813	setup_link_trb(ep, trb: &ep->transfer_ring[XUDC_TRANSFER_RING_SIZE - 1]);
1814
1815	ep->enq_ptr = 0;
1816	ep->deq_ptr = 0;
1817	ep->pcs = true;
1818	ep->ring_full = false;
1819	xudc->nr_enabled_eps++;
1820
1821	tegra_xudc_ep_context_setup(ep);
1822
1823	/*
1824	* No need to reload and un-halt EP0. This will be done automatically
1825	* once a valid SETUP packet is received.
1826	*/
1827	if (usb_endpoint_xfer_control(epd: desc))
1828	goto out;
1829
1830	/*
1831	* Transition to configured state once the first non-control
1832	* endpoint is enabled.
1833	*/
1834	if (xudc->device_state == USB_STATE_ADDRESS) {
1835	val = xudc_readl(xudc, CTRL);
1836	val |= CTRL_RUN;
1837	xudc_writel(xudc, val, CTRL);
1838
1839	xudc->device_state = USB_STATE_CONFIGURED;
1840	usb_gadget_set_state(gadget: &xudc->gadget, state: xudc->device_state);
1841	}
1842
1843	if (usb_endpoint_xfer_isoc(epd: desc)) {
1844	/*
1845	* Pause all bulk endpoints when enabling an isoch endpoint
1846	* to ensure the isoch endpoint is allocated enough bandwidth.
1847	*/
1848	for (i = 0; i < ARRAY_SIZE(xudc->ep); i++) {
1849	if (xudc->ep[i].desc &&
1850	usb_endpoint_xfer_bulk(epd: xudc->ep[i].desc))
1851	ep_pause(xudc, ep: i);
1852	}
1853	}
1854
1855	ep_reload(xudc, ep: ep->index);
1856	ep_unpause(xudc, ep: ep->index);
1857	ep_unhalt(xudc, ep: ep->index);
1858
1859	if (usb_endpoint_xfer_isoc(epd: desc)) {
1860	for (i = 0; i < ARRAY_SIZE(xudc->ep); i++) {
1861	if (xudc->ep[i].desc &&
1862	usb_endpoint_xfer_bulk(epd: xudc->ep[i].desc))
1863	ep_unpause(xudc, ep: i);
1864	}
1865	}
1866
1867	out:
1868	dev_info(xudc->dev, "EP %u (type: %s, dir: %s) enabled\n", ep->index,
1869	usb_ep_type_string(usb_endpoint_type(ep->desc)),
1870	usb_endpoint_dir_in(ep->desc) ? "in" : "out");
1871
1872	return 0;
1873	}
1874
1875	static int tegra_xudc_ep_enable(struct usb_ep *usb_ep,
1876	const struct usb_endpoint_descriptor *desc)
1877	{
1878	struct tegra_xudc_ep *ep;
1879	struct tegra_xudc *xudc;
1880	unsigned long flags;
1881	int ret;
1882
1883	if (!usb_ep || !desc || (desc->bDescriptorType != USB_DT_ENDPOINT))
1884	return -EINVAL;
1885
1886	ep = to_xudc_ep(ep: usb_ep);
1887	xudc = ep->xudc;
1888
1889	spin_lock_irqsave(&xudc->lock, flags);
1890	if (xudc->powergated) {
1891	ret = -ESHUTDOWN;
1892	goto unlock;
1893	}
1894
1895	ret = __tegra_xudc_ep_enable(ep, desc);
1896	unlock:
1897	spin_unlock_irqrestore(lock: &xudc->lock, flags);
1898
1899	return ret;
1900	}
1901
1902	static struct usb_request *
1903	tegra_xudc_ep_alloc_request(struct usb_ep *usb_ep, gfp_t gfp)
1904	{
1905	struct tegra_xudc_request *req;
1906
1907	req = kzalloc(size: sizeof(*req), flags: gfp);
1908	if (!req)
1909	return NULL;
1910
1911	INIT_LIST_HEAD(list: &req->list);
1912
1913	return &req->usb_req;
1914	}
1915
1916	static void tegra_xudc_ep_free_request(struct usb_ep *usb_ep,
1917	struct usb_request *usb_req)
1918	{
1919	struct tegra_xudc_request *req = to_xudc_req(req: usb_req);
1920
1921	kfree(objp: req);
1922	}
1923
1924	static const struct usb_ep_ops tegra_xudc_ep_ops = {
1925	.enable = tegra_xudc_ep_enable,
1926	.disable = tegra_xudc_ep_disable,
1927	.alloc_request = tegra_xudc_ep_alloc_request,
1928	.free_request = tegra_xudc_ep_free_request,
1929	.queue = tegra_xudc_ep_queue,
1930	.dequeue = tegra_xudc_ep_dequeue,
1931	.set_halt = tegra_xudc_ep_set_halt,
1932	};
1933
1934	static int tegra_xudc_ep0_enable(struct usb_ep *usb_ep,
1935	const struct usb_endpoint_descriptor *desc)
1936	{
1937	return -EBUSY;
1938	}
1939
1940	static int tegra_xudc_ep0_disable(struct usb_ep *usb_ep)
1941	{
1942	return -EBUSY;
1943	}
1944
1945	static const struct usb_ep_ops tegra_xudc_ep0_ops = {
1946	.enable = tegra_xudc_ep0_enable,
1947	.disable = tegra_xudc_ep0_disable,
1948	.alloc_request = tegra_xudc_ep_alloc_request,
1949	.free_request = tegra_xudc_ep_free_request,
1950	.queue = tegra_xudc_ep_queue,
1951	.dequeue = tegra_xudc_ep_dequeue,
1952	.set_halt = tegra_xudc_ep_set_halt,
1953	};
1954
1955	static int tegra_xudc_gadget_get_frame(struct usb_gadget *gadget)
1956	{
1957	struct tegra_xudc *xudc = to_xudc(gadget);
1958	unsigned long flags;
1959	int ret;
1960
1961	spin_lock_irqsave(&xudc->lock, flags);
1962	if (xudc->powergated) {
1963	ret = -ESHUTDOWN;
1964	goto unlock;
1965	}
1966
1967	ret = (xudc_readl(xudc, MFINDEX) & MFINDEX_FRAME_MASK) >>
1968	MFINDEX_FRAME_SHIFT;
1969	unlock:
1970	spin_unlock_irqrestore(lock: &xudc->lock, flags);
1971
1972	return ret;
1973	}
1974
1975	static void tegra_xudc_resume_device_state(struct tegra_xudc *xudc)
1976	{
1977	unsigned int i;
1978	u32 val;
1979
1980	ep_unpause_all(xudc);
1981
1982	/* Direct link to U0. */
1983	val = xudc_readl(xudc, PORTSC);
1984	if (((val & PORTSC_PLS_MASK) >> PORTSC_PLS_SHIFT) != PORTSC_PLS_U0) {
1985	val &= ~(PORTSC_CHANGE_MASK | PORTSC_PLS_MASK);
1986	val |= PORTSC_LWS | PORTSC_PLS(PORTSC_PLS_U0);
1987	xudc_writel(xudc, val, PORTSC);
1988	}
1989
1990	if (xudc->device_state == USB_STATE_SUSPENDED) {
1991	xudc->device_state = xudc->resume_state;
1992	usb_gadget_set_state(gadget: &xudc->gadget, state: xudc->device_state);
1993	xudc->resume_state = 0;
1994	}
1995
1996	/*
1997	* Doorbells may be dropped if they are sent too soon (< ~200ns)
1998	* after unpausing the endpoint. Wait for 500ns just to be safe.
1999	*/
2000	ndelay(500);
2001	for (i = 0; i < ARRAY_SIZE(xudc->ep); i++)
2002	tegra_xudc_ep_ring_doorbell(ep: &xudc->ep[i]);
2003	}
2004
2005	static int tegra_xudc_gadget_wakeup(struct usb_gadget *gadget)
2006	{
2007	struct tegra_xudc *xudc = to_xudc(gadget);
2008	unsigned long flags;
2009	int ret = 0;
2010	u32 val;
2011
2012	spin_lock_irqsave(&xudc->lock, flags);
2013
2014	if (xudc->powergated) {
2015	ret = -ESHUTDOWN;
2016	goto unlock;
2017	}
2018	val = xudc_readl(xudc, PORTPM);
2019	dev_dbg(xudc->dev, "%s: PORTPM=%#x, speed=%x\n", __func__,
2020	val, gadget->speed);
2021
2022	if (((xudc->gadget.speed <= USB_SPEED_HIGH) &&
2023	(val & PORTPM_RWE)) ||
2024	((xudc->gadget.speed == USB_SPEED_SUPER) &&
2025	(val & PORTPM_FRWE))) {
2026	tegra_xudc_resume_device_state(xudc);
2027
2028	/* Send Device Notification packet. */
2029	if (xudc->gadget.speed == USB_SPEED_SUPER) {
2030	val = DEVNOTIF_LO_TYPE(DEVNOTIF_LO_TYPE_FUNCTION_WAKE)
2031	| DEVNOTIF_LO_TRIG;
2032	xudc_writel(xudc, val: 0, DEVNOTIF_HI);
2033	xudc_writel(xudc, val, DEVNOTIF_LO);
2034	}
2035	}
2036
2037	unlock:
2038	dev_dbg(xudc->dev, "%s: ret value is %d", __func__, ret);
2039	spin_unlock_irqrestore(lock: &xudc->lock, flags);
2040
2041	return ret;
2042	}
2043
2044	static int tegra_xudc_gadget_pullup(struct usb_gadget *gadget, int is_on)
2045	{
2046	struct tegra_xudc *xudc = to_xudc(gadget);
2047	unsigned long flags;
2048	u32 val;
2049
2050	pm_runtime_get_sync(dev: xudc->dev);
2051
2052	spin_lock_irqsave(&xudc->lock, flags);
2053
2054	if (is_on != xudc->pullup) {
2055	val = xudc_readl(xudc, CTRL);
2056	if (is_on)
2057	val |= CTRL_ENABLE;
2058	else
2059	val &= ~CTRL_ENABLE;
2060	xudc_writel(xudc, val, CTRL);
2061	}
2062
2063	xudc->pullup = is_on;
2064	dev_dbg(xudc->dev, "%s: pullup:%d", __func__, is_on);
2065
2066	spin_unlock_irqrestore(lock: &xudc->lock, flags);
2067
2068	pm_runtime_put(dev: xudc->dev);
2069
2070	return 0;
2071	}
2072
2073	static int tegra_xudc_gadget_start(struct usb_gadget *gadget,
2074	struct usb_gadget_driver *driver)
2075	{
2076	struct tegra_xudc *xudc = to_xudc(gadget);
2077	unsigned long flags;
2078	u32 val;
2079	int ret;
2080	unsigned int i;
2081
2082	if (!driver)
2083	return -EINVAL;
2084
2085	pm_runtime_get_sync(dev: xudc->dev);
2086
2087	spin_lock_irqsave(&xudc->lock, flags);
2088
2089	if (xudc->driver) {
2090	ret = -EBUSY;
2091	goto unlock;
2092	}
2093
2094	xudc->setup_state = WAIT_FOR_SETUP;
2095	xudc->device_state = USB_STATE_DEFAULT;
2096	usb_gadget_set_state(gadget: &xudc->gadget, state: xudc->device_state);
2097
2098	ret = __tegra_xudc_ep_enable(ep: &xudc->ep[0], desc: &tegra_xudc_ep0_desc);
2099	if (ret < 0)
2100	goto unlock;
2101
2102	val = xudc_readl(xudc, CTRL);
2103	val |= CTRL_IE | CTRL_LSE;
2104	xudc_writel(xudc, val, CTRL);
2105
2106	val = xudc_readl(xudc, PORTHALT);
2107	val |= PORTHALT_STCHG_INTR_EN;
2108	xudc_writel(xudc, val, PORTHALT);
2109
2110	if (xudc->pullup) {
2111	val = xudc_readl(xudc, CTRL);
2112	val |= CTRL_ENABLE;
2113	xudc_writel(xudc, val, CTRL);
2114	}
2115
2116	for (i = 0; i < xudc->soc->num_phys; i++)
2117	if (xudc->usbphy[i])
2118	otg_set_peripheral(otg: xudc->usbphy[i]->otg, periph: gadget);
2119
2120	xudc->driver = driver;
2121	unlock:
2122	dev_dbg(xudc->dev, "%s: ret value is %d", __func__, ret);
2123	spin_unlock_irqrestore(lock: &xudc->lock, flags);
2124
2125	pm_runtime_put(dev: xudc->dev);
2126
2127	return ret;
2128	}
2129
2130	static int tegra_xudc_gadget_stop(struct usb_gadget *gadget)
2131	{
2132	struct tegra_xudc *xudc = to_xudc(gadget);
2133	unsigned long flags;
2134	u32 val;
2135	unsigned int i;
2136
2137	pm_runtime_get_sync(dev: xudc->dev);
2138
2139	spin_lock_irqsave(&xudc->lock, flags);
2140
2141	for (i = 0; i < xudc->soc->num_phys; i++)
2142	if (xudc->usbphy[i])
2143	otg_set_peripheral(otg: xudc->usbphy[i]->otg, NULL);
2144
2145	val = xudc_readl(xudc, CTRL);
2146	val &= ~(CTRL_IE | CTRL_ENABLE);
2147	xudc_writel(xudc, val, CTRL);
2148
2149	__tegra_xudc_ep_disable(ep: &xudc->ep[0]);
2150
2151	xudc->driver = NULL;
2152	dev_dbg(xudc->dev, "Gadget stopped");
2153
2154	spin_unlock_irqrestore(lock: &xudc->lock, flags);
2155
2156	pm_runtime_put(dev: xudc->dev);
2157
2158	return 0;
2159	}
2160
2161	static int tegra_xudc_gadget_vbus_draw(struct usb_gadget *gadget,
2162	unsigned int m_a)
2163	{
2164	struct tegra_xudc *xudc = to_xudc(gadget);
2165
2166	dev_dbg(xudc->dev, "%s: %u mA\n", __func__, m_a);
2167
2168	if (xudc->curr_usbphy && xudc->curr_usbphy->chg_type == SDP_TYPE)
2169	return usb_phy_set_power(x: xudc->curr_usbphy, mA: m_a);
2170
2171	return 0;
2172	}
2173
2174	static int tegra_xudc_set_selfpowered(struct usb_gadget *gadget, int is_on)
2175	{
2176	struct tegra_xudc *xudc = to_xudc(gadget);
2177
2178	dev_dbg(xudc->dev, "%s: %d\n", __func__, is_on);
2179	xudc->selfpowered = !!is_on;
2180
2181	return 0;
2182	}
2183
2184	static const struct usb_gadget_ops tegra_xudc_gadget_ops = {
2185	.get_frame = tegra_xudc_gadget_get_frame,
2186	.wakeup = tegra_xudc_gadget_wakeup,
2187	.pullup = tegra_xudc_gadget_pullup,
2188	.udc_start = tegra_xudc_gadget_start,
2189	.udc_stop = tegra_xudc_gadget_stop,
2190	.vbus_draw = tegra_xudc_gadget_vbus_draw,
2191	.set_selfpowered = tegra_xudc_set_selfpowered,
2192	};
2193
2194	static void no_op_complete(struct usb_ep *ep, struct usb_request *req)
2195	{
2196	}
2197
2198	static int
2199	tegra_xudc_ep0_queue_status(struct tegra_xudc *xudc,
2200	void (*cmpl)(struct usb_ep *, struct usb_request *))
2201	{
2202	xudc->ep0_req->usb_req.buf = NULL;
2203	xudc->ep0_req->usb_req.dma = 0;
2204	xudc->ep0_req->usb_req.length = 0;
2205	xudc->ep0_req->usb_req.complete = cmpl;
2206	xudc->ep0_req->usb_req.context = xudc;
2207
2208	return __tegra_xudc_ep_queue(ep: &xudc->ep[0], req: xudc->ep0_req);
2209	}
2210
2211	static int
2212	tegra_xudc_ep0_queue_data(struct tegra_xudc *xudc, void *buf, size_t len,
2213	void (*cmpl)(struct usb_ep *, struct usb_request *))
2214	{
2215	xudc->ep0_req->usb_req.buf = buf;
2216	xudc->ep0_req->usb_req.length = len;
2217	xudc->ep0_req->usb_req.complete = cmpl;
2218	xudc->ep0_req->usb_req.context = xudc;
2219
2220	return __tegra_xudc_ep_queue(ep: &xudc->ep[0], req: xudc->ep0_req);
2221	}
2222
2223	static void tegra_xudc_ep0_req_done(struct tegra_xudc *xudc)
2224	{
2225	switch (xudc->setup_state) {
2226	case DATA_STAGE_XFER:
2227	xudc->setup_state = STATUS_STAGE_RECV;
2228	tegra_xudc_ep0_queue_status(xudc, cmpl: no_op_complete);
2229	break;
2230	case DATA_STAGE_RECV:
2231	xudc->setup_state = STATUS_STAGE_XFER;
2232	tegra_xudc_ep0_queue_status(xudc, cmpl: no_op_complete);
2233	break;
2234	default:
2235	xudc->setup_state = WAIT_FOR_SETUP;
2236	break;
2237	}
2238	}
2239
2240	static int tegra_xudc_ep0_delegate_req(struct tegra_xudc *xudc,
2241	struct usb_ctrlrequest *ctrl)
2242	{
2243	int ret;
2244
2245	spin_unlock(lock: &xudc->lock);
2246	ret = xudc->driver->setup(&xudc->gadget, ctrl);
2247	spin_lock(lock: &xudc->lock);
2248
2249	return ret;
2250	}
2251
2252	static void set_feature_complete(struct usb_ep *ep, struct usb_request *req)
2253	{
2254	struct tegra_xudc *xudc = req->context;
2255
2256	if (xudc->test_mode_pattern) {
2257	xudc_writel(xudc, val: xudc->test_mode_pattern, PORT_TM);
2258	xudc->test_mode_pattern = 0;
2259	}
2260	}
2261
2262	static int tegra_xudc_ep0_set_feature(struct tegra_xudc *xudc,
2263	struct usb_ctrlrequest *ctrl)
2264	{
2265	bool set = (ctrl->bRequest == USB_REQ_SET_FEATURE);
2266	u32 feature = le16_to_cpu(ctrl->wValue);
2267	u32 index = le16_to_cpu(ctrl->wIndex);
2268	u32 val, ep;
2269	int ret;
2270
2271	if (le16_to_cpu(ctrl->wLength) != 0)
2272	return -EINVAL;
2273
2274	switch (ctrl->bRequestType & USB_RECIP_MASK) {
2275	case USB_RECIP_DEVICE:
2276	switch (feature) {
2277	case USB_DEVICE_REMOTE_WAKEUP:
2278	if ((xudc->gadget.speed == USB_SPEED_SUPER) ||
2279	(xudc->device_state == USB_STATE_DEFAULT))
2280	return -EINVAL;
2281
2282	val = xudc_readl(xudc, PORTPM);
2283	if (set)
2284	val |= PORTPM_RWE;
2285	else
2286	val &= ~PORTPM_RWE;
2287
2288	xudc_writel(xudc, val, PORTPM);
2289	break;
2290	case USB_DEVICE_U1_ENABLE:
2291	case USB_DEVICE_U2_ENABLE:
2292	if ((xudc->device_state != USB_STATE_CONFIGURED) ||
2293	(xudc->gadget.speed != USB_SPEED_SUPER))
2294	return -EINVAL;
2295
2296	val = xudc_readl(xudc, PORTPM);
2297	if ((feature == USB_DEVICE_U1_ENABLE) &&
2298	xudc->soc->u1_enable) {
2299	if (set)
2300	val |= PORTPM_U1E;
2301	else
2302	val &= ~PORTPM_U1E;
2303	}
2304
2305	if ((feature == USB_DEVICE_U2_ENABLE) &&
2306	xudc->soc->u2_enable) {
2307	if (set)
2308	val |= PORTPM_U2E;
2309	else
2310	val &= ~PORTPM_U2E;
2311	}
2312
2313	xudc_writel(xudc, val, PORTPM);
2314	break;
2315	case USB_DEVICE_TEST_MODE:
2316	if (xudc->gadget.speed != USB_SPEED_HIGH)
2317	return -EINVAL;
2318
2319	if (!set)
2320	return -EINVAL;
2321
2322	xudc->test_mode_pattern = index >> 8;
2323	break;
2324	default:
2325	return -EINVAL;
2326	}
2327
2328	break;
2329	case USB_RECIP_INTERFACE:
2330	if (xudc->device_state != USB_STATE_CONFIGURED)
2331	return -EINVAL;
2332
2333	switch (feature) {
2334	case USB_INTRF_FUNC_SUSPEND:
2335	if (set) {
2336	val = xudc_readl(xudc, PORTPM);
2337
2338	if (index & USB_INTRF_FUNC_SUSPEND_RW)
2339	val |= PORTPM_FRWE;
2340	else
2341	val &= ~PORTPM_FRWE;
2342
2343	xudc_writel(xudc, val, PORTPM);
2344	}
2345
2346	return tegra_xudc_ep0_delegate_req(xudc, ctrl);
2347	default:
2348	return -EINVAL;
2349	}
2350
2351	break;
2352	case USB_RECIP_ENDPOINT:
2353	ep = (index & USB_ENDPOINT_NUMBER_MASK) * 2 +
2354	((index & USB_DIR_IN) ? 1 : 0);
2355
2356	if ((xudc->device_state == USB_STATE_DEFAULT) ||
2357	((xudc->device_state == USB_STATE_ADDRESS) &&
2358	(index != 0)))
2359	return -EINVAL;
2360
2361	ret = __tegra_xudc_ep_set_halt(ep: &xudc->ep[ep], halt: set);
2362	if (ret < 0)
2363	return ret;
2364	break;
2365	default:
2366	return -EINVAL;
2367	}
2368
2369	return tegra_xudc_ep0_queue_status(xudc, cmpl: set_feature_complete);
2370	}
2371
2372	static int tegra_xudc_ep0_get_status(struct tegra_xudc *xudc,
2373	struct usb_ctrlrequest *ctrl)
2374	{
2375	struct tegra_xudc_ep_context *ep_ctx;
2376	u32 val, ep, index = le16_to_cpu(ctrl->wIndex);
2377	u16 status = 0;
2378
2379	if (!(ctrl->bRequestType & USB_DIR_IN))
2380	return -EINVAL;
2381
2382	if ((le16_to_cpu(ctrl->wValue) != 0) ||
2383	(le16_to_cpu(ctrl->wLength) != 2))
2384	return -EINVAL;
2385
2386	switch (ctrl->bRequestType & USB_RECIP_MASK) {
2387	case USB_RECIP_DEVICE:
2388	val = xudc_readl(xudc, PORTPM);
2389
2390	if (xudc->selfpowered)
2391	status |= BIT(USB_DEVICE_SELF_POWERED);
2392
2393	if ((xudc->gadget.speed < USB_SPEED_SUPER) &&
2394	(val & PORTPM_RWE))
2395	status |= BIT(USB_DEVICE_REMOTE_WAKEUP);
2396
2397	if (xudc->gadget.speed == USB_SPEED_SUPER) {
2398	if (val & PORTPM_U1E)
2399	status |= BIT(USB_DEV_STAT_U1_ENABLED);
2400	if (val & PORTPM_U2E)
2401	status |= BIT(USB_DEV_STAT_U2_ENABLED);
2402	}
2403	break;
2404	case USB_RECIP_INTERFACE:
2405	if (xudc->gadget.speed == USB_SPEED_SUPER) {
2406	status |= USB_INTRF_STAT_FUNC_RW_CAP;
2407	val = xudc_readl(xudc, PORTPM);
2408	if (val & PORTPM_FRWE)
2409	status |= USB_INTRF_STAT_FUNC_RW;
2410	}
2411	break;
2412	case USB_RECIP_ENDPOINT:
2413	ep = (index & USB_ENDPOINT_NUMBER_MASK) * 2 +
2414	((index & USB_DIR_IN) ? 1 : 0);
2415	ep_ctx = &xudc->ep_context[ep];
2416
2417	if ((xudc->device_state != USB_STATE_CONFIGURED) &&
2418	((xudc->device_state != USB_STATE_ADDRESS) || (ep != 0)))
2419	return -EINVAL;
2420
2421	if (ep_ctx_read_state(ctx: ep_ctx) == EP_STATE_DISABLED)
2422	return -EINVAL;
2423
2424	if (xudc_readl(xudc, EP_HALT) & BIT(ep))
2425	status |= BIT(USB_ENDPOINT_HALT);
2426	break;
2427	default:
2428	return -EINVAL;
2429	}
2430
2431	xudc->status_buf = cpu_to_le16(status);
2432	return tegra_xudc_ep0_queue_data(xudc, buf: &xudc->status_buf,
2433	len: sizeof(xudc->status_buf),
2434	cmpl: no_op_complete);
2435	}
2436
2437	static void set_sel_complete(struct usb_ep *ep, struct usb_request *req)
2438	{
2439	/* Nothing to do with SEL values */
2440	}
2441
2442	static int tegra_xudc_ep0_set_sel(struct tegra_xudc *xudc,
2443	struct usb_ctrlrequest *ctrl)
2444	{
2445	if (ctrl->bRequestType != (USB_DIR_OUT | USB_RECIP_DEVICE |
2446	USB_TYPE_STANDARD))
2447	return -EINVAL;
2448
2449	if (xudc->device_state == USB_STATE_DEFAULT)
2450	return -EINVAL;
2451
2452	if ((le16_to_cpu(ctrl->wIndex) != 0) ||
2453	(le16_to_cpu(ctrl->wValue) != 0) ||
2454	(le16_to_cpu(ctrl->wLength) != 6))
2455	return -EINVAL;
2456
2457	return tegra_xudc_ep0_queue_data(xudc, buf: &xudc->sel_timing,
2458	len: sizeof(xudc->sel_timing),
2459	cmpl: set_sel_complete);
2460	}
2461
2462	static void set_isoch_delay_complete(struct usb_ep *ep, struct usb_request *req)
2463	{
2464	/* Nothing to do with isoch delay */
2465	}
2466
2467	static int tegra_xudc_ep0_set_isoch_delay(struct tegra_xudc *xudc,
2468	struct usb_ctrlrequest *ctrl)
2469	{
2470	u32 delay = le16_to_cpu(ctrl->wValue);
2471
2472	if (ctrl->bRequestType != (USB_DIR_OUT | USB_RECIP_DEVICE |
2473	USB_TYPE_STANDARD))
2474	return -EINVAL;
2475
2476	if ((delay > 65535) || (le16_to_cpu(ctrl->wIndex) != 0) ||
2477	(le16_to_cpu(ctrl->wLength) != 0))
2478	return -EINVAL;
2479
2480	xudc->isoch_delay = delay;
2481
2482	return tegra_xudc_ep0_queue_status(xudc, cmpl: set_isoch_delay_complete);
2483	}
2484
2485	static void set_address_complete(struct usb_ep *ep, struct usb_request *req)
2486	{
2487	struct tegra_xudc *xudc = req->context;
2488
2489	if ((xudc->device_state == USB_STATE_DEFAULT) &&
2490	(xudc->dev_addr != 0)) {
2491	xudc->device_state = USB_STATE_ADDRESS;
2492	usb_gadget_set_state(gadget: &xudc->gadget, state: xudc->device_state);
2493	} else if ((xudc->device_state == USB_STATE_ADDRESS) &&
2494	(xudc->dev_addr == 0)) {
2495	xudc->device_state = USB_STATE_DEFAULT;
2496	usb_gadget_set_state(gadget: &xudc->gadget, state: xudc->device_state);
2497	}
2498	}
2499
2500	static int tegra_xudc_ep0_set_address(struct tegra_xudc *xudc,
2501	struct usb_ctrlrequest *ctrl)
2502	{
2503	struct tegra_xudc_ep *ep0 = &xudc->ep[0];
2504	u32 val, addr = le16_to_cpu(ctrl->wValue);
2505
2506	if (ctrl->bRequestType != (USB_DIR_OUT | USB_RECIP_DEVICE |
2507	USB_TYPE_STANDARD))
2508	return -EINVAL;
2509
2510	if ((addr > 127) || (le16_to_cpu(ctrl->wIndex) != 0) ||
2511	(le16_to_cpu(ctrl->wLength) != 0))
2512	return -EINVAL;
2513
2514	if (xudc->device_state == USB_STATE_CONFIGURED)
2515	return -EINVAL;
2516
2517	dev_dbg(xudc->dev, "set address: %u\n", addr);
2518
2519	xudc->dev_addr = addr;
2520	val = xudc_readl(xudc, CTRL);
2521	val &= ~(CTRL_DEVADDR_MASK);
2522	val |= CTRL_DEVADDR(addr);
2523	xudc_writel(xudc, val, CTRL);
2524
2525	ep_ctx_write_devaddr(ctx: ep0->context, val: addr);
2526
2527	return tegra_xudc_ep0_queue_status(xudc, cmpl: set_address_complete);
2528	}
2529
2530	static int tegra_xudc_ep0_standard_req(struct tegra_xudc *xudc,
2531	struct usb_ctrlrequest *ctrl)
2532	{
2533	int ret;
2534
2535	switch (ctrl->bRequest) {
2536	case USB_REQ_GET_STATUS:
2537	dev_dbg(xudc->dev, "USB_REQ_GET_STATUS\n");
2538	ret = tegra_xudc_ep0_get_status(xudc, ctrl);
2539	break;
2540	case USB_REQ_SET_ADDRESS:
2541	dev_dbg(xudc->dev, "USB_REQ_SET_ADDRESS\n");
2542	ret = tegra_xudc_ep0_set_address(xudc, ctrl);
2543	break;
2544	case USB_REQ_SET_SEL:
2545	dev_dbg(xudc->dev, "USB_REQ_SET_SEL\n");
2546	ret = tegra_xudc_ep0_set_sel(xudc, ctrl);
2547	break;
2548	case USB_REQ_SET_ISOCH_DELAY:
2549	dev_dbg(xudc->dev, "USB_REQ_SET_ISOCH_DELAY\n");
2550	ret = tegra_xudc_ep0_set_isoch_delay(xudc, ctrl);
2551	break;
2552	case USB_REQ_CLEAR_FEATURE:
2553	case USB_REQ_SET_FEATURE:
2554	dev_dbg(xudc->dev, "USB_REQ_CLEAR/SET_FEATURE\n");
2555	ret = tegra_xudc_ep0_set_feature(xudc, ctrl);
2556	break;
2557	case USB_REQ_SET_CONFIGURATION:
2558	dev_dbg(xudc->dev, "USB_REQ_SET_CONFIGURATION\n");
2559	/*
2560	* In theory we need to clear RUN bit before status stage of
2561	* deconfig request sent, but this seems to be causing problems.
2562	* Clear RUN once all endpoints are disabled instead.
2563	*/
2564	fallthrough;
2565	default:
2566	ret = tegra_xudc_ep0_delegate_req(xudc, ctrl);
2567	break;
2568	}
2569
2570	return ret;
2571	}
2572
2573	static void tegra_xudc_handle_ep0_setup_packet(struct tegra_xudc *xudc,
2574	struct usb_ctrlrequest *ctrl,
2575	u16 seq_num)
2576	{
2577	int ret;
2578
2579	xudc->setup_seq_num = seq_num;
2580
2581	/* Ensure EP0 is unhalted. */
2582	ep_unhalt(xudc, ep: 0);
2583
2584	/*
2585	* On Tegra210, setup packets with sequence numbers 0xfffe or 0xffff
2586	* are invalid. Halt EP0 until we get a valid packet.
2587	*/
2588	if (xudc->soc->invalid_seq_num &&
2589	(seq_num == 0xfffe || seq_num == 0xffff)) {
2590	dev_warn(xudc->dev, "invalid sequence number detected\n");
2591	ep_halt(xudc, ep: 0);
2592	return;
2593	}
2594
2595	if (ctrl->wLength)
2596	xudc->setup_state = (ctrl->bRequestType & USB_DIR_IN) ?
2597	DATA_STAGE_XFER : DATA_STAGE_RECV;
2598	else
2599	xudc->setup_state = STATUS_STAGE_XFER;
2600
2601	if ((ctrl->bRequestType & USB_TYPE_MASK) == USB_TYPE_STANDARD)
2602	ret = tegra_xudc_ep0_standard_req(xudc, ctrl);
2603	else
2604	ret = tegra_xudc_ep0_delegate_req(xudc, ctrl);
2605
2606	if (ret < 0) {
2607	dev_warn(xudc->dev, "setup request failed: %d\n", ret);
2608	xudc->setup_state = WAIT_FOR_SETUP;
2609	ep_halt(xudc, ep: 0);
2610	}
2611	}
2612
2613	static void tegra_xudc_handle_ep0_event(struct tegra_xudc *xudc,
2614	struct tegra_xudc_trb *event)
2615	{
2616	struct usb_ctrlrequest *ctrl = (struct usb_ctrlrequest *)event;
2617	u16 seq_num = trb_read_seq_num(trb: event);
2618
2619	if (xudc->setup_state != WAIT_FOR_SETUP) {
2620	/*
2621	* The controller is in the process of handling another
2622	* setup request. Queue subsequent requests and handle
2623	* the last one once the controller reports a sequence
2624	* number error.
2625	*/
2626	memcpy(&xudc->setup_packet.ctrl_req, ctrl, sizeof(*ctrl));
2627	xudc->setup_packet.seq_num = seq_num;
2628	xudc->queued_setup_packet = true;
2629	} else {
2630	tegra_xudc_handle_ep0_setup_packet(xudc, ctrl, seq_num);
2631	}
2632	}
2633
2634	static struct tegra_xudc_request *
2635	trb_to_request(struct tegra_xudc_ep *ep, struct tegra_xudc_trb *trb)
2636	{
2637	struct tegra_xudc_request *req;
2638
2639	list_for_each_entry(req, &ep->queue, list) {
2640	if (!req->trbs_queued)
2641	break;
2642
2643	if (trb_in_request(ep, req, trb))
2644	return req;
2645	}
2646
2647	return NULL;
2648	}
2649
2650	static void tegra_xudc_handle_transfer_completion(struct tegra_xudc *xudc,
2651	struct tegra_xudc_ep *ep,
2652	struct tegra_xudc_trb *event)
2653	{
2654	struct tegra_xudc_request *req;
2655	struct tegra_xudc_trb *trb;
2656	bool short_packet;
2657
2658	short_packet = (trb_read_cmpl_code(trb: event) ==
2659	TRB_CMPL_CODE_SHORT_PACKET);
2660
2661	trb = trb_phys_to_virt(ep, addr: trb_read_data_ptr(trb: event));
2662	req = trb_to_request(ep, trb);
2663
2664	/*
2665	* TDs are complete on short packet or when the completed TRB is the
2666	* last TRB in the TD (the CHAIN bit is unset).
2667	*/
2668	if (req && (short_packet || (!trb_read_chain(trb) &&
2669	(req->trbs_needed == req->trbs_queued)))) {
2670	struct tegra_xudc_trb *last = req->last_trb;
2671	unsigned int residual;
2672
2673	residual = trb_read_transfer_len(trb: event);
2674	req->usb_req.actual = req->usb_req.length - residual;
2675
2676	dev_dbg(xudc->dev, "bytes transferred %u / %u\n",
2677	req->usb_req.actual, req->usb_req.length);
2678
2679	tegra_xudc_req_done(ep, req, status: 0);
2680
2681	if (ep->desc && usb_endpoint_xfer_control(epd: ep->desc))
2682	tegra_xudc_ep0_req_done(xudc);
2683
2684	/*
2685	* Advance the dequeue pointer past the end of the current TD
2686	* on short packet completion.
2687	*/
2688	if (short_packet) {
2689	ep->deq_ptr = (last - ep->transfer_ring) + 1;
2690	if (ep->deq_ptr == XUDC_TRANSFER_RING_SIZE - 1)
2691	ep->deq_ptr = 0;
2692	}
2693	} else if (!req) {
2694	dev_warn(xudc->dev, "transfer event on dequeued request\n");
2695	}
2696
2697	if (ep->desc)
2698	tegra_xudc_ep_kick_queue(ep);
2699	}
2700
2701	static void tegra_xudc_handle_transfer_event(struct tegra_xudc *xudc,
2702	struct tegra_xudc_trb *event)
2703	{
2704	unsigned int ep_index = trb_read_endpoint_id(trb: event);
2705	struct tegra_xudc_ep *ep = &xudc->ep[ep_index];
2706	struct tegra_xudc_trb *trb;
2707	u16 comp_code;
2708
2709	if (ep_ctx_read_state(ctx: ep->context) == EP_STATE_DISABLED) {
2710	dev_warn(xudc->dev, "transfer event on disabled EP %u\n",
2711	ep_index);
2712	return;
2713	}
2714
2715	/* Update transfer ring dequeue pointer. */
2716	trb = trb_phys_to_virt(ep, addr: trb_read_data_ptr(trb: event));
2717	comp_code = trb_read_cmpl_code(trb: event);
2718	if (comp_code != TRB_CMPL_CODE_BABBLE_DETECTED_ERR) {
2719	ep->deq_ptr = (trb - ep->transfer_ring) + 1;
2720
2721	if (ep->deq_ptr == XUDC_TRANSFER_RING_SIZE - 1)
2722	ep->deq_ptr = 0;
2723	ep->ring_full = false;
2724	}
2725
2726	switch (comp_code) {
2727	case TRB_CMPL_CODE_SUCCESS:
2728	case TRB_CMPL_CODE_SHORT_PACKET:
2729	tegra_xudc_handle_transfer_completion(xudc, ep, event);
2730	break;
2731	case TRB_CMPL_CODE_HOST_REJECTED:
2732	dev_info(xudc->dev, "stream rejected on EP %u\n", ep_index);
2733
2734	ep->stream_rejected = true;
2735	break;
2736	case TRB_CMPL_CODE_PRIME_PIPE_RECEIVED:
2737	dev_info(xudc->dev, "prime pipe received on EP %u\n", ep_index);
2738
2739	if (ep->stream_rejected) {
2740	ep->stream_rejected = false;
2741	/*
2742	* An EP is stopped when a stream is rejected. Wait
2743	* for the EP to report that it is stopped and then
2744	* un-stop it.
2745	*/
2746	ep_wait_for_stopped(xudc, ep: ep_index);
2747	}
2748	tegra_xudc_ep_ring_doorbell(ep);
2749	break;
2750	case TRB_CMPL_CODE_BABBLE_DETECTED_ERR:
2751	/*
2752	* Wait for the EP to be stopped so the controller stops
2753	* processing doorbells.
2754	*/
2755	ep_wait_for_stopped(xudc, ep: ep_index);
2756	ep->enq_ptr = ep->deq_ptr;
2757	tegra_xudc_ep_nuke(ep, status: -EIO);
2758	fallthrough;
2759	case TRB_CMPL_CODE_STREAM_NUMP_ERROR:
2760	case TRB_CMPL_CODE_CTRL_DIR_ERR:
2761	case TRB_CMPL_CODE_INVALID_STREAM_TYPE_ERR:
2762	case TRB_CMPL_CODE_RING_UNDERRUN:
2763	case TRB_CMPL_CODE_RING_OVERRUN:
2764	case TRB_CMPL_CODE_ISOCH_BUFFER_OVERRUN:
2765	case TRB_CMPL_CODE_USB_TRANS_ERR:
2766	case TRB_CMPL_CODE_TRB_ERR:
2767	dev_err(xudc->dev, "completion error %#x on EP %u\n",
2768	comp_code, ep_index);
2769
2770	ep_halt(xudc, ep: ep_index);
2771	break;
2772	case TRB_CMPL_CODE_CTRL_SEQNUM_ERR:
2773	dev_info(xudc->dev, "sequence number error\n");
2774
2775	/*
2776	* Kill any queued control request and skip to the last
2777	* setup packet we received.
2778	*/
2779	tegra_xudc_ep_nuke(ep, status: -EINVAL);
2780	xudc->setup_state = WAIT_FOR_SETUP;
2781	if (!xudc->queued_setup_packet)
2782	break;
2783
2784	tegra_xudc_handle_ep0_setup_packet(xudc,
2785	ctrl: &xudc->setup_packet.ctrl_req,
2786	seq_num: xudc->setup_packet.seq_num);
2787	xudc->queued_setup_packet = false;
2788	break;
2789	case TRB_CMPL_CODE_STOPPED:
2790	dev_dbg(xudc->dev, "stop completion code on EP %u\n",
2791	ep_index);
2792
2793	/* Disconnected. */
2794	tegra_xudc_ep_nuke(ep, status: -ECONNREFUSED);
2795	break;
2796	default:
2797	dev_dbg(xudc->dev, "completion event %#x on EP %u\n",
2798	comp_code, ep_index);
2799	break;
2800	}
2801	}
2802
2803	static void tegra_xudc_reset(struct tegra_xudc *xudc)
2804	{
2805	struct tegra_xudc_ep *ep0 = &xudc->ep[0];
2806	dma_addr_t deq_ptr;
2807	unsigned int i;
2808
2809	xudc->setup_state = WAIT_FOR_SETUP;
2810	xudc->device_state = USB_STATE_DEFAULT;
2811	usb_gadget_set_state(gadget: &xudc->gadget, state: xudc->device_state);
2812
2813	ep_unpause_all(xudc);
2814
2815	for (i = 0; i < ARRAY_SIZE(xudc->ep); i++)
2816	tegra_xudc_ep_nuke(ep: &xudc->ep[i], status: -ESHUTDOWN);
2817
2818	/*
2819	* Reset sequence number and dequeue pointer to flush the transfer
2820	* ring.
2821	*/
2822	ep0->deq_ptr = ep0->enq_ptr;
2823	ep0->ring_full = false;
2824
2825	xudc->setup_seq_num = 0;
2826	xudc->queued_setup_packet = false;
2827
2828	ep_ctx_write_rsvd(ctx: ep0->context, val: 0);
2829	ep_ctx_write_partial_td(ctx: ep0->context, val: 0);
2830	ep_ctx_write_splitxstate(ctx: ep0->context, val: 0);
2831	ep_ctx_write_seq_num(ctx: ep0->context, val: 0);
2832
2833	deq_ptr = trb_virt_to_phys(ep: ep0, trb: &ep0->transfer_ring[ep0->deq_ptr]);
2834
2835	if (!dma_mapping_error(dev: xudc->dev, dma_addr: deq_ptr)) {
2836	ep_ctx_write_deq_ptr(ctx: ep0->context, addr: deq_ptr);
2837	ep_ctx_write_dcs(ctx: ep0->context, val: ep0->pcs);
2838	}
2839
2840	ep_unhalt_all(xudc);
2841	ep_reload(xudc, ep: 0);
2842	ep_unpause(xudc, ep: 0);
2843	}
2844
2845	static void tegra_xudc_port_connect(struct tegra_xudc *xudc)
2846	{
2847	struct tegra_xudc_ep *ep0 = &xudc->ep[0];
2848	u16 maxpacket;
2849	u32 val;
2850
2851	val = (xudc_readl(xudc, PORTSC) & PORTSC_PS_MASK) >> PORTSC_PS_SHIFT;
2852	switch (val) {
2853	case PORTSC_PS_LS:
2854	xudc->gadget.speed = USB_SPEED_LOW;
2855	break;
2856	case PORTSC_PS_FS:
2857	xudc->gadget.speed = USB_SPEED_FULL;
2858	break;
2859	case PORTSC_PS_HS:
2860	xudc->gadget.speed = USB_SPEED_HIGH;
2861	break;
2862	case PORTSC_PS_SS:
2863	xudc->gadget.speed = USB_SPEED_SUPER;
2864	break;
2865	default:
2866	xudc->gadget.speed = USB_SPEED_UNKNOWN;
2867	break;
2868	}
2869
2870	xudc->device_state = USB_STATE_DEFAULT;
2871	usb_gadget_set_state(gadget: &xudc->gadget, state: xudc->device_state);
2872
2873	xudc->setup_state = WAIT_FOR_SETUP;
2874
2875	if (xudc->gadget.speed == USB_SPEED_SUPER)
2876	maxpacket = 512;
2877	else
2878	maxpacket = 64;
2879
2880	ep_ctx_write_max_packet_size(ctx: ep0->context, val: maxpacket);
2881	tegra_xudc_ep0_desc.wMaxPacketSize = cpu_to_le16(maxpacket);
2882	usb_ep_set_maxpacket_limit(ep: &ep0->usb_ep, maxpacket_limit: maxpacket);
2883
2884	if (!xudc->soc->u1_enable) {
2885	val = xudc_readl(xudc, PORTPM);
2886	val &= ~(PORTPM_U1TIMEOUT_MASK);
2887	xudc_writel(xudc, val, PORTPM);
2888	}
2889
2890	if (!xudc->soc->u2_enable) {
2891	val = xudc_readl(xudc, PORTPM);
2892	val &= ~(PORTPM_U2TIMEOUT_MASK);
2893	xudc_writel(xudc, val, PORTPM);
2894	}
2895
2896	if (xudc->gadget.speed <= USB_SPEED_HIGH) {
2897	val = xudc_readl(xudc, PORTPM);
2898	val &= ~(PORTPM_L1S_MASK);
2899	if (xudc->soc->lpm_enable)
2900	val |= PORTPM_L1S(PORTPM_L1S_ACCEPT);
2901	else
2902	val |= PORTPM_L1S(PORTPM_L1S_NYET);
2903	xudc_writel(xudc, val, PORTPM);
2904	}
2905
2906	val = xudc_readl(xudc, ST);
2907	if (val & ST_RC)
2908	xudc_writel(xudc, ST_RC, ST);
2909	}
2910
2911	static void tegra_xudc_port_disconnect(struct tegra_xudc *xudc)
2912	{
2913	tegra_xudc_reset(xudc);
2914
2915	if (xudc->driver && xudc->driver->disconnect) {
2916	spin_unlock(lock: &xudc->lock);
2917	xudc->driver->disconnect(&xudc->gadget);
2918	spin_lock(lock: &xudc->lock);
2919	}
2920
2921	xudc->device_state = USB_STATE_NOTATTACHED;
2922	usb_gadget_set_state(gadget: &xudc->gadget, state: xudc->device_state);
2923
2924	complete(&xudc->disconnect_complete);
2925	}
2926
2927	static void tegra_xudc_port_reset(struct tegra_xudc *xudc)
2928	{
2929	tegra_xudc_reset(xudc);
2930
2931	if (xudc->driver) {
2932	spin_unlock(lock: &xudc->lock);
2933	usb_gadget_udc_reset(gadget: &xudc->gadget, driver: xudc->driver);
2934	spin_lock(lock: &xudc->lock);
2935	}
2936
2937	tegra_xudc_port_connect(xudc);
2938	}
2939
2940	static void tegra_xudc_port_suspend(struct tegra_xudc *xudc)
2941	{
2942	dev_dbg(xudc->dev, "port suspend\n");
2943
2944	xudc->resume_state = xudc->device_state;
2945	xudc->device_state = USB_STATE_SUSPENDED;
2946	usb_gadget_set_state(gadget: &xudc->gadget, state: xudc->device_state);
2947
2948	if (xudc->driver->suspend) {
2949	spin_unlock(lock: &xudc->lock);
2950	xudc->driver->suspend(&xudc->gadget);
2951	spin_lock(lock: &xudc->lock);
2952	}
2953	}
2954
2955	static void tegra_xudc_port_resume(struct tegra_xudc *xudc)
2956	{
2957	dev_dbg(xudc->dev, "port resume\n");
2958
2959	tegra_xudc_resume_device_state(xudc);
2960
2961	if (xudc->driver->resume) {
2962	spin_unlock(lock: &xudc->lock);
2963	xudc->driver->resume(&xudc->gadget);
2964	spin_lock(lock: &xudc->lock);
2965	}
2966	}
2967
2968	static inline void clear_port_change(struct tegra_xudc *xudc, u32 flag)
2969	{
2970	u32 val;
2971
2972	val = xudc_readl(xudc, PORTSC);
2973	val &= ~PORTSC_CHANGE_MASK;
2974	val |= flag;
2975	xudc_writel(xudc, val, PORTSC);
2976	}
2977
2978	static void __tegra_xudc_handle_port_status(struct tegra_xudc *xudc)
2979	{
2980	u32 portsc, porthalt;
2981
2982	porthalt = xudc_readl(xudc, PORTHALT);
2983	if ((porthalt & PORTHALT_STCHG_REQ) &&
2984	(porthalt & PORTHALT_HALT_LTSSM)) {
2985	dev_dbg(xudc->dev, "STCHG_REQ, PORTHALT = %#x\n", porthalt);
2986	porthalt &= ~PORTHALT_HALT_LTSSM;
2987	xudc_writel(xudc, val: porthalt, PORTHALT);
2988	}
2989
2990	portsc = xudc_readl(xudc, PORTSC);
2991	if ((portsc & PORTSC_PRC) && (portsc & PORTSC_PR)) {
2992	dev_dbg(xudc->dev, "PRC, PR, PORTSC = %#x\n", portsc);
2993	clear_port_change(xudc, PORTSC_PRC | PORTSC_PED);
2994	#define TOGGLE_VBUS_WAIT_MS 100
2995	if (xudc->soc->port_reset_quirk) {
2996	schedule_delayed_work(dwork: &xudc->port_reset_war_work,
2997	delay: msecs_to_jiffies(TOGGLE_VBUS_WAIT_MS));
2998	xudc->wait_for_sec_prc = 1;
2999	}
3000	}
3001
3002	if ((portsc & PORTSC_PRC) && !(portsc & PORTSC_PR)) {
3003	dev_dbg(xudc->dev, "PRC, Not PR, PORTSC = %#x\n", portsc);
3004	clear_port_change(xudc, PORTSC_PRC | PORTSC_PED);
3005	tegra_xudc_port_reset(xudc);
3006	cancel_delayed_work(dwork: &xudc->port_reset_war_work);
3007	xudc->wait_for_sec_prc = 0;
3008	}
3009
3010	portsc = xudc_readl(xudc, PORTSC);
3011	if (portsc & PORTSC_WRC) {
3012	dev_dbg(xudc->dev, "WRC, PORTSC = %#x\n", portsc);
3013	clear_port_change(xudc, PORTSC_WRC | PORTSC_PED);
3014	if (!(xudc_readl(xudc, PORTSC) & PORTSC_WPR))
3015	tegra_xudc_port_reset(xudc);
3016	}
3017
3018	portsc = xudc_readl(xudc, PORTSC);
3019	if (portsc & PORTSC_CSC) {
3020	dev_dbg(xudc->dev, "CSC, PORTSC = %#x\n", portsc);
3021	clear_port_change(xudc, PORTSC_CSC);
3022
3023	if (portsc & PORTSC_CCS)
3024	tegra_xudc_port_connect(xudc);
3025	else
3026	tegra_xudc_port_disconnect(xudc);
3027
3028	if (xudc->wait_csc) {
3029	cancel_delayed_work(dwork: &xudc->plc_reset_work);
3030	xudc->wait_csc = false;
3031	}
3032	}
3033
3034	portsc = xudc_readl(xudc, PORTSC);
3035	if (portsc & PORTSC_PLC) {
3036	u32 pls = (portsc & PORTSC_PLS_MASK) >> PORTSC_PLS_SHIFT;
3037
3038	dev_dbg(xudc->dev, "PLC, PORTSC = %#x\n", portsc);
3039	clear_port_change(xudc, PORTSC_PLC);
3040	switch (pls) {
3041	case PORTSC_PLS_U3:
3042	tegra_xudc_port_suspend(xudc);
3043	break;
3044	case PORTSC_PLS_U0:
3045	if (xudc->gadget.speed < USB_SPEED_SUPER)
3046	tegra_xudc_port_resume(xudc);
3047	break;
3048	case PORTSC_PLS_RESUME:
3049	if (xudc->gadget.speed == USB_SPEED_SUPER)
3050	tegra_xudc_port_resume(xudc);
3051	break;
3052	case PORTSC_PLS_INACTIVE:
3053	schedule_delayed_work(dwork: &xudc->plc_reset_work,
3054	delay: msecs_to_jiffies(TOGGLE_VBUS_WAIT_MS));
3055	xudc->wait_csc = true;
3056	break;
3057	default:
3058	break;
3059	}
3060	}
3061
3062	if (portsc & PORTSC_CEC) {
3063	dev_warn(xudc->dev, "CEC, PORTSC = %#x\n", portsc);
3064	clear_port_change(xudc, PORTSC_CEC);
3065	}
3066
3067	dev_dbg(xudc->dev, "PORTSC = %#x\n", xudc_readl(xudc, PORTSC));
3068	}
3069
3070	static void tegra_xudc_handle_port_status(struct tegra_xudc *xudc)
3071	{
3072	while ((xudc_readl(xudc, PORTSC) & PORTSC_CHANGE_MASK) ||
3073	(xudc_readl(xudc, PORTHALT) & PORTHALT_STCHG_REQ))
3074	__tegra_xudc_handle_port_status(xudc);
3075	}
3076
3077	static void tegra_xudc_handle_event(struct tegra_xudc *xudc,
3078	struct tegra_xudc_trb *event)
3079	{
3080	u32 type = trb_read_type(trb: event);
3081
3082	dump_trb(xudc, type: "EVENT", trb: event);
3083
3084	switch (type) {
3085	case TRB_TYPE_PORT_STATUS_CHANGE_EVENT:
3086	tegra_xudc_handle_port_status(xudc);
3087	break;
3088	case TRB_TYPE_TRANSFER_EVENT:
3089	tegra_xudc_handle_transfer_event(xudc, event);
3090	break;
3091	case TRB_TYPE_SETUP_PACKET_EVENT:
3092	tegra_xudc_handle_ep0_event(xudc, event);
3093	break;
3094	default:
3095	dev_info(xudc->dev, "Unrecognized TRB type = %#x\n", type);
3096	break;
3097	}
3098	}
3099
3100	static void tegra_xudc_process_event_ring(struct tegra_xudc *xudc)
3101	{
3102	struct tegra_xudc_trb *event;
3103	dma_addr_t erdp;
3104
3105	while (true) {
3106	event = xudc->event_ring[xudc->event_ring_index] +
3107	xudc->event_ring_deq_ptr;
3108
3109	if (trb_read_cycle(trb: event) != xudc->ccs)
3110	break;
3111
3112	tegra_xudc_handle_event(xudc, event);
3113
3114	xudc->event_ring_deq_ptr++;
3115	if (xudc->event_ring_deq_ptr == XUDC_EVENT_RING_SIZE) {
3116	xudc->event_ring_deq_ptr = 0;
3117	xudc->event_ring_index++;
3118	}
3119
3120	if (xudc->event_ring_index == XUDC_NR_EVENT_RINGS) {
3121	xudc->event_ring_index = 0;
3122	xudc->ccs = !xudc->ccs;
3123	}
3124	}
3125
3126	erdp = xudc->event_ring_phys[xudc->event_ring_index] +
3127	xudc->event_ring_deq_ptr * sizeof(*event);
3128
3129	xudc_writel(xudc, upper_32_bits(erdp), ERDPHI);
3130	xudc_writel(xudc, lower_32_bits(erdp) | ERDPLO_EHB, ERDPLO);
3131	}
3132
3133	static irqreturn_t tegra_xudc_irq(int irq, void *data)
3134	{
3135	struct tegra_xudc *xudc = data;
3136	unsigned long flags;
3137	u32 val;
3138
3139	val = xudc_readl(xudc, ST);
3140	if (!(val & ST_IP))
3141	return IRQ_NONE;
3142	xudc_writel(xudc, ST_IP, ST);
3143
3144	spin_lock_irqsave(&xudc->lock, flags);
3145	tegra_xudc_process_event_ring(xudc);
3146	spin_unlock_irqrestore(lock: &xudc->lock, flags);
3147
3148	return IRQ_HANDLED;
3149	}
3150
3151	static int tegra_xudc_alloc_ep(struct tegra_xudc *xudc, unsigned int index)
3152	{
3153	struct tegra_xudc_ep *ep = &xudc->ep[index];
3154
3155	ep->xudc = xudc;
3156	ep->index = index;
3157	ep->context = &xudc->ep_context[index];
3158	INIT_LIST_HEAD(list: &ep->queue);
3159
3160	/*
3161	* EP1 would be the input endpoint corresponding to EP0, but since
3162	* EP0 is bi-directional, EP1 is unused.
3163	*/
3164	if (index == 1)
3165	return 0;
3166
3167	ep->transfer_ring = dma_pool_alloc(pool: xudc->transfer_ring_pool,
3168	GFP_KERNEL,
3169	handle: &ep->transfer_ring_phys);
3170	if (!ep->transfer_ring)
3171	return -ENOMEM;
3172
3173	if (index) {
3174	snprintf(buf: ep->name, size: sizeof(ep->name), fmt: "ep%u%s", index / 2,
3175	(index % 2 == 0) ? "out" : "in");
3176	ep->usb_ep.name = ep->name;
3177	usb_ep_set_maxpacket_limit(ep: &ep->usb_ep, maxpacket_limit: 1024);
3178	ep->usb_ep.max_streams = 16;
3179	ep->usb_ep.ops = &tegra_xudc_ep_ops;
3180	ep->usb_ep.caps.type_bulk = true;
3181	ep->usb_ep.caps.type_int = true;
3182	if (index & 1)
3183	ep->usb_ep.caps.dir_in = true;
3184	else
3185	ep->usb_ep.caps.dir_out = true;
3186	list_add_tail(new: &ep->usb_ep.ep_list, head: &xudc->gadget.ep_list);
3187	} else {
3188	strscpy(ep->name, "ep0", 3);
3189	ep->usb_ep.name = ep->name;
3190	usb_ep_set_maxpacket_limit(ep: &ep->usb_ep, maxpacket_limit: 512);
3191	ep->usb_ep.ops = &tegra_xudc_ep0_ops;
3192	ep->usb_ep.caps.type_control = true;
3193	ep->usb_ep.caps.dir_in = true;
3194	ep->usb_ep.caps.dir_out = true;
3195	}
3196
3197	return 0;
3198	}
3199
3200	static void tegra_xudc_free_ep(struct tegra_xudc *xudc, unsigned int index)
3201	{
3202	struct tegra_xudc_ep *ep = &xudc->ep[index];
3203
3204	/*
3205	* EP1 would be the input endpoint corresponding to EP0, but since
3206	* EP0 is bi-directional, EP1 is unused.
3207	*/
3208	if (index == 1)
3209	return;
3210
3211	dma_pool_free(pool: xudc->transfer_ring_pool, vaddr: ep->transfer_ring,
3212	addr: ep->transfer_ring_phys);
3213	}
3214
3215	static int tegra_xudc_alloc_eps(struct tegra_xudc *xudc)
3216	{
3217	struct usb_request *req;
3218	unsigned int i;
3219	int err;
3220
3221	xudc->ep_context =
3222	dma_alloc_coherent(dev: xudc->dev, XUDC_NR_EPS *
3223	sizeof(*xudc->ep_context),
3224	dma_handle: &xudc->ep_context_phys, GFP_KERNEL);
3225	if (!xudc->ep_context)
3226	return -ENOMEM;
3227
3228	xudc->transfer_ring_pool =
3229	dmam_pool_create(name: dev_name(dev: xudc->dev), dev: xudc->dev,
3230	XUDC_TRANSFER_RING_SIZE *
3231	sizeof(struct tegra_xudc_trb),
3232	align: sizeof(struct tegra_xudc_trb), allocation: 0);
3233	if (!xudc->transfer_ring_pool) {
3234	err = -ENOMEM;
3235	goto free_ep_context;
3236	}
3237
3238	INIT_LIST_HEAD(list: &xudc->gadget.ep_list);
3239	for (i = 0; i < ARRAY_SIZE(xudc->ep); i++) {
3240	err = tegra_xudc_alloc_ep(xudc, index: i);
3241	if (err < 0)
3242	goto free_eps;
3243	}
3244
3245	req = tegra_xudc_ep_alloc_request(usb_ep: &xudc->ep[0].usb_ep, GFP_KERNEL);
3246	if (!req) {
3247	err = -ENOMEM;
3248	goto free_eps;
3249	}
3250	xudc->ep0_req = to_xudc_req(req);
3251
3252	return 0;
3253
3254	free_eps:
3255	for (; i > 0; i--)
3256	tegra_xudc_free_ep(xudc, index: i - 1);
3257	free_ep_context:
3258	dma_free_coherent(dev: xudc->dev, XUDC_NR_EPS * sizeof(*xudc->ep_context),
3259	cpu_addr: xudc->ep_context, dma_handle: xudc->ep_context_phys);
3260	return err;
3261	}
3262
3263	static void tegra_xudc_init_eps(struct tegra_xudc *xudc)
3264	{
3265	xudc_writel(xudc, lower_32_bits(xudc->ep_context_phys), ECPLO);
3266	xudc_writel(xudc, upper_32_bits(xudc->ep_context_phys), ECPHI);
3267	}
3268
3269	static void tegra_xudc_free_eps(struct tegra_xudc *xudc)
3270	{
3271	unsigned int i;
3272
3273	tegra_xudc_ep_free_request(usb_ep: &xudc->ep[0].usb_ep,
3274	usb_req: &xudc->ep0_req->usb_req);
3275
3276	for (i = 0; i < ARRAY_SIZE(xudc->ep); i++)
3277	tegra_xudc_free_ep(xudc, index: i);
3278
3279	dma_free_coherent(dev: xudc->dev, XUDC_NR_EPS * sizeof(*xudc->ep_context),
3280	cpu_addr: xudc->ep_context, dma_handle: xudc->ep_context_phys);
3281	}
3282
3283	static int tegra_xudc_alloc_event_ring(struct tegra_xudc *xudc)
3284	{
3285	unsigned int i;
3286
3287	for (i = 0; i < ARRAY_SIZE(xudc->event_ring); i++) {
3288	xudc->event_ring[i] =
3289	dma_alloc_coherent(dev: xudc->dev, XUDC_EVENT_RING_SIZE *
3290	sizeof(*xudc->event_ring[i]),
3291	dma_handle: &xudc->event_ring_phys[i],
3292	GFP_KERNEL);
3293	if (!xudc->event_ring[i])
3294	goto free_dma;
3295	}
3296
3297	return 0;
3298
3299	free_dma:
3300	for (; i > 0; i--) {
3301	dma_free_coherent(dev: xudc->dev, XUDC_EVENT_RING_SIZE *
3302	sizeof(*xudc->event_ring[i - 1]),
3303	cpu_addr: xudc->event_ring[i - 1],
3304	dma_handle: xudc->event_ring_phys[i - 1]);
3305	}
3306	return -ENOMEM;
3307	}
3308
3309	static void tegra_xudc_init_event_ring(struct tegra_xudc *xudc)
3310	{
3311	unsigned int i;
3312	u32 val;
3313
3314	for (i = 0; i < ARRAY_SIZE(xudc->event_ring); i++) {
3315	memset(xudc->event_ring[i], 0, XUDC_EVENT_RING_SIZE *
3316	sizeof(*xudc->event_ring[i]));
3317
3318	val = xudc_readl(xudc, ERSTSZ);
3319	val &= ~(ERSTSZ_ERSTXSZ_MASK << ERSTSZ_ERSTXSZ_SHIFT(i));
3320	val |= XUDC_EVENT_RING_SIZE << ERSTSZ_ERSTXSZ_SHIFT(i);
3321	xudc_writel(xudc, val, ERSTSZ);
3322
3323	xudc_writel(xudc, lower_32_bits(xudc->event_ring_phys[i]),
3324	ERSTXBALO(i));
3325	xudc_writel(xudc, upper_32_bits(xudc->event_ring_phys[i]),
3326	ERSTXBAHI(i));
3327	}
3328
3329	val = lower_32_bits(xudc->event_ring_phys[0]);
3330	xudc_writel(xudc, val, ERDPLO);
3331	val |= EREPLO_ECS;
3332	xudc_writel(xudc, val, EREPLO);
3333
3334	val = upper_32_bits(xudc->event_ring_phys[0]);
3335	xudc_writel(xudc, val, ERDPHI);
3336	xudc_writel(xudc, val, EREPHI);
3337
3338	xudc->ccs = true;
3339	xudc->event_ring_index = 0;
3340	xudc->event_ring_deq_ptr = 0;
3341	}
3342
3343	static void tegra_xudc_free_event_ring(struct tegra_xudc *xudc)
3344	{
3345	unsigned int i;
3346
3347	for (i = 0; i < ARRAY_SIZE(xudc->event_ring); i++) {
3348	dma_free_coherent(dev: xudc->dev, XUDC_EVENT_RING_SIZE *
3349	sizeof(*xudc->event_ring[i]),
3350	cpu_addr: xudc->event_ring[i],
3351	dma_handle: xudc->event_ring_phys[i]);
3352	}
3353	}
3354
3355	static void tegra_xudc_fpci_ipfs_init(struct tegra_xudc *xudc)
3356	{
3357	u32 val;
3358
3359	if (xudc->soc->has_ipfs) {
3360	val = ipfs_readl(xudc, XUSB_DEV_CONFIGURATION_0);
3361	val |= XUSB_DEV_CONFIGURATION_0_EN_FPCI;
3362	ipfs_writel(xudc, val, XUSB_DEV_CONFIGURATION_0);
3363	usleep_range(min: 10, max: 15);
3364	}
3365
3366	/* Enable bus master */
3367	val = XUSB_DEV_CFG_1_IO_SPACE_EN | XUSB_DEV_CFG_1_MEMORY_SPACE_EN |
3368	XUSB_DEV_CFG_1_BUS_MASTER_EN;
3369	fpci_writel(xudc, val, XUSB_DEV_CFG_1);
3370
3371	/* Program BAR0 space */
3372	val = fpci_readl(xudc, XUSB_DEV_CFG_4);
3373	val &= ~(XUSB_DEV_CFG_4_BASE_ADDR_MASK);
3374	val |= xudc->phys_base & (XUSB_DEV_CFG_4_BASE_ADDR_MASK);
3375
3376	fpci_writel(xudc, val, XUSB_DEV_CFG_4);
3377	fpci_writel(xudc, upper_32_bits(xudc->phys_base), XUSB_DEV_CFG_5);
3378
3379	usleep_range(min: 100, max: 200);
3380
3381	if (xudc->soc->has_ipfs) {
3382	/* Enable interrupt assertion */
3383	val = ipfs_readl(xudc, XUSB_DEV_INTR_MASK_0);
3384	val |= XUSB_DEV_INTR_MASK_0_IP_INT_MASK;
3385	ipfs_writel(xudc, val, XUSB_DEV_INTR_MASK_0);
3386	}
3387	}
3388
3389	static void tegra_xudc_device_params_init(struct tegra_xudc *xudc)
3390	{
3391	u32 val, imod;
3392
3393	if (xudc->soc->has_ipfs) {
3394	val = xudc_readl(xudc, BLCG);
3395	val |= BLCG_ALL;
3396	val &= ~(BLCG_DFPCI | BLCG_UFPCI | BLCG_FE |
3397	BLCG_COREPLL_PWRDN);
3398	val |= BLCG_IOPLL_0_PWRDN;
3399	val |= BLCG_IOPLL_1_PWRDN;
3400	val |= BLCG_IOPLL_2_PWRDN;
3401
3402	xudc_writel(xudc, val, BLCG);
3403	}
3404
3405	if (xudc->soc->port_speed_quirk)
3406	tegra_xudc_limit_port_speed(xudc);
3407
3408	/* Set a reasonable U3 exit timer value. */
3409	val = xudc_readl(xudc, SSPX_CORE_PADCTL4);
3410	val &= ~(SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3_MASK);
3411	val |= SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3(0x5dc0);
3412	xudc_writel(xudc, val, SSPX_CORE_PADCTL4);
3413
3414	/* Default ping LFPS tBurst is too large. */
3415	val = xudc_readl(xudc, SSPX_CORE_CNT0);
3416	val &= ~(SSPX_CORE_CNT0_PING_TBURST_MASK);
3417	val |= SSPX_CORE_CNT0_PING_TBURST(0xa);
3418	xudc_writel(xudc, val, SSPX_CORE_CNT0);
3419
3420	/* Default tPortConfiguration timeout is too small. */
3421	val = xudc_readl(xudc, SSPX_CORE_CNT30);
3422	val &= ~(SSPX_CORE_CNT30_LMPITP_TIMER_MASK);
3423	val |= SSPX_CORE_CNT30_LMPITP_TIMER(0x978);
3424	xudc_writel(xudc, val, SSPX_CORE_CNT30);
3425
3426	if (xudc->soc->lpm_enable) {
3427	/* Set L1 resume duration to 95 us. */
3428	val = xudc_readl(xudc, HSFSPI_COUNT13);
3429	val &= ~(HSFSPI_COUNT13_U2_RESUME_K_DURATION_MASK);
3430	val |= HSFSPI_COUNT13_U2_RESUME_K_DURATION(0x2c88);
3431	xudc_writel(xudc, val, HSFSPI_COUNT13);
3432	}
3433
3434	/*
3435	* Compliance suite appears to be violating polling LFPS tBurst max
3436	* of 1.4us. Send 1.45us instead.
3437	*/
3438	val = xudc_readl(xudc, SSPX_CORE_CNT32);
3439	val &= ~(SSPX_CORE_CNT32_POLL_TBURST_MAX_MASK);
3440	val |= SSPX_CORE_CNT32_POLL_TBURST_MAX(0xb0);
3441	xudc_writel(xudc, val, SSPX_CORE_CNT32);
3442
3443	/* Direct HS/FS port instance to RxDetect. */
3444	val = xudc_readl(xudc, CFG_DEV_FE);
3445	val &= ~(CFG_DEV_FE_PORTREGSEL_MASK);
3446	val |= CFG_DEV_FE_PORTREGSEL(CFG_DEV_FE_PORTREGSEL_HSFS_PI);
3447	xudc_writel(xudc, val, CFG_DEV_FE);
3448
3449	val = xudc_readl(xudc, PORTSC);
3450	val &= ~(PORTSC_CHANGE_MASK | PORTSC_PLS_MASK);
3451	val |= PORTSC_LWS | PORTSC_PLS(PORTSC_PLS_RXDETECT);
3452	xudc_writel(xudc, val, PORTSC);
3453
3454	/* Direct SS port instance to RxDetect. */
3455	val = xudc_readl(xudc, CFG_DEV_FE);
3456	val &= ~(CFG_DEV_FE_PORTREGSEL_MASK);
3457	val |= CFG_DEV_FE_PORTREGSEL_SS_PI & CFG_DEV_FE_PORTREGSEL_MASK;
3458	xudc_writel(xudc, val, CFG_DEV_FE);
3459
3460	val = xudc_readl(xudc, PORTSC);
3461	val &= ~(PORTSC_CHANGE_MASK | PORTSC_PLS_MASK);
3462	val |= PORTSC_LWS | PORTSC_PLS(PORTSC_PLS_RXDETECT);
3463	xudc_writel(xudc, val, PORTSC);
3464
3465	/* Restore port instance. */
3466	val = xudc_readl(xudc, CFG_DEV_FE);
3467	val &= ~(CFG_DEV_FE_PORTREGSEL_MASK);
3468	xudc_writel(xudc, val, CFG_DEV_FE);
3469
3470	/*
3471	* Enable INFINITE_SS_RETRY to prevent device from entering
3472	* Disabled.Error when attached to buggy SuperSpeed hubs.
3473	*/
3474	val = xudc_readl(xudc, CFG_DEV_FE);
3475	val |= CFG_DEV_FE_INFINITE_SS_RETRY;
3476	xudc_writel(xudc, val, CFG_DEV_FE);
3477
3478	/* Set interrupt moderation. */
3479	imod = XUDC_INTERRUPT_MODERATION_US * 4;
3480	val = xudc_readl(xudc, RT_IMOD);
3481	val &= ~((RT_IMOD_IMODI_MASK) | (RT_IMOD_IMODC_MASK));
3482	val |= (RT_IMOD_IMODI(imod) | RT_IMOD_IMODC(imod));
3483	xudc_writel(xudc, val, RT_IMOD);
3484
3485	/* increase SSPI transaction timeout from 32us to 512us */
3486	val = xudc_readl(xudc, CFG_DEV_SSPI_XFER);
3487	val &= ~(CFG_DEV_SSPI_XFER_ACKTIMEOUT_MASK);
3488	val |= CFG_DEV_SSPI_XFER_ACKTIMEOUT(0xf000);
3489	xudc_writel(xudc, val, CFG_DEV_SSPI_XFER);
3490	}
3491
3492	static int tegra_xudc_phy_get(struct tegra_xudc *xudc)
3493	{
3494	int err = 0, usb3_companion_port;
3495	unsigned int i, j;
3496
3497	xudc->utmi_phy = devm_kcalloc(dev: xudc->dev, n: xudc->soc->num_phys,
3498	size: sizeof(*xudc->utmi_phy), GFP_KERNEL);
3499	if (!xudc->utmi_phy)
3500	return -ENOMEM;
3501
3502	xudc->usb3_phy = devm_kcalloc(dev: xudc->dev, n: xudc->soc->num_phys,
3503	size: sizeof(*xudc->usb3_phy), GFP_KERNEL);
3504	if (!xudc->usb3_phy)
3505	return -ENOMEM;
3506
3507	xudc->usbphy = devm_kcalloc(dev: xudc->dev, n: xudc->soc->num_phys,
3508	size: sizeof(*xudc->usbphy), GFP_KERNEL);
3509	if (!xudc->usbphy)
3510	return -ENOMEM;
3511
3512	xudc->vbus_nb.notifier_call = tegra_xudc_vbus_notify;
3513
3514	for (i = 0; i < xudc->soc->num_phys; i++) {
3515	char phy_name[] = "usb.-.";
3516
3517	/* Get USB2 phy */
3518	snprintf(buf: phy_name, size: sizeof(phy_name), fmt: "usb2-%d", i);
3519	xudc->utmi_phy[i] = devm_phy_optional_get(dev: xudc->dev, string: phy_name);
3520	if (IS_ERR(ptr: xudc->utmi_phy[i])) {
3521	err = PTR_ERR(ptr: xudc->utmi_phy[i]);
3522	dev_err_probe(dev: xudc->dev, err,
3523	fmt: "failed to get PHY for phy-name usb2-%d\n", i);
3524	goto clean_up;
3525	} else if (xudc->utmi_phy[i]) {
3526	/* Get usb-phy, if utmi phy is available */
3527	xudc->usbphy[i] = devm_usb_get_phy_by_node(dev: xudc->dev,
3528	node: xudc->utmi_phy[i]->dev.of_node,
3529	NULL);
3530	if (IS_ERR(ptr: xudc->usbphy[i])) {
3531	err = PTR_ERR(ptr: xudc->usbphy[i]);
3532	dev_err_probe(dev: xudc->dev, err,
3533	fmt: "failed to get usbphy-%d\n", i);
3534	goto clean_up;
3535	}
3536	} else if (!xudc->utmi_phy[i]) {
3537	/* if utmi phy is not available, ignore USB3 phy get */
3538	continue;
3539	}
3540
3541	/* Get USB3 phy */
3542	usb3_companion_port = tegra_xusb_padctl_get_usb3_companion(padctl: xudc->padctl, port: i);
3543	if (usb3_companion_port < 0)
3544	continue;
3545
3546	for (j = 0; j < xudc->soc->num_phys; j++) {
3547	snprintf(buf: phy_name, size: sizeof(phy_name), fmt: "usb3-%d", j);
3548	xudc->usb3_phy[i] = devm_phy_optional_get(dev: xudc->dev, string: phy_name);
3549	if (IS_ERR(ptr: xudc->usb3_phy[i])) {
3550	err = PTR_ERR(ptr: xudc->usb3_phy[i]);
3551	dev_err_probe(dev: xudc->dev, err,
3552	fmt: "failed to get PHY for phy-name usb3-%d\n", j);
3553	goto clean_up;
3554	} else if (xudc->usb3_phy[i]) {
3555	int usb2_port =
3556	tegra_xusb_padctl_get_port_number(phy: xudc->utmi_phy[i]);
3557	int usb3_port =
3558	tegra_xusb_padctl_get_port_number(phy: xudc->usb3_phy[i]);
3559	if (usb3_port == usb3_companion_port) {
3560	dev_dbg(xudc->dev, "USB2 port %d is paired with USB3 port %d for device mode port %d\n",
3561	usb2_port, usb3_port, i);
3562	break;
3563	}
3564	}
3565	}
3566	}
3567
3568	return err;
3569
3570	clean_up:
3571	for (i = 0; i < xudc->soc->num_phys; i++) {
3572	xudc->usb3_phy[i] = NULL;
3573	xudc->utmi_phy[i] = NULL;
3574	xudc->usbphy[i] = NULL;
3575	}
3576
3577	return err;
3578	}
3579
3580	static void tegra_xudc_phy_exit(struct tegra_xudc *xudc)
3581	{
3582	unsigned int i;
3583
3584	for (i = 0; i < xudc->soc->num_phys; i++) {
3585	phy_exit(phy: xudc->usb3_phy[i]);
3586	phy_exit(phy: xudc->utmi_phy[i]);
3587	}
3588	}
3589
3590	static int tegra_xudc_phy_init(struct tegra_xudc *xudc)
3591	{
3592	int err;
3593	unsigned int i;
3594
3595	for (i = 0; i < xudc->soc->num_phys; i++) {
3596	err = phy_init(phy: xudc->utmi_phy[i]);
3597	if (err < 0) {
3598	dev_err(xudc->dev, "UTMI PHY #%u initialization failed: %d\n", i, err);
3599	goto exit_phy;
3600	}
3601
3602	err = phy_init(phy: xudc->usb3_phy[i]);
3603	if (err < 0) {
3604	dev_err(xudc->dev, "USB3 PHY #%u initialization failed: %d\n", i, err);
3605	goto exit_phy;
3606	}
3607	}
3608	return 0;
3609
3610	exit_phy:
3611	tegra_xudc_phy_exit(xudc);
3612	return err;
3613	}
3614
3615	static const char * const tegra210_xudc_supply_names[] = {
3616	"hvdd-usb",
3617	"avddio-usb",
3618	};
3619
3620	static const char * const tegra210_xudc_clock_names[] = {
3621	"dev",
3622	"ss",
3623	"ss_src",
3624	"hs_src",
3625	"fs_src",
3626	};
3627
3628	static const char * const tegra186_xudc_clock_names[] = {
3629	"dev",
3630	"ss",
3631	"ss_src",
3632	"fs_src",
3633	};
3634
3635	static struct tegra_xudc_soc tegra210_xudc_soc_data = {
3636	.supply_names = tegra210_xudc_supply_names,
3637	.num_supplies = ARRAY_SIZE(tegra210_xudc_supply_names),
3638	.clock_names = tegra210_xudc_clock_names,
3639	.num_clks = ARRAY_SIZE(tegra210_xudc_clock_names),
3640	.num_phys = 4,
3641	.u1_enable = false,
3642	.u2_enable = true,
3643	.lpm_enable = false,
3644	.invalid_seq_num = true,
3645	.pls_quirk = true,
3646	.port_reset_quirk = true,
3647	.port_speed_quirk = false,
3648	.has_ipfs = true,
3649	};
3650
3651	static struct tegra_xudc_soc tegra186_xudc_soc_data = {
3652	.clock_names = tegra186_xudc_clock_names,
3653	.num_clks = ARRAY_SIZE(tegra186_xudc_clock_names),
3654	.num_phys = 4,
3655	.u1_enable = true,
3656	.u2_enable = true,
3657	.lpm_enable = false,
3658	.invalid_seq_num = false,
3659	.pls_quirk = false,
3660	.port_reset_quirk = false,
3661	.port_speed_quirk = false,
3662	.has_ipfs = false,
3663	};
3664
3665	static struct tegra_xudc_soc tegra194_xudc_soc_data = {
3666	.clock_names = tegra186_xudc_clock_names,
3667	.num_clks = ARRAY_SIZE(tegra186_xudc_clock_names),
3668	.num_phys = 4,
3669	.u1_enable = true,
3670	.u2_enable = true,
3671	.lpm_enable = true,
3672	.invalid_seq_num = false,
3673	.pls_quirk = false,
3674	.port_reset_quirk = false,
3675	.port_speed_quirk = true,
3676	.has_ipfs = false,
3677	};
3678
3679	static struct tegra_xudc_soc tegra234_xudc_soc_data = {
3680	.clock_names = tegra186_xudc_clock_names,
3681	.num_clks = ARRAY_SIZE(tegra186_xudc_clock_names),
3682	.num_phys = 4,
3683	.u1_enable = true,
3684	.u2_enable = true,
3685	.lpm_enable = true,
3686	.invalid_seq_num = false,
3687	.pls_quirk = false,
3688	.port_reset_quirk = false,
3689	.has_ipfs = false,
3690	};
3691
3692	static const struct of_device_id tegra_xudc_of_match[] = {
3693	{
3694	.compatible = "nvidia,tegra210-xudc",
3695	.data = &tegra210_xudc_soc_data
3696	},
3697	{
3698	.compatible = "nvidia,tegra186-xudc",
3699	.data = &tegra186_xudc_soc_data
3700	},
3701	{
3702	.compatible = "nvidia,tegra194-xudc",
3703	.data = &tegra194_xudc_soc_data
3704	},
3705	{
3706	.compatible = "nvidia,tegra234-xudc",
3707	.data = &tegra234_xudc_soc_data
3708	},
3709	{ }
3710	};
3711	MODULE_DEVICE_TABLE(of, tegra_xudc_of_match);
3712
3713	static void tegra_xudc_powerdomain_remove(struct tegra_xudc *xudc)
3714	{
3715	if (xudc->genpd_dl_ss)
3716	device_link_del(link: xudc->genpd_dl_ss);
3717	if (xudc->genpd_dl_device)
3718	device_link_del(link: xudc->genpd_dl_device);
3719	if (xudc->genpd_dev_ss)
3720	dev_pm_domain_detach(dev: xudc->genpd_dev_ss, power_off: true);
3721	if (xudc->genpd_dev_device)
3722	dev_pm_domain_detach(dev: xudc->genpd_dev_device, power_off: true);
3723	}
3724
3725	static int tegra_xudc_powerdomain_init(struct tegra_xudc *xudc)
3726	{
3727	struct device *dev = xudc->dev;
3728	int err;
3729
3730	xudc->genpd_dev_device = dev_pm_domain_attach_by_name(dev, name: "dev");
3731	if (IS_ERR(ptr: xudc->genpd_dev_device)) {
3732	err = PTR_ERR(ptr: xudc->genpd_dev_device);
3733	dev_err(dev, "failed to get device power domain: %d\n", err);
3734	return err;
3735	}
3736
3737	xudc->genpd_dev_ss = dev_pm_domain_attach_by_name(dev, name: "ss");
3738	if (IS_ERR(ptr: xudc->genpd_dev_ss)) {
3739	err = PTR_ERR(ptr: xudc->genpd_dev_ss);
3740	dev_err(dev, "failed to get SuperSpeed power domain: %d\n", err);
3741	return err;
3742	}
3743
3744	xudc->genpd_dl_device = device_link_add(consumer: dev, supplier: xudc->genpd_dev_device,
3745	DL_FLAG_PM_RUNTIME |
3746	DL_FLAG_STATELESS);
3747	if (!xudc->genpd_dl_device) {
3748	dev_err(dev, "failed to add USB device link\n");
3749	return -ENODEV;
3750	}
3751
3752	xudc->genpd_dl_ss = device_link_add(consumer: dev, supplier: xudc->genpd_dev_ss,
3753	DL_FLAG_PM_RUNTIME |
3754	DL_FLAG_STATELESS);
3755	if (!xudc->genpd_dl_ss) {
3756	dev_err(dev, "failed to add SuperSpeed device link\n");
3757	return -ENODEV;
3758	}
3759
3760	return 0;
3761	}
3762
3763	static int tegra_xudc_probe(struct platform_device *pdev)
3764	{
3765	struct tegra_xudc *xudc;
3766	struct resource *res;
3767	unsigned int i;
3768	int err;
3769
3770	xudc = devm_kzalloc(dev: &pdev->dev, size: sizeof(*xudc), GFP_KERNEL);
3771	if (!xudc)
3772	return -ENOMEM;
3773
3774	xudc->dev = &pdev->dev;
3775	platform_set_drvdata(pdev, data: xudc);
3776
3777	xudc->soc = of_device_get_match_data(dev: &pdev->dev);
3778	if (!xudc->soc)
3779	return -ENODEV;
3780
3781	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "base");
3782	xudc->base = devm_ioremap_resource(dev: &pdev->dev, res);
3783	if (IS_ERR(ptr: xudc->base))
3784	return PTR_ERR(ptr: xudc->base);
3785	xudc->phys_base = res->start;
3786
3787	xudc->fpci = devm_platform_ioremap_resource_byname(pdev, name: "fpci");
3788	if (IS_ERR(ptr: xudc->fpci))
3789	return PTR_ERR(ptr: xudc->fpci);
3790
3791	if (xudc->soc->has_ipfs) {
3792	xudc->ipfs = devm_platform_ioremap_resource_byname(pdev, name: "ipfs");
3793	if (IS_ERR(ptr: xudc->ipfs))
3794	return PTR_ERR(ptr: xudc->ipfs);
3795	}
3796
3797	xudc->irq = platform_get_irq(pdev, 0);
3798	if (xudc->irq < 0)
3799	return xudc->irq;
3800
3801	err = devm_request_irq(dev: &pdev->dev, irq: xudc->irq, handler: tegra_xudc_irq, irqflags: 0,
3802	devname: dev_name(dev: &pdev->dev), dev_id: xudc);
3803	if (err < 0) {
3804	dev_err(xudc->dev, "failed to claim IRQ#%u: %d\n", xudc->irq,
3805	err);
3806	return err;
3807	}
3808
3809	xudc->clks = devm_kcalloc(dev: &pdev->dev, n: xudc->soc->num_clks, size: sizeof(*xudc->clks),
3810	GFP_KERNEL);
3811	if (!xudc->clks)
3812	return -ENOMEM;
3813
3814	for (i = 0; i < xudc->soc->num_clks; i++)
3815	xudc->clks[i].id = xudc->soc->clock_names[i];
3816
3817	err = devm_clk_bulk_get(dev: &pdev->dev, num_clks: xudc->soc->num_clks, clks: xudc->clks);
3818	if (err) {
3819	dev_err_probe(dev: xudc->dev, err, fmt: "failed to request clocks\n");
3820	return err;
3821	}
3822
3823	xudc->supplies = devm_kcalloc(dev: &pdev->dev, n: xudc->soc->num_supplies,
3824	size: sizeof(*xudc->supplies), GFP_KERNEL);
3825	if (!xudc->supplies)
3826	return -ENOMEM;
3827
3828	for (i = 0; i < xudc->soc->num_supplies; i++)
3829	xudc->supplies[i].supply = xudc->soc->supply_names[i];
3830
3831	err = devm_regulator_bulk_get(dev: &pdev->dev, num_consumers: xudc->soc->num_supplies,
3832	consumers: xudc->supplies);
3833	if (err) {
3834	dev_err_probe(dev: xudc->dev, err, fmt: "failed to request regulators\n");
3835	return err;
3836	}
3837
3838	xudc->padctl = tegra_xusb_padctl_get(dev: &pdev->dev);
3839	if (IS_ERR(ptr: xudc->padctl))
3840	return PTR_ERR(ptr: xudc->padctl);
3841
3842	err = regulator_bulk_enable(num_consumers: xudc->soc->num_supplies, consumers: xudc->supplies);
3843	if (err) {
3844	dev_err(xudc->dev, "failed to enable regulators: %d\n", err);
3845	goto put_padctl;
3846	}
3847
3848	err = tegra_xudc_phy_get(xudc);
3849	if (err)
3850	goto disable_regulator;
3851
3852	err = tegra_xudc_powerdomain_init(xudc);
3853	if (err)
3854	goto put_powerdomains;
3855
3856	err = tegra_xudc_phy_init(xudc);
3857	if (err)
3858	goto put_powerdomains;
3859
3860	err = tegra_xudc_alloc_event_ring(xudc);
3861	if (err)
3862	goto disable_phy;
3863
3864	err = tegra_xudc_alloc_eps(xudc);
3865	if (err)
3866	goto free_event_ring;
3867
3868	spin_lock_init(&xudc->lock);
3869
3870	init_completion(x: &xudc->disconnect_complete);
3871
3872	INIT_WORK(&xudc->usb_role_sw_work, tegra_xudc_usb_role_sw_work);
3873
3874	INIT_DELAYED_WORK(&xudc->plc_reset_work, tegra_xudc_plc_reset_work);
3875
3876	INIT_DELAYED_WORK(&xudc->port_reset_war_work,
3877	tegra_xudc_port_reset_war_work);
3878
3879	pm_runtime_enable(dev: &pdev->dev);
3880
3881	xudc->gadget.ops = &tegra_xudc_gadget_ops;
3882	xudc->gadget.ep0 = &xudc->ep[0].usb_ep;
3883	xudc->gadget.name = "tegra-xudc";
3884	xudc->gadget.max_speed = USB_SPEED_SUPER;
3885
3886	err = usb_add_gadget_udc(parent: &pdev->dev, gadget: &xudc->gadget);
3887	if (err) {
3888	dev_err(&pdev->dev, "failed to add USB gadget: %d\n", err);
3889	goto free_eps;
3890	}
3891
3892	for (i = 0; i < xudc->soc->num_phys; i++) {
3893	if (!xudc->usbphy[i])
3894	continue;
3895
3896	usb_register_notifier(x: xudc->usbphy[i], nb: &xudc->vbus_nb);
3897	tegra_xudc_update_data_role(xudc, usbphy: xudc->usbphy[i]);
3898	}
3899
3900	return 0;
3901
3902	free_eps:
3903	pm_runtime_disable(dev: &pdev->dev);
3904	tegra_xudc_free_eps(xudc);
3905	free_event_ring:
3906	tegra_xudc_free_event_ring(xudc);
3907	disable_phy:
3908	tegra_xudc_phy_exit(xudc);
3909	put_powerdomains:
3910	tegra_xudc_powerdomain_remove(xudc);
3911	disable_regulator:
3912	regulator_bulk_disable(num_consumers: xudc->soc->num_supplies, consumers: xudc->supplies);
3913	put_padctl:
3914	tegra_xusb_padctl_put(padctl: xudc->padctl);
3915
3916	return err;
3917	}
3918
3919	static void tegra_xudc_remove(struct platform_device *pdev)
3920	{
3921	struct tegra_xudc *xudc = platform_get_drvdata(pdev);
3922	unsigned int i;
3923
3924	pm_runtime_get_sync(dev: xudc->dev);
3925
3926	cancel_delayed_work_sync(dwork: &xudc->plc_reset_work);
3927	cancel_work_sync(work: &xudc->usb_role_sw_work);
3928
3929	usb_del_gadget_udc(gadget: &xudc->gadget);
3930
3931	tegra_xudc_free_eps(xudc);
3932	tegra_xudc_free_event_ring(xudc);
3933
3934	tegra_xudc_powerdomain_remove(xudc);
3935
3936	regulator_bulk_disable(num_consumers: xudc->soc->num_supplies, consumers: xudc->supplies);
3937
3938	for (i = 0; i < xudc->soc->num_phys; i++) {
3939	phy_power_off(phy: xudc->utmi_phy[i]);
3940	phy_power_off(phy: xudc->usb3_phy[i]);
3941	}
3942
3943	tegra_xudc_phy_exit(xudc);
3944
3945	pm_runtime_disable(dev: xudc->dev);
3946	pm_runtime_put(dev: xudc->dev);
3947
3948	tegra_xusb_padctl_put(padctl: xudc->padctl);
3949	}
3950
3951	static int __maybe_unused tegra_xudc_powergate(struct tegra_xudc *xudc)
3952	{
3953	unsigned long flags;
3954
3955	dev_dbg(xudc->dev, "entering ELPG\n");
3956
3957	spin_lock_irqsave(&xudc->lock, flags);
3958
3959	xudc->powergated = true;
3960	xudc->saved_regs.ctrl = xudc_readl(xudc, CTRL);
3961	xudc->saved_regs.portpm = xudc_readl(xudc, PORTPM);
3962	xudc_writel(xudc, val: 0, CTRL);
3963
3964	spin_unlock_irqrestore(lock: &xudc->lock, flags);
3965
3966	clk_bulk_disable_unprepare(num_clks: xudc->soc->num_clks, clks: xudc->clks);
3967
3968	regulator_bulk_disable(num_consumers: xudc->soc->num_supplies, consumers: xudc->supplies);
3969
3970	dev_dbg(xudc->dev, "entering ELPG done\n");
3971	return 0;
3972	}
3973
3974	static int __maybe_unused tegra_xudc_unpowergate(struct tegra_xudc *xudc)
3975	{
3976	unsigned long flags;
3977	int err;
3978
3979	dev_dbg(xudc->dev, "exiting ELPG\n");
3980
3981	err = regulator_bulk_enable(num_consumers: xudc->soc->num_supplies,
3982	consumers: xudc->supplies);
3983	if (err < 0)
3984	return err;
3985
3986	err = clk_bulk_prepare_enable(num_clks: xudc->soc->num_clks, clks: xudc->clks);
3987	if (err < 0)
3988	return err;
3989
3990	tegra_xudc_fpci_ipfs_init(xudc);
3991
3992	tegra_xudc_device_params_init(xudc);
3993
3994	tegra_xudc_init_event_ring(xudc);
3995
3996	tegra_xudc_init_eps(xudc);
3997
3998	xudc_writel(xudc, val: xudc->saved_regs.portpm, PORTPM);
3999	xudc_writel(xudc, val: xudc->saved_regs.ctrl, CTRL);
4000
4001	spin_lock_irqsave(&xudc->lock, flags);
4002	xudc->powergated = false;
4003	spin_unlock_irqrestore(lock: &xudc->lock, flags);
4004
4005	dev_dbg(xudc->dev, "exiting ELPG done\n");
4006	return 0;
4007	}
4008
4009	static int __maybe_unused tegra_xudc_suspend(struct device *dev)
4010	{
4011	struct tegra_xudc *xudc = dev_get_drvdata(dev);
4012	unsigned long flags;
4013
4014	spin_lock_irqsave(&xudc->lock, flags);
4015	xudc->suspended = true;
4016	spin_unlock_irqrestore(lock: &xudc->lock, flags);
4017
4018	flush_work(work: &xudc->usb_role_sw_work);
4019
4020	if (!pm_runtime_status_suspended(dev)) {
4021	/* Forcibly disconnect before powergating. */
4022	tegra_xudc_device_mode_off(xudc);
4023	tegra_xudc_powergate(xudc);
4024	}
4025
4026	pm_runtime_disable(dev);
4027
4028	return 0;
4029	}
4030
4031	static int __maybe_unused tegra_xudc_resume(struct device *dev)
4032	{
4033	struct tegra_xudc *xudc = dev_get_drvdata(dev);
4034	unsigned long flags;
4035	int err;
4036
4037	err = tegra_xudc_unpowergate(xudc);
4038	if (err < 0)
4039	return err;
4040
4041	spin_lock_irqsave(&xudc->lock, flags);
4042	xudc->suspended = false;
4043	spin_unlock_irqrestore(lock: &xudc->lock, flags);
4044
4045	schedule_work(work: &xudc->usb_role_sw_work);
4046
4047	pm_runtime_enable(dev);
4048
4049	return 0;
4050	}
4051
4052	static int __maybe_unused tegra_xudc_runtime_suspend(struct device *dev)
4053	{
4054	struct tegra_xudc *xudc = dev_get_drvdata(dev);
4055
4056	return tegra_xudc_powergate(xudc);
4057	}
4058
4059	static int __maybe_unused tegra_xudc_runtime_resume(struct device *dev)
4060	{
4061	struct tegra_xudc *xudc = dev_get_drvdata(dev);
4062
4063	return tegra_xudc_unpowergate(xudc);
4064	}
4065
4066	static const struct dev_pm_ops tegra_xudc_pm_ops = {
4067	SET_SYSTEM_SLEEP_PM_OPS(tegra_xudc_suspend, tegra_xudc_resume)
4068	SET_RUNTIME_PM_OPS(tegra_xudc_runtime_suspend,
4069	tegra_xudc_runtime_resume, NULL)
4070	};
4071
4072	static struct platform_driver tegra_xudc_driver = {
4073	.probe = tegra_xudc_probe,
4074	.remove_new = tegra_xudc_remove,
4075	.driver = {
4076	.name = "tegra-xudc",
4077	.pm = &tegra_xudc_pm_ops,
4078	.of_match_table = tegra_xudc_of_match,
4079	},
4080	};
4081	module_platform_driver(tegra_xudc_driver);
4082
4083	MODULE_DESCRIPTION("NVIDIA Tegra XUSB Device Controller");
4084	MODULE_AUTHOR("Andrew Bresticker <abrestic@chromium.org>");
4085	MODULE_AUTHOR("Hui Fu <hfu@nvidia.com>");
4086	MODULE_AUTHOR("Nagarjuna Kristam <nkristam@nvidia.com>");
4087	MODULE_LICENSE("GPL v2");
4088