1	// SPDX-License-Identifier: GPL-2.0+
2	/*
3	* PCIe host controller driver for the following SoCs
4	* Tegra194
5	* Tegra234
6	*
7	* Copyright (C) 2019-2022 NVIDIA Corporation.
8	*
9	* Author: Vidya Sagar <vidyas@nvidia.com>
10	*/
11
12	#include <linux/bitfield.h>
13	#include <linux/clk.h>
14	#include <linux/debugfs.h>
15	#include <linux/delay.h>
16	#include <linux/gpio/consumer.h>
17	#include <linux/interconnect.h>
18	#include <linux/interrupt.h>
19	#include <linux/iopoll.h>
20	#include <linux/kernel.h>
21	#include <linux/module.h>
22	#include <linux/of.h>
23	#include <linux/of_pci.h>
24	#include <linux/pci.h>
25	#include <linux/phy/phy.h>
26	#include <linux/pinctrl/consumer.h>
27	#include <linux/platform_device.h>
28	#include <linux/pm_runtime.h>
29	#include <linux/random.h>
30	#include <linux/reset.h>
31	#include <linux/resource.h>
32	#include <linux/types.h>
33	#include "pcie-designware.h"
34	#include <soc/tegra/bpmp.h>
35	#include <soc/tegra/bpmp-abi.h>
36	#include "../../pci.h"
37
38	#define TEGRA194_DWC_IP_VER 0x490A
39	#define TEGRA234_DWC_IP_VER 0x562A
40
41	#define APPL_PINMUX 0x0
42	#define APPL_PINMUX_PEX_RST BIT(0)
43	#define APPL_PINMUX_CLKREQ_OVERRIDE_EN BIT(2)
44	#define APPL_PINMUX_CLKREQ_OVERRIDE BIT(3)
45	#define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN BIT(4)
46	#define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE BIT(5)
47
48	#define APPL_CTRL 0x4
49	#define APPL_CTRL_SYS_PRE_DET_STATE BIT(6)
50	#define APPL_CTRL_LTSSM_EN BIT(7)
51	#define APPL_CTRL_HW_HOT_RST_EN BIT(20)
52	#define APPL_CTRL_HW_HOT_RST_MODE_MASK GENMASK(1, 0)
53	#define APPL_CTRL_HW_HOT_RST_MODE_SHIFT 22
54	#define APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST 0x1
55	#define APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST_LTSSM_EN 0x2
56
57	#define APPL_INTR_EN_L0_0 0x8
58	#define APPL_INTR_EN_L0_0_LINK_STATE_INT_EN BIT(0)
59	#define APPL_INTR_EN_L0_0_MSI_RCV_INT_EN BIT(4)
60	#define APPL_INTR_EN_L0_0_INT_INT_EN BIT(8)
61	#define APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN BIT(15)
62	#define APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN BIT(19)
63	#define APPL_INTR_EN_L0_0_SYS_INTR_EN BIT(30)
64	#define APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN BIT(31)
65
66	#define APPL_INTR_STATUS_L0 0xC
67	#define APPL_INTR_STATUS_L0_LINK_STATE_INT BIT(0)
68	#define APPL_INTR_STATUS_L0_INT_INT BIT(8)
69	#define APPL_INTR_STATUS_L0_PCI_CMD_EN_INT BIT(15)
70	#define APPL_INTR_STATUS_L0_PEX_RST_INT BIT(16)
71	#define APPL_INTR_STATUS_L0_CDM_REG_CHK_INT BIT(18)
72
73	#define APPL_INTR_EN_L1_0_0 0x1C
74	#define APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN BIT(1)
75	#define APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN BIT(3)
76	#define APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN BIT(30)
77
78	#define APPL_INTR_STATUS_L1_0_0 0x20
79	#define APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED BIT(1)
80	#define APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED BIT(3)
81	#define APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE BIT(30)
82
83	#define APPL_INTR_STATUS_L1_1 0x2C
84	#define APPL_INTR_STATUS_L1_2 0x30
85	#define APPL_INTR_STATUS_L1_3 0x34
86	#define APPL_INTR_STATUS_L1_6 0x3C
87	#define APPL_INTR_STATUS_L1_7 0x40
88	#define APPL_INTR_STATUS_L1_15_CFG_BME_CHGED BIT(1)
89
90	#define APPL_INTR_EN_L1_8_0 0x44
91	#define APPL_INTR_EN_L1_8_BW_MGT_INT_EN BIT(2)
92	#define APPL_INTR_EN_L1_8_AUTO_BW_INT_EN BIT(3)
93	#define APPL_INTR_EN_L1_8_INTX_EN BIT(11)
94	#define APPL_INTR_EN_L1_8_AER_INT_EN BIT(15)
95
96	#define APPL_INTR_STATUS_L1_8_0 0x4C
97	#define APPL_INTR_STATUS_L1_8_0_EDMA_INT_MASK GENMASK(11, 6)
98	#define APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS BIT(2)
99	#define APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS BIT(3)
100
101	#define APPL_INTR_STATUS_L1_9 0x54
102	#define APPL_INTR_STATUS_L1_10 0x58
103	#define APPL_INTR_STATUS_L1_11 0x64
104	#define APPL_INTR_STATUS_L1_13 0x74
105	#define APPL_INTR_STATUS_L1_14 0x78
106	#define APPL_INTR_STATUS_L1_15 0x7C
107	#define APPL_INTR_STATUS_L1_17 0x88
108
109	#define APPL_INTR_EN_L1_18 0x90
110	#define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMPLT BIT(2)
111	#define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR BIT(1)
112	#define APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR BIT(0)
113
114	#define APPL_INTR_STATUS_L1_18 0x94
115	#define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT BIT(2)
116	#define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR BIT(1)
117	#define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR BIT(0)
118
119	#define APPL_MSI_CTRL_1 0xAC
120
121	#define APPL_MSI_CTRL_2 0xB0
122
123	#define APPL_LEGACY_INTX 0xB8
124
125	#define APPL_LTR_MSG_1 0xC4
126	#define LTR_MSG_REQ BIT(15)
127	#define LTR_NOSNOOP_MSG_REQ BIT(31)
128
129	#define APPL_LTR_MSG_2 0xC8
130	#define APPL_LTR_MSG_2_LTR_MSG_REQ_STATE BIT(3)
131
132	#define APPL_LINK_STATUS 0xCC
133	#define APPL_LINK_STATUS_RDLH_LINK_UP BIT(0)
134
135	#define APPL_DEBUG 0xD0
136	#define APPL_DEBUG_PM_LINKST_IN_L2_LAT BIT(21)
137	#define APPL_DEBUG_PM_LINKST_IN_L0 0x11
138	#define APPL_DEBUG_LTSSM_STATE_MASK GENMASK(8, 3)
139	#define APPL_DEBUG_LTSSM_STATE_SHIFT 3
140	#define LTSSM_STATE_PRE_DETECT 5
141
142	#define APPL_RADM_STATUS 0xE4
143	#define APPL_PM_XMT_TURNOFF_STATE BIT(0)
144
145	#define APPL_DM_TYPE 0x100
146	#define APPL_DM_TYPE_MASK GENMASK(3, 0)
147	#define APPL_DM_TYPE_RP 0x4
148	#define APPL_DM_TYPE_EP 0x0
149
150	#define APPL_CFG_BASE_ADDR 0x104
151	#define APPL_CFG_BASE_ADDR_MASK GENMASK(31, 12)
152
153	#define APPL_CFG_IATU_DMA_BASE_ADDR 0x108
154	#define APPL_CFG_IATU_DMA_BASE_ADDR_MASK GENMASK(31, 18)
155
156	#define APPL_CFG_MISC 0x110
157	#define APPL_CFG_MISC_SLV_EP_MODE BIT(14)
158	#define APPL_CFG_MISC_ARCACHE_MASK GENMASK(13, 10)
159	#define APPL_CFG_MISC_ARCACHE_SHIFT 10
160	#define APPL_CFG_MISC_ARCACHE_VAL 3
161
162	#define APPL_CFG_SLCG_OVERRIDE 0x114
163	#define APPL_CFG_SLCG_OVERRIDE_SLCG_EN_MASTER BIT(0)
164
165	#define APPL_CAR_RESET_OVRD 0x12C
166	#define APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N BIT(0)
167
168	#define IO_BASE_IO_DECODE BIT(0)
169	#define IO_BASE_IO_DECODE_BIT8 BIT(8)
170
171	#define CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE BIT(0)
172	#define CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE BIT(16)
173
174	#define CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF 0x718
175	#define CFG_TIMER_CTRL_ACK_NAK_SHIFT (19)
176
177	#define N_FTS_VAL 52
178	#define FTS_VAL 52
179
180	#define PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT 0x8D0
181	#define AMBA_ERROR_RESPONSE_RRS_SHIFT 3
182	#define AMBA_ERROR_RESPONSE_RRS_MASK GENMASK(1, 0)
183	#define AMBA_ERROR_RESPONSE_RRS_OKAY 0
184	#define AMBA_ERROR_RESPONSE_RRS_OKAY_FFFFFFFF 1
185	#define AMBA_ERROR_RESPONSE_RRS_OKAY_FFFF0001 2
186
187	#define MSIX_ADDR_MATCH_LOW_OFF 0x940
188	#define MSIX_ADDR_MATCH_LOW_OFF_EN BIT(0)
189	#define MSIX_ADDR_MATCH_LOW_OFF_MASK GENMASK(31, 2)
190
191	#define MSIX_ADDR_MATCH_HIGH_OFF 0x944
192	#define MSIX_ADDR_MATCH_HIGH_OFF_MASK GENMASK(31, 0)
193
194	#define PORT_LOGIC_MSIX_DOORBELL 0x948
195
196	#define CAP_SPCIE_CAP_OFF 0x154
197	#define CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK GENMASK(3, 0)
198	#define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK GENMASK(11, 8)
199	#define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT 8
200
201	#define PME_ACK_TIMEOUT 10000
202
203	#define LTSSM_TIMEOUT 50000 /* 50ms */
204
205	#define GEN3_GEN4_EQ_PRESET_INIT 5
206
207	#define GEN1_CORE_CLK_FREQ 62500000
208	#define GEN2_CORE_CLK_FREQ 125000000
209	#define GEN3_CORE_CLK_FREQ 250000000
210	#define GEN4_CORE_CLK_FREQ 500000000
211
212	#define LTR_MSG_TIMEOUT (100 * 1000)
213
214	#define PERST_DEBOUNCE_TIME (5 * 1000)
215
216	#define EP_STATE_DISABLED 0
217	#define EP_STATE_ENABLED 1
218
219	static const unsigned int pcie_gen_freq[] = {
220	GEN1_CORE_CLK_FREQ, /* PCI_EXP_LNKSTA_CLS == 0; undefined */
221	GEN1_CORE_CLK_FREQ,
222	GEN2_CORE_CLK_FREQ,
223	GEN3_CORE_CLK_FREQ,
224	GEN4_CORE_CLK_FREQ
225	};
226
227	struct tegra_pcie_dw_of_data {
228	u32 version;
229	enum dw_pcie_device_mode mode;
230	bool has_msix_doorbell_access_fix;
231	bool has_sbr_reset_fix;
232	bool has_l1ss_exit_fix;
233	bool has_ltr_req_fix;
234	u32 cdm_chk_int_en_bit;
235	u32 gen4_preset_vec;
236	u8 n_fts[2];
237	};
238
239	struct tegra_pcie_dw {
240	struct device *dev;
241	struct resource *appl_res;
242	struct resource *dbi_res;
243	struct resource *atu_dma_res;
244	void __iomem *appl_base;
245	struct clk *core_clk;
246	struct reset_control *core_apb_rst;
247	struct reset_control *core_rst;
248	struct dw_pcie pci;
249	struct tegra_bpmp *bpmp;
250
251	struct tegra_pcie_dw_of_data *of_data;
252
253	bool supports_clkreq;
254	bool enable_cdm_check;
255	bool enable_srns;
256	bool link_state;
257	bool update_fc_fixup;
258	bool enable_ext_refclk;
259	u8 init_link_width;
260	u32 msi_ctrl_int;
261	u32 num_lanes;
262	u32 cid;
263	u32 cfg_link_cap_l1sub;
264	u32 ras_des_cap;
265	u32 pcie_cap_base;
266	u32 aspm_cmrt;
267	u32 aspm_pwr_on_t;
268	u32 aspm_l0s_enter_lat;
269
270	struct regulator *pex_ctl_supply;
271	struct regulator *slot_ctl_3v3;
272	struct regulator *slot_ctl_12v;
273
274	unsigned int phy_count;
275	struct phy **phys;
276
277	struct dentry *debugfs;
278
279	/* Endpoint mode specific */
280	struct gpio_desc *pex_rst_gpiod;
281	struct gpio_desc *pex_refclk_sel_gpiod;
282	unsigned int pex_rst_irq;
283	int ep_state;
284	long link_status;
285	struct icc_path *icc_path;
286	};
287
288	static inline struct tegra_pcie_dw *to_tegra_pcie(struct dw_pcie *pci)
289	{
290	return container_of(pci, struct tegra_pcie_dw, pci);
291	}
292
293	static inline void appl_writel(struct tegra_pcie_dw *pcie, const u32 value,
294	const u32 reg)
295	{
296	writel_relaxed(value, pcie->appl_base + reg);
297	}
298
299	static inline u32 appl_readl(struct tegra_pcie_dw *pcie, const u32 reg)
300	{
301	return readl_relaxed(pcie->appl_base + reg);
302	}
303
304	static void tegra_pcie_icc_set(struct tegra_pcie_dw *pcie)
305	{
306	struct dw_pcie *pci = &pcie->pci;
307	u32 val, speed, width;
308
309	val = dw_pcie_readw_dbi(pci, reg: pcie->pcie_cap_base + PCI_EXP_LNKSTA);
310
311	speed = FIELD_GET(PCI_EXP_LNKSTA_CLS, val);
312	width = FIELD_GET(PCI_EXP_LNKSTA_NLW, val);
313
314	val = width * PCIE_SPEED2MBS_ENC(pcie_link_speed[speed]);
315
316	if (icc_set_bw(path: pcie->icc_path, Mbps_to_icc(val), peak_bw: 0))
317	dev_err(pcie->dev, "can't set bw[%u]\n", val);
318
319	if (speed >= ARRAY_SIZE(pcie_gen_freq))
320	speed = 0;
321
322	clk_set_rate(clk: pcie->core_clk, rate: pcie_gen_freq[speed]);
323	}
324
325	static void apply_bad_link_workaround(struct dw_pcie_rp *pp)
326	{
327	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
328	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
329	u32 current_link_width;
330	u16 val;
331
332	/*
333	* NOTE:- Since this scenario is uncommon and link as such is not
334	* stable anyway, not waiting to confirm if link is really
335	* transitioning to Gen-2 speed
336	*/
337	val = dw_pcie_readw_dbi(pci, reg: pcie->pcie_cap_base + PCI_EXP_LNKSTA);
338	if (val & PCI_EXP_LNKSTA_LBMS) {
339	current_link_width = FIELD_GET(PCI_EXP_LNKSTA_NLW, val);
340	if (pcie->init_link_width > current_link_width) {
341	dev_warn(pci->dev, "PCIe link is bad, width reduced\n");
342	val = dw_pcie_readw_dbi(pci, reg: pcie->pcie_cap_base +
343	PCI_EXP_LNKCTL2);
344	val &= ~PCI_EXP_LNKCTL2_TLS;
345	val |= PCI_EXP_LNKCTL2_TLS_2_5GT;
346	dw_pcie_writew_dbi(pci, reg: pcie->pcie_cap_base +
347	PCI_EXP_LNKCTL2, val);
348
349	val = dw_pcie_readw_dbi(pci, reg: pcie->pcie_cap_base +
350	PCI_EXP_LNKCTL);
351	val |= PCI_EXP_LNKCTL_RL;
352	dw_pcie_writew_dbi(pci, reg: pcie->pcie_cap_base +
353	PCI_EXP_LNKCTL, val);
354	}
355	}
356	}
357
358	static irqreturn_t tegra_pcie_rp_irq_handler(int irq, void *arg)
359	{
360	struct tegra_pcie_dw *pcie = arg;
361	struct dw_pcie *pci = &pcie->pci;
362	struct dw_pcie_rp *pp = &pci->pp;
363	u32 val, status_l0, status_l1;
364	u16 val_w;
365
366	status_l0 = appl_readl(pcie, APPL_INTR_STATUS_L0);
367	if (status_l0 & APPL_INTR_STATUS_L0_LINK_STATE_INT) {
368	status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0);
369	appl_writel(pcie, value: status_l1, APPL_INTR_STATUS_L1_0_0);
370	if (!pcie->of_data->has_sbr_reset_fix &&
371	status_l1 & APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED) {
372	/* SBR & Surprise Link Down WAR */
373	val = appl_readl(pcie, APPL_CAR_RESET_OVRD);
374	val &= ~APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N;
375	appl_writel(pcie, value: val, APPL_CAR_RESET_OVRD);
376	udelay(usec: 1);
377	val = appl_readl(pcie, APPL_CAR_RESET_OVRD);
378	val |= APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N;
379	appl_writel(pcie, value: val, APPL_CAR_RESET_OVRD);
380
381	val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL);
382	val |= PORT_LOGIC_SPEED_CHANGE;
383	dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val);
384	}
385	}
386
387	if (status_l0 & APPL_INTR_STATUS_L0_INT_INT) {
388	status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_8_0);
389	if (status_l1 & APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS) {
390	appl_writel(pcie,
391	APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS,
392	APPL_INTR_STATUS_L1_8_0);
393	apply_bad_link_workaround(pp);
394	}
395	if (status_l1 & APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS) {
396	val_w = dw_pcie_readw_dbi(pci, reg: pcie->pcie_cap_base +
397	PCI_EXP_LNKSTA);
398	val_w |= PCI_EXP_LNKSTA_LBMS;
399	dw_pcie_writew_dbi(pci, reg: pcie->pcie_cap_base +
400	PCI_EXP_LNKSTA, val: val_w);
401
402	appl_writel(pcie,
403	APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS,
404	APPL_INTR_STATUS_L1_8_0);
405
406	val_w = dw_pcie_readw_dbi(pci, reg: pcie->pcie_cap_base +
407	PCI_EXP_LNKSTA);
408	dev_dbg(pci->dev, "Link Speed : Gen-%u\n", val_w &
409	PCI_EXP_LNKSTA_CLS);
410	}
411	}
412
413	if (status_l0 & APPL_INTR_STATUS_L0_CDM_REG_CHK_INT) {
414	status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_18);
415	val = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS);
416	if (status_l1 & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT) {
417	dev_info(pci->dev, "CDM check complete\n");
418	val |= PCIE_PL_CHK_REG_CHK_REG_COMPLETE;
419	}
420	if (status_l1 & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR) {
421	dev_err(pci->dev, "CDM comparison mismatch\n");
422	val |= PCIE_PL_CHK_REG_CHK_REG_COMPARISON_ERROR;
423	}
424	if (status_l1 & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR) {
425	dev_err(pci->dev, "CDM Logic error\n");
426	val |= PCIE_PL_CHK_REG_CHK_REG_LOGIC_ERROR;
427	}
428	dw_pcie_writel_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS, val);
429	val = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_ERR_ADDR);
430	dev_err(pci->dev, "CDM Error Address Offset = 0x%08X\n", val);
431	}
432
433	return IRQ_HANDLED;
434	}
435
436	static void pex_ep_event_hot_rst_done(struct tegra_pcie_dw *pcie)
437	{
438	u32 val;
439
440	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L0);
441	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
442	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
443	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
444	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
445	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
446	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
447	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
448	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
449	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
450	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
451	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
452	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
453	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
454	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
455	appl_writel(pcie, value: 0xFFFFFFFF, APPL_MSI_CTRL_2);
456
457	val = appl_readl(pcie, APPL_CTRL);
458	val |= APPL_CTRL_LTSSM_EN;
459	appl_writel(pcie, value: val, APPL_CTRL);
460	}
461
462	static irqreturn_t tegra_pcie_ep_irq_thread(int irq, void *arg)
463	{
464	struct tegra_pcie_dw *pcie = arg;
465	struct dw_pcie_ep *ep = &pcie->pci.ep;
466	struct dw_pcie *pci = &pcie->pci;
467	u32 val;
468
469	if (test_and_clear_bit(nr: 0, addr: &pcie->link_status))
470	dw_pcie_ep_linkup(ep);
471
472	tegra_pcie_icc_set(pcie);
473
474	if (pcie->of_data->has_ltr_req_fix)
475	return IRQ_HANDLED;
476
477	/* If EP doesn't advertise L1SS, just return */
478	val = dw_pcie_readl_dbi(pci, reg: pcie->cfg_link_cap_l1sub);
479	if (!(val & (PCI_L1SS_CAP_ASPM_L1_1 | PCI_L1SS_CAP_ASPM_L1_2)))
480	return IRQ_HANDLED;
481
482	/* Check if BME is set to '1' */
483	val = dw_pcie_readl_dbi(pci, PCI_COMMAND);
484	if (val & PCI_COMMAND_MASTER) {
485	ktime_t timeout;
486
487	/* 110us for both snoop and no-snoop */
488	val = FIELD_PREP(PCI_LTR_VALUE_MASK, 110) |
489	FIELD_PREP(PCI_LTR_SCALE_MASK, 2) |
490	LTR_MSG_REQ |
491	FIELD_PREP(PCI_LTR_NOSNOOP_VALUE, 110) |
492	FIELD_PREP(PCI_LTR_NOSNOOP_SCALE, 2) |
493	LTR_NOSNOOP_MSG_REQ;
494	appl_writel(pcie, value: val, APPL_LTR_MSG_1);
495
496	/* Send LTR upstream */
497	val = appl_readl(pcie, APPL_LTR_MSG_2);
498	val |= APPL_LTR_MSG_2_LTR_MSG_REQ_STATE;
499	appl_writel(pcie, value: val, APPL_LTR_MSG_2);
500
501	timeout = ktime_add_us(kt: ktime_get(), LTR_MSG_TIMEOUT);
502	for (;;) {
503	val = appl_readl(pcie, APPL_LTR_MSG_2);
504	if (!(val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE))
505	break;
506	if (ktime_after(cmp1: ktime_get(), cmp2: timeout))
507	break;
508	usleep_range(min: 1000, max: 1100);
509	}
510	if (val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE)
511	dev_err(pcie->dev, "Failed to send LTR message\n");
512	}
513
514	return IRQ_HANDLED;
515	}
516
517	static irqreturn_t tegra_pcie_ep_hard_irq(int irq, void *arg)
518	{
519	struct tegra_pcie_dw *pcie = arg;
520	int spurious = 1;
521	u32 status_l0, status_l1, link_status;
522
523	status_l0 = appl_readl(pcie, APPL_INTR_STATUS_L0);
524	if (status_l0 & APPL_INTR_STATUS_L0_LINK_STATE_INT) {
525	status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0);
526	appl_writel(pcie, value: status_l1, APPL_INTR_STATUS_L1_0_0);
527
528	if (status_l1 & APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE)
529	pex_ep_event_hot_rst_done(pcie);
530
531	if (status_l1 & APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED) {
532	link_status = appl_readl(pcie, APPL_LINK_STATUS);
533	if (link_status & APPL_LINK_STATUS_RDLH_LINK_UP) {
534	dev_dbg(pcie->dev, "Link is up with Host\n");
535	set_bit(nr: 0, addr: &pcie->link_status);
536	return IRQ_WAKE_THREAD;
537	}
538	}
539
540	spurious = 0;
541	}
542
543	if (status_l0 & APPL_INTR_STATUS_L0_PCI_CMD_EN_INT) {
544	status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_15);
545	appl_writel(pcie, value: status_l1, APPL_INTR_STATUS_L1_15);
546
547	if (status_l1 & APPL_INTR_STATUS_L1_15_CFG_BME_CHGED)
548	return IRQ_WAKE_THREAD;
549
550	spurious = 0;
551	}
552
553	if (spurious) {
554	dev_warn(pcie->dev, "Random interrupt (STATUS = 0x%08X)\n",
555	status_l0);
556	appl_writel(pcie, value: status_l0, APPL_INTR_STATUS_L0);
557	}
558
559	return IRQ_HANDLED;
560	}
561
562	static int tegra_pcie_dw_rd_own_conf(struct pci_bus *bus, u32 devfn, int where,
563	int size, u32 *val)
564	{
565	struct dw_pcie_rp *pp = bus->sysdata;
566	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
567	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
568
569	/*
570	* This is an endpoint mode specific register happen to appear even
571	* when controller is operating in root port mode and system hangs
572	* when it is accessed with link being in ASPM-L1 state.
573	* So skip accessing it altogether
574	*/
575	if (!pcie->of_data->has_msix_doorbell_access_fix &&
576	!PCI_SLOT(devfn) && where == PORT_LOGIC_MSIX_DOORBELL) {
577	*val = 0x00000000;
578	return PCIBIOS_SUCCESSFUL;
579	}
580
581	return pci_generic_config_read(bus, devfn, where, size, val);
582	}
583
584	static int tegra_pcie_dw_wr_own_conf(struct pci_bus *bus, u32 devfn, int where,
585	int size, u32 val)
586	{
587	struct dw_pcie_rp *pp = bus->sysdata;
588	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
589	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
590
591	/*
592	* This is an endpoint mode specific register happen to appear even
593	* when controller is operating in root port mode and system hangs
594	* when it is accessed with link being in ASPM-L1 state.
595	* So skip accessing it altogether
596	*/
597	if (!pcie->of_data->has_msix_doorbell_access_fix &&
598	!PCI_SLOT(devfn) && where == PORT_LOGIC_MSIX_DOORBELL)
599	return PCIBIOS_SUCCESSFUL;
600
601	return pci_generic_config_write(bus, devfn, where, size, val);
602	}
603
604	static struct pci_ops tegra_pci_ops = {
605	.map_bus = dw_pcie_own_conf_map_bus,
606	.read = tegra_pcie_dw_rd_own_conf,
607	.write = tegra_pcie_dw_wr_own_conf,
608	};
609
610	#if defined(CONFIG_PCIEASPM)
611	static void disable_aspm_l11(struct tegra_pcie_dw *pcie)
612	{
613	u32 val;
614
615	val = dw_pcie_readl_dbi(pci: &pcie->pci, reg: pcie->cfg_link_cap_l1sub);
616	val &= ~PCI_L1SS_CAP_ASPM_L1_1;
617	dw_pcie_writel_dbi(pci: &pcie->pci, reg: pcie->cfg_link_cap_l1sub, val);
618	}
619
620	static void disable_aspm_l12(struct tegra_pcie_dw *pcie)
621	{
622	u32 val;
623
624	val = dw_pcie_readl_dbi(pci: &pcie->pci, reg: pcie->cfg_link_cap_l1sub);
625	val &= ~PCI_L1SS_CAP_ASPM_L1_2;
626	dw_pcie_writel_dbi(pci: &pcie->pci, reg: pcie->cfg_link_cap_l1sub, val);
627	}
628
629	static inline u32 event_counter_prog(struct tegra_pcie_dw *pcie, u32 event)
630	{
631	u32 val;
632
633	val = dw_pcie_readl_dbi(pci: &pcie->pci, reg: pcie->ras_des_cap +
634	PCIE_RAS_DES_EVENT_COUNTER_CONTROL);
635	val &= ~(EVENT_COUNTER_EVENT_SEL_MASK << EVENT_COUNTER_EVENT_SEL_SHIFT);
636	val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
637	val |= event << EVENT_COUNTER_EVENT_SEL_SHIFT;
638	val |= EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
639	dw_pcie_writel_dbi(pci: &pcie->pci, reg: pcie->ras_des_cap +
640	PCIE_RAS_DES_EVENT_COUNTER_CONTROL, val);
641	val = dw_pcie_readl_dbi(pci: &pcie->pci, reg: pcie->ras_des_cap +
642	PCIE_RAS_DES_EVENT_COUNTER_DATA);
643
644	return val;
645	}
646
647	static int aspm_state_cnt(struct seq_file *s, void *data)
648	{
649	struct tegra_pcie_dw *pcie = (struct tegra_pcie_dw *)
650	dev_get_drvdata(dev: s->private);
651	u32 val;
652
653	seq_printf(m: s, fmt: "Tx L0s entry count : %u\n",
654	event_counter_prog(pcie, EVENT_COUNTER_EVENT_Tx_L0S));
655
656	seq_printf(m: s, fmt: "Rx L0s entry count : %u\n",
657	event_counter_prog(pcie, EVENT_COUNTER_EVENT_Rx_L0S));
658
659	seq_printf(m: s, fmt: "Link L1 entry count : %u\n",
660	event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1));
661
662	seq_printf(m: s, fmt: "Link L1.1 entry count : %u\n",
663	event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_1));
664
665	seq_printf(m: s, fmt: "Link L1.2 entry count : %u\n",
666	event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_2));
667
668	/* Clear all counters */
669	dw_pcie_writel_dbi(pci: &pcie->pci, reg: pcie->ras_des_cap +
670	PCIE_RAS_DES_EVENT_COUNTER_CONTROL,
671	EVENT_COUNTER_ALL_CLEAR);
672
673	/* Re-enable counting */
674	val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
675	val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
676	dw_pcie_writel_dbi(pci: &pcie->pci, reg: pcie->ras_des_cap +
677	PCIE_RAS_DES_EVENT_COUNTER_CONTROL, val);
678
679	return 0;
680	}
681
682	static void init_host_aspm(struct tegra_pcie_dw *pcie)
683	{
684	struct dw_pcie *pci = &pcie->pci;
685	u32 val;
686
687	val = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_L1SS);
688	pcie->cfg_link_cap_l1sub = val + PCI_L1SS_CAP;
689
690	pcie->ras_des_cap = dw_pcie_find_ext_capability(pci: &pcie->pci,
691	PCI_EXT_CAP_ID_VNDR);
692
693	/* Enable ASPM counters */
694	val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
695	val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
696	dw_pcie_writel_dbi(pci, reg: pcie->ras_des_cap +
697	PCIE_RAS_DES_EVENT_COUNTER_CONTROL, val);
698
699	/* Program T_cmrt and T_pwr_on values */
700	val = dw_pcie_readl_dbi(pci, reg: pcie->cfg_link_cap_l1sub);
701	val &= ~(PCI_L1SS_CAP_CM_RESTORE_TIME | PCI_L1SS_CAP_P_PWR_ON_VALUE);
702	val |= (pcie->aspm_cmrt << 8);
703	val |= (pcie->aspm_pwr_on_t << 19);
704	dw_pcie_writel_dbi(pci, reg: pcie->cfg_link_cap_l1sub, val);
705
706	/* Program L0s and L1 entrance latencies */
707	val = dw_pcie_readl_dbi(pci, PCIE_PORT_AFR);
708	val &= ~PORT_AFR_L0S_ENTRANCE_LAT_MASK;
709	val |= (pcie->aspm_l0s_enter_lat << PORT_AFR_L0S_ENTRANCE_LAT_SHIFT);
710	val |= PORT_AFR_ENTER_ASPM;
711	dw_pcie_writel_dbi(pci, PCIE_PORT_AFR, val);
712	}
713
714	static void init_debugfs(struct tegra_pcie_dw *pcie)
715	{
716	struct device *dev = pcie->dev;
717	char *name;
718
719	name = devm_kasprintf(dev, GFP_KERNEL, fmt: "%pOFP", dev->of_node);
720	if (!name)
721	return;
722
723	pcie->debugfs = debugfs_create_dir(name, NULL);
724
725	debugfs_create_devm_seqfile(dev, name: "aspm_state_cnt", parent: pcie->debugfs,
726	read_fn: aspm_state_cnt);
727	}
728	#else
729	static inline void disable_aspm_l12(struct tegra_pcie_dw *pcie) { return; }
730	static inline void disable_aspm_l11(struct tegra_pcie_dw *pcie) { return; }
731	static inline void init_host_aspm(struct tegra_pcie_dw *pcie) { return; }
732	static inline void init_debugfs(struct tegra_pcie_dw *pcie) { return; }
733	#endif
734
735	static void tegra_pcie_enable_system_interrupts(struct dw_pcie_rp *pp)
736	{
737	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
738	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
739	u32 val;
740	u16 val_w;
741
742	val = appl_readl(pcie, APPL_INTR_EN_L0_0);
743	val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN;
744	appl_writel(pcie, value: val, APPL_INTR_EN_L0_0);
745
746	if (!pcie->of_data->has_sbr_reset_fix) {
747	val = appl_readl(pcie, APPL_INTR_EN_L1_0_0);
748	val |= APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN;
749	appl_writel(pcie, value: val, APPL_INTR_EN_L1_0_0);
750	}
751
752	if (pcie->enable_cdm_check) {
753	val = appl_readl(pcie, APPL_INTR_EN_L0_0);
754	val |= pcie->of_data->cdm_chk_int_en_bit;
755	appl_writel(pcie, value: val, APPL_INTR_EN_L0_0);
756
757	val = appl_readl(pcie, APPL_INTR_EN_L1_18);
758	val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR;
759	val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR;
760	appl_writel(pcie, value: val, APPL_INTR_EN_L1_18);
761	}
762
763	val_w = dw_pcie_readw_dbi(pci: &pcie->pci, reg: pcie->pcie_cap_base +
764	PCI_EXP_LNKSTA);
765	pcie->init_link_width = FIELD_GET(PCI_EXP_LNKSTA_NLW, val_w);
766
767	val_w = dw_pcie_readw_dbi(pci: &pcie->pci, reg: pcie->pcie_cap_base +
768	PCI_EXP_LNKCTL);
769	val_w |= PCI_EXP_LNKCTL_LBMIE;
770	dw_pcie_writew_dbi(pci: &pcie->pci, reg: pcie->pcie_cap_base + PCI_EXP_LNKCTL,
771	val: val_w);
772	}
773
774	static void tegra_pcie_enable_intx_interrupts(struct dw_pcie_rp *pp)
775	{
776	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
777	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
778	u32 val;
779
780	/* Enable INTX interrupt generation */
781	val = appl_readl(pcie, APPL_INTR_EN_L0_0);
782	val |= APPL_INTR_EN_L0_0_SYS_INTR_EN;
783	val |= APPL_INTR_EN_L0_0_INT_INT_EN;
784	appl_writel(pcie, value: val, APPL_INTR_EN_L0_0);
785
786	val = appl_readl(pcie, APPL_INTR_EN_L1_8_0);
787	val |= APPL_INTR_EN_L1_8_INTX_EN;
788	val |= APPL_INTR_EN_L1_8_AUTO_BW_INT_EN;
789	val |= APPL_INTR_EN_L1_8_BW_MGT_INT_EN;
790	if (IS_ENABLED(CONFIG_PCIEAER))
791	val |= APPL_INTR_EN_L1_8_AER_INT_EN;
792	appl_writel(pcie, value: val, APPL_INTR_EN_L1_8_0);
793	}
794
795	static void tegra_pcie_enable_msi_interrupts(struct dw_pcie_rp *pp)
796	{
797	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
798	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
799	u32 val;
800
801	/* Enable MSI interrupt generation */
802	val = appl_readl(pcie, APPL_INTR_EN_L0_0);
803	val |= APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN;
804	val |= APPL_INTR_EN_L0_0_MSI_RCV_INT_EN;
805	appl_writel(pcie, value: val, APPL_INTR_EN_L0_0);
806	}
807
808	static void tegra_pcie_enable_interrupts(struct dw_pcie_rp *pp)
809	{
810	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
811	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
812
813	/* Clear interrupt statuses before enabling interrupts */
814	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L0);
815	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
816	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
817	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
818	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
819	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
820	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
821	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
822	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
823	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
824	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
825	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
826	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
827	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
828	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
829
830	tegra_pcie_enable_system_interrupts(pp);
831	tegra_pcie_enable_intx_interrupts(pp);
832	if (IS_ENABLED(CONFIG_PCI_MSI))
833	tegra_pcie_enable_msi_interrupts(pp);
834	}
835
836	static void config_gen3_gen4_eq_presets(struct tegra_pcie_dw *pcie)
837	{
838	struct dw_pcie *pci = &pcie->pci;
839	u32 val, offset, i;
840
841	/* Program init preset */
842	for (i = 0; i < pcie->num_lanes; i++) {
843	val = dw_pcie_readw_dbi(pci, CAP_SPCIE_CAP_OFF + (i * 2));
844	val &= ~CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK;
845	val |= GEN3_GEN4_EQ_PRESET_INIT;
846	val &= ~CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK;
847	val |= (GEN3_GEN4_EQ_PRESET_INIT <<
848	CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT);
849	dw_pcie_writew_dbi(pci, CAP_SPCIE_CAP_OFF + (i * 2), val);
850
851	offset = dw_pcie_find_ext_capability(pci,
852	PCI_EXT_CAP_ID_PL_16GT) +
853	PCI_PL_16GT_LE_CTRL;
854	val = dw_pcie_readb_dbi(pci, reg: offset + i);
855	val &= ~PCI_PL_16GT_LE_CTRL_DSP_TX_PRESET_MASK;
856	val |= GEN3_GEN4_EQ_PRESET_INIT;
857	val &= ~PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_MASK;
858	val |= (GEN3_GEN4_EQ_PRESET_INIT <<
859	PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_SHIFT);
860	dw_pcie_writeb_dbi(pci, reg: offset + i, val);
861	}
862
863	val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
864	val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
865	dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
866
867	val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF);
868	val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC;
869	val |= FIELD_PREP(GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC, 0x3ff);
870	val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE;
871	dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val);
872
873	val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
874	val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
875	val |= (0x1 << GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT);
876	dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
877
878	val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF);
879	val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC;
880	val |= FIELD_PREP(GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC,
881	pcie->of_data->gen4_preset_vec);
882	val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE;
883	dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val);
884
885	val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
886	val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
887	dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
888	}
889
890	static int tegra_pcie_dw_host_init(struct dw_pcie_rp *pp)
891	{
892	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
893	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
894	u32 val;
895	u16 val_16;
896
897	pp->bridge->ops = &tegra_pci_ops;
898
899	if (!pcie->pcie_cap_base)
900	pcie->pcie_cap_base = dw_pcie_find_capability(pci: &pcie->pci,
901	PCI_CAP_ID_EXP);
902
903	val = dw_pcie_readl_dbi(pci, PCI_IO_BASE);
904	val &= ~(IO_BASE_IO_DECODE | IO_BASE_IO_DECODE_BIT8);
905	dw_pcie_writel_dbi(pci, PCI_IO_BASE, val);
906
907	val = dw_pcie_readl_dbi(pci, PCI_PREF_MEMORY_BASE);
908	val |= CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE;
909	val |= CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE;
910	dw_pcie_writel_dbi(pci, PCI_PREF_MEMORY_BASE, val);
911
912	dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, val: 0);
913
914	/* Enable as 0xFFFF0001 response for RRS */
915	val = dw_pcie_readl_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT);
916	val &= ~(AMBA_ERROR_RESPONSE_RRS_MASK << AMBA_ERROR_RESPONSE_RRS_SHIFT);
917	val |= (AMBA_ERROR_RESPONSE_RRS_OKAY_FFFF0001 <<
918	AMBA_ERROR_RESPONSE_RRS_SHIFT);
919	dw_pcie_writel_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT, val);
920
921	/* Clear Slot Clock Configuration bit if SRNS configuration */
922	if (pcie->enable_srns) {
923	val_16 = dw_pcie_readw_dbi(pci, reg: pcie->pcie_cap_base +
924	PCI_EXP_LNKSTA);
925	val_16 &= ~PCI_EXP_LNKSTA_SLC;
926	dw_pcie_writew_dbi(pci, reg: pcie->pcie_cap_base + PCI_EXP_LNKSTA,
927	val: val_16);
928	}
929
930	config_gen3_gen4_eq_presets(pcie);
931
932	init_host_aspm(pcie);
933
934	/* Disable ASPM-L1SS advertisement if there is no CLKREQ routing */
935	if (!pcie->supports_clkreq) {
936	disable_aspm_l11(pcie);
937	disable_aspm_l12(pcie);
938	}
939
940	if (!pcie->of_data->has_l1ss_exit_fix) {
941	val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
942	val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL;
943	dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
944	}
945
946	if (pcie->update_fc_fixup) {
947	val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF);
948	val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT;
949	dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val);
950	}
951
952	clk_set_rate(clk: pcie->core_clk, GEN4_CORE_CLK_FREQ);
953
954	return 0;
955	}
956
957	static int tegra_pcie_dw_start_link(struct dw_pcie *pci)
958	{
959	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
960	struct dw_pcie_rp *pp = &pci->pp;
961	u32 val, offset, tmp;
962	bool retry = true;
963
964	if (pcie->of_data->mode == DW_PCIE_EP_TYPE) {
965	enable_irq(irq: pcie->pex_rst_irq);
966	return 0;
967	}
968
969	retry_link:
970	/* Assert RST */
971	val = appl_readl(pcie, APPL_PINMUX);
972	val &= ~APPL_PINMUX_PEX_RST;
973	appl_writel(pcie, value: val, APPL_PINMUX);
974
975	usleep_range(min: 100, max: 200);
976
977	/* Enable LTSSM */
978	val = appl_readl(pcie, APPL_CTRL);
979	val |= APPL_CTRL_LTSSM_EN;
980	appl_writel(pcie, value: val, APPL_CTRL);
981
982	/* De-assert RST */
983	val = appl_readl(pcie, APPL_PINMUX);
984	val |= APPL_PINMUX_PEX_RST;
985	appl_writel(pcie, value: val, APPL_PINMUX);
986
987	msleep(msecs: 100);
988
989	if (dw_pcie_wait_for_link(pci)) {
990	if (!retry)
991	return 0;
992	/*
993	* There are some endpoints which can't get the link up if
994	* root port has Data Link Feature (DLF) enabled.
995	* Refer Spec rev 4.0 ver 1.0 sec 3.4.2 & 7.7.4 for more info
996	* on Scaled Flow Control and DLF.
997	* So, need to confirm that is indeed the case here and attempt
998	* link up once again with DLF disabled.
999	*/
1000	val = appl_readl(pcie, APPL_DEBUG);
1001	val &= APPL_DEBUG_LTSSM_STATE_MASK;
1002	val >>= APPL_DEBUG_LTSSM_STATE_SHIFT;
1003	tmp = appl_readl(pcie, APPL_LINK_STATUS);
1004	tmp &= APPL_LINK_STATUS_RDLH_LINK_UP;
1005	if (!(val == 0x11 && !tmp)) {
1006	/* Link is down for all good reasons */
1007	return 0;
1008	}
1009
1010	dev_info(pci->dev, "Link is down in DLL");
1011	dev_info(pci->dev, "Trying again with DLFE disabled\n");
1012	/* Disable LTSSM */
1013	val = appl_readl(pcie, APPL_CTRL);
1014	val &= ~APPL_CTRL_LTSSM_EN;
1015	appl_writel(pcie, value: val, APPL_CTRL);
1016
1017	reset_control_assert(rstc: pcie->core_rst);
1018	reset_control_deassert(rstc: pcie->core_rst);
1019
1020	offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_DLF);
1021	val = dw_pcie_readl_dbi(pci, reg: offset + PCI_DLF_CAP);
1022	val &= ~PCI_DLF_EXCHANGE_ENABLE;
1023	dw_pcie_writel_dbi(pci, reg: offset + PCI_DLF_CAP, val);
1024
1025	tegra_pcie_dw_host_init(pp);
1026	dw_pcie_setup_rc(pp);
1027
1028	retry = false;
1029	goto retry_link;
1030	}
1031
1032	tegra_pcie_icc_set(pcie);
1033
1034	tegra_pcie_enable_interrupts(pp);
1035
1036	return 0;
1037	}
1038
1039	static bool tegra_pcie_dw_link_up(struct dw_pcie *pci)
1040	{
1041	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1042	u32 val = dw_pcie_readw_dbi(pci, reg: pcie->pcie_cap_base + PCI_EXP_LNKSTA);
1043
1044	return val & PCI_EXP_LNKSTA_DLLLA;
1045	}
1046
1047	static void tegra_pcie_dw_stop_link(struct dw_pcie *pci)
1048	{
1049	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1050
1051	disable_irq(irq: pcie->pex_rst_irq);
1052	}
1053
1054	static const struct dw_pcie_ops tegra_dw_pcie_ops = {
1055	.link_up = tegra_pcie_dw_link_up,
1056	.start_link = tegra_pcie_dw_start_link,
1057	.stop_link = tegra_pcie_dw_stop_link,
1058	};
1059
1060	static const struct dw_pcie_host_ops tegra_pcie_dw_host_ops = {
1061	.init = tegra_pcie_dw_host_init,
1062	};
1063
1064	static void tegra_pcie_disable_phy(struct tegra_pcie_dw *pcie)
1065	{
1066	unsigned int phy_count = pcie->phy_count;
1067
1068	while (phy_count--) {
1069	phy_power_off(phy: pcie->phys[phy_count]);
1070	phy_exit(phy: pcie->phys[phy_count]);
1071	}
1072	}
1073
1074	static int tegra_pcie_enable_phy(struct tegra_pcie_dw *pcie)
1075	{
1076	unsigned int i;
1077	int ret;
1078
1079	for (i = 0; i < pcie->phy_count; i++) {
1080	ret = phy_init(phy: pcie->phys[i]);
1081	if (ret < 0)
1082	goto phy_power_off;
1083
1084	ret = phy_power_on(phy: pcie->phys[i]);
1085	if (ret < 0)
1086	goto phy_exit;
1087	}
1088
1089	return 0;
1090
1091	phy_power_off:
1092	while (i--) {
1093	phy_power_off(phy: pcie->phys[i]);
1094	phy_exit:
1095	phy_exit(phy: pcie->phys[i]);
1096	}
1097
1098	return ret;
1099	}
1100
1101	static int tegra_pcie_dw_parse_dt(struct tegra_pcie_dw *pcie)
1102	{
1103	struct platform_device *pdev = to_platform_device(pcie->dev);
1104	struct device_node *np = pcie->dev->of_node;
1105	int ret;
1106
1107	pcie->dbi_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi");
1108	if (!pcie->dbi_res) {
1109	dev_err(pcie->dev, "Failed to find \"dbi\" region\n");
1110	return -ENODEV;
1111	}
1112
1113	ret = of_property_read_u32(np, propname: "nvidia,aspm-cmrt-us", out_value: &pcie->aspm_cmrt);
1114	if (ret < 0) {
1115	dev_info(pcie->dev, "Failed to read ASPM T_cmrt: %d\n", ret);
1116	return ret;
1117	}
1118
1119	ret = of_property_read_u32(np, propname: "nvidia,aspm-pwr-on-t-us",
1120	out_value: &pcie->aspm_pwr_on_t);
1121	if (ret < 0)
1122	dev_info(pcie->dev, "Failed to read ASPM Power On time: %d\n",
1123	ret);
1124
1125	ret = of_property_read_u32(np, propname: "nvidia,aspm-l0s-entrance-latency-us",
1126	out_value: &pcie->aspm_l0s_enter_lat);
1127	if (ret < 0)
1128	dev_info(pcie->dev,
1129	"Failed to read ASPM L0s Entrance latency: %d\n", ret);
1130
1131	ret = of_property_read_u32(np, propname: "num-lanes", out_value: &pcie->num_lanes);
1132	if (ret < 0) {
1133	dev_err(pcie->dev, "Failed to read num-lanes: %d\n", ret);
1134	return ret;
1135	}
1136
1137	ret = of_property_read_u32_index(np, propname: "nvidia,bpmp", index: 1, out_value: &pcie->cid);
1138	if (ret) {
1139	dev_err(pcie->dev, "Failed to read Controller-ID: %d\n", ret);
1140	return ret;
1141	}
1142
1143	ret = of_property_count_strings(np, propname: "phy-names");
1144	if (ret < 0) {
1145	dev_err(pcie->dev, "Failed to find PHY entries: %d\n",
1146	ret);
1147	return ret;
1148	}
1149	pcie->phy_count = ret;
1150
1151	if (of_property_read_bool(np, propname: "nvidia,update-fc-fixup"))
1152	pcie->update_fc_fixup = true;
1153
1154	/* RP using an external REFCLK is supported only in Tegra234 */
1155	if (pcie->of_data->version == TEGRA194_DWC_IP_VER) {
1156	if (pcie->of_data->mode == DW_PCIE_EP_TYPE)
1157	pcie->enable_ext_refclk = true;
1158	} else {
1159	pcie->enable_ext_refclk =
1160	of_property_read_bool(np: pcie->dev->of_node,
1161	propname: "nvidia,enable-ext-refclk");
1162	}
1163
1164	pcie->supports_clkreq =
1165	of_property_read_bool(np: pcie->dev->of_node, propname: "supports-clkreq");
1166
1167	pcie->enable_cdm_check =
1168	of_property_read_bool(np, propname: "snps,enable-cdm-check");
1169
1170	if (pcie->of_data->version == TEGRA234_DWC_IP_VER)
1171	pcie->enable_srns =
1172	of_property_read_bool(np, propname: "nvidia,enable-srns");
1173
1174	if (pcie->of_data->mode == DW_PCIE_RC_TYPE)
1175	return 0;
1176
1177	/* Endpoint mode specific DT entries */
1178	pcie->pex_rst_gpiod = devm_gpiod_get(dev: pcie->dev, con_id: "reset", flags: GPIOD_IN);
1179	if (IS_ERR(ptr: pcie->pex_rst_gpiod)) {
1180	int err = PTR_ERR(ptr: pcie->pex_rst_gpiod);
1181	const char *level = KERN_ERR;
1182
1183	if (err == -EPROBE_DEFER)
1184	level = KERN_DEBUG;
1185
1186	dev_printk(level, pcie->dev,
1187	dev_fmt("Failed to get PERST GPIO: %d\n"),
1188	err);
1189	return err;
1190	}
1191
1192	pcie->pex_refclk_sel_gpiod = devm_gpiod_get(dev: pcie->dev,
1193	con_id: "nvidia,refclk-select",
1194	flags: GPIOD_OUT_HIGH);
1195	if (IS_ERR(ptr: pcie->pex_refclk_sel_gpiod)) {
1196	int err = PTR_ERR(ptr: pcie->pex_refclk_sel_gpiod);
1197	const char *level = KERN_ERR;
1198
1199	if (err == -EPROBE_DEFER)
1200	level = KERN_DEBUG;
1201
1202	dev_printk(level, pcie->dev,
1203	dev_fmt("Failed to get REFCLK select GPIOs: %d\n"),
1204	err);
1205	pcie->pex_refclk_sel_gpiod = NULL;
1206	}
1207
1208	return 0;
1209	}
1210
1211	static int tegra_pcie_bpmp_set_ctrl_state(struct tegra_pcie_dw *pcie,
1212	bool enable)
1213	{
1214	struct mrq_uphy_response resp;
1215	struct tegra_bpmp_message msg;
1216	struct mrq_uphy_request req;
1217
1218	/*
1219	* Controller-5 doesn't need to have its state set by BPMP-FW in
1220	* Tegra194
1221	*/
1222	if (pcie->of_data->version == TEGRA194_DWC_IP_VER && pcie->cid == 5)
1223	return 0;
1224
1225	memset(&req, 0, sizeof(req));
1226	memset(&resp, 0, sizeof(resp));
1227
1228	req.cmd = CMD_UPHY_PCIE_CONTROLLER_STATE;
1229	req.controller_state.pcie_controller = pcie->cid;
1230	req.controller_state.enable = enable;
1231
1232	memset(&msg, 0, sizeof(msg));
1233	msg.mrq = MRQ_UPHY;
1234	msg.tx.data = &req;
1235	msg.tx.size = sizeof(req);
1236	msg.rx.data = &resp;
1237	msg.rx.size = sizeof(resp);
1238
1239	return tegra_bpmp_transfer(bpmp: pcie->bpmp, msg: &msg);
1240	}
1241
1242	static int tegra_pcie_bpmp_set_pll_state(struct tegra_pcie_dw *pcie,
1243	bool enable)
1244	{
1245	struct mrq_uphy_response resp;
1246	struct tegra_bpmp_message msg;
1247	struct mrq_uphy_request req;
1248
1249	memset(&req, 0, sizeof(req));
1250	memset(&resp, 0, sizeof(resp));
1251
1252	if (enable) {
1253	req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_INIT;
1254	req.ep_ctrlr_pll_init.ep_controller = pcie->cid;
1255	} else {
1256	req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_OFF;
1257	req.ep_ctrlr_pll_off.ep_controller = pcie->cid;
1258	}
1259
1260	memset(&msg, 0, sizeof(msg));
1261	msg.mrq = MRQ_UPHY;
1262	msg.tx.data = &req;
1263	msg.tx.size = sizeof(req);
1264	msg.rx.data = &resp;
1265	msg.rx.size = sizeof(resp);
1266
1267	return tegra_bpmp_transfer(bpmp: pcie->bpmp, msg: &msg);
1268	}
1269
1270	static void tegra_pcie_downstream_dev_to_D0(struct tegra_pcie_dw *pcie)
1271	{
1272	struct dw_pcie_rp *pp = &pcie->pci.pp;
1273	struct pci_bus *child, *root_bus = NULL;
1274	struct pci_dev *pdev;
1275
1276	/*
1277	* link doesn't go into L2 state with some of the endpoints with Tegra
1278	* if they are not in D0 state. So, need to make sure that immediate
1279	* downstream devices are in D0 state before sending PME_TurnOff to put
1280	* link into L2 state.
1281	* This is as per PCI Express Base r4.0 v1.0 September 27-2017,
1282	* 5.2 Link State Power Management (Page #428).
1283	*/
1284
1285	list_for_each_entry(child, &pp->bridge->bus->children, node) {
1286	/* Bring downstream devices to D0 if they are not already in */
1287	if (child->parent == pp->bridge->bus) {
1288	root_bus = child;
1289	break;
1290	}
1291	}
1292
1293	if (!root_bus) {
1294	dev_err(pcie->dev, "Failed to find downstream devices\n");
1295	return;
1296	}
1297
1298	list_for_each_entry(pdev, &root_bus->devices, bus_list) {
1299	if (PCI_SLOT(pdev->devfn) == 0) {
1300	if (pci_set_power_state(dev: pdev, PCI_D0))
1301	dev_err(pcie->dev,
1302	"Failed to transition %s to D0 state\n",
1303	dev_name(&pdev->dev));
1304	}
1305	}
1306	}
1307
1308	static int tegra_pcie_get_slot_regulators(struct tegra_pcie_dw *pcie)
1309	{
1310	pcie->slot_ctl_3v3 = devm_regulator_get_optional(dev: pcie->dev, id: "vpcie3v3");
1311	if (IS_ERR(ptr: pcie->slot_ctl_3v3)) {
1312	if (PTR_ERR(ptr: pcie->slot_ctl_3v3) != -ENODEV)
1313	return PTR_ERR(ptr: pcie->slot_ctl_3v3);
1314
1315	pcie->slot_ctl_3v3 = NULL;
1316	}
1317
1318	pcie->slot_ctl_12v = devm_regulator_get_optional(dev: pcie->dev, id: "vpcie12v");
1319	if (IS_ERR(ptr: pcie->slot_ctl_12v)) {
1320	if (PTR_ERR(ptr: pcie->slot_ctl_12v) != -ENODEV)
1321	return PTR_ERR(ptr: pcie->slot_ctl_12v);
1322
1323	pcie->slot_ctl_12v = NULL;
1324	}
1325
1326	return 0;
1327	}
1328
1329	static int tegra_pcie_enable_slot_regulators(struct tegra_pcie_dw *pcie)
1330	{
1331	int ret;
1332
1333	if (pcie->slot_ctl_3v3) {
1334	ret = regulator_enable(regulator: pcie->slot_ctl_3v3);
1335	if (ret < 0) {
1336	dev_err(pcie->dev,
1337	"Failed to enable 3.3V slot supply: %d\n", ret);
1338	return ret;
1339	}
1340	}
1341
1342	if (pcie->slot_ctl_12v) {
1343	ret = regulator_enable(regulator: pcie->slot_ctl_12v);
1344	if (ret < 0) {
1345	dev_err(pcie->dev,
1346	"Failed to enable 12V slot supply: %d\n", ret);
1347	goto fail_12v_enable;
1348	}
1349	}
1350
1351	/*
1352	* According to PCI Express Card Electromechanical Specification
1353	* Revision 1.1, Table-2.4, T_PVPERL (Power stable to PERST# inactive)
1354	* should be a minimum of 100ms.
1355	*/
1356	if (pcie->slot_ctl_3v3 || pcie->slot_ctl_12v)
1357	msleep(msecs: 100);
1358
1359	return 0;
1360
1361	fail_12v_enable:
1362	if (pcie->slot_ctl_3v3)
1363	regulator_disable(regulator: pcie->slot_ctl_3v3);
1364	return ret;
1365	}
1366
1367	static void tegra_pcie_disable_slot_regulators(struct tegra_pcie_dw *pcie)
1368	{
1369	if (pcie->slot_ctl_12v)
1370	regulator_disable(regulator: pcie->slot_ctl_12v);
1371	if (pcie->slot_ctl_3v3)
1372	regulator_disable(regulator: pcie->slot_ctl_3v3);
1373	}
1374
1375	static int tegra_pcie_config_controller(struct tegra_pcie_dw *pcie,
1376	bool en_hw_hot_rst)
1377	{
1378	int ret;
1379	u32 val;
1380
1381	ret = tegra_pcie_bpmp_set_ctrl_state(pcie, enable: true);
1382	if (ret) {
1383	dev_err(pcie->dev,
1384	"Failed to enable controller %u: %d\n", pcie->cid, ret);
1385	return ret;
1386	}
1387
1388	if (pcie->enable_ext_refclk) {
1389	ret = tegra_pcie_bpmp_set_pll_state(pcie, enable: true);
1390	if (ret) {
1391	dev_err(pcie->dev, "Failed to init UPHY: %d\n", ret);
1392	goto fail_pll_init;
1393	}
1394	}
1395
1396	ret = tegra_pcie_enable_slot_regulators(pcie);
1397	if (ret < 0)
1398	goto fail_slot_reg_en;
1399
1400	ret = regulator_enable(regulator: pcie->pex_ctl_supply);
1401	if (ret < 0) {
1402	dev_err(pcie->dev, "Failed to enable regulator: %d\n", ret);
1403	goto fail_reg_en;
1404	}
1405
1406	ret = clk_prepare_enable(clk: pcie->core_clk);
1407	if (ret) {
1408	dev_err(pcie->dev, "Failed to enable core clock: %d\n", ret);
1409	goto fail_core_clk;
1410	}
1411
1412	ret = reset_control_deassert(rstc: pcie->core_apb_rst);
1413	if (ret) {
1414	dev_err(pcie->dev, "Failed to deassert core APB reset: %d\n",
1415	ret);
1416	goto fail_core_apb_rst;
1417	}
1418
1419	if (en_hw_hot_rst || pcie->of_data->has_sbr_reset_fix) {
1420	/* Enable HW_HOT_RST mode */
1421	val = appl_readl(pcie, APPL_CTRL);
1422	val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
1423	APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
1424	val |= (APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST_LTSSM_EN <<
1425	APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
1426	val |= APPL_CTRL_HW_HOT_RST_EN;
1427	appl_writel(pcie, value: val, APPL_CTRL);
1428	}
1429
1430	ret = tegra_pcie_enable_phy(pcie);
1431	if (ret) {
1432	dev_err(pcie->dev, "Failed to enable PHY: %d\n", ret);
1433	goto fail_phy;
1434	}
1435
1436	/* Update CFG base address */
1437	appl_writel(pcie, value: pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK,
1438	APPL_CFG_BASE_ADDR);
1439
1440	/* Configure this core for RP mode operation */
1441	appl_writel(pcie, APPL_DM_TYPE_RP, APPL_DM_TYPE);
1442
1443	appl_writel(pcie, value: 0x0, APPL_CFG_SLCG_OVERRIDE);
1444
1445	val = appl_readl(pcie, APPL_CTRL);
1446	appl_writel(pcie, value: val | APPL_CTRL_SYS_PRE_DET_STATE, APPL_CTRL);
1447
1448	val = appl_readl(pcie, APPL_CFG_MISC);
1449	val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT);
1450	appl_writel(pcie, value: val, APPL_CFG_MISC);
1451
1452	if (pcie->enable_srns || pcie->enable_ext_refclk) {
1453	/*
1454	* When Tegra PCIe RP is using external clock, it cannot supply
1455	* same clock to its downstream hierarchy. Hence, gate PCIe RP
1456	* REFCLK out pads when RP & EP are using separate clocks or RP
1457	* is using an external REFCLK.
1458	*/
1459	val = appl_readl(pcie, APPL_PINMUX);
1460	val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;
1461	val &= ~APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;
1462	appl_writel(pcie, value: val, APPL_PINMUX);
1463	}
1464
1465	if (!pcie->supports_clkreq) {
1466	val = appl_readl(pcie, APPL_PINMUX);
1467	val |= APPL_PINMUX_CLKREQ_OVERRIDE_EN;
1468	val &= ~APPL_PINMUX_CLKREQ_OVERRIDE;
1469	appl_writel(pcie, value: val, APPL_PINMUX);
1470	}
1471
1472	/* Update iATU_DMA base address */
1473	appl_writel(pcie,
1474	value: pcie->atu_dma_res->start & APPL_CFG_IATU_DMA_BASE_ADDR_MASK,
1475	APPL_CFG_IATU_DMA_BASE_ADDR);
1476
1477	reset_control_deassert(rstc: pcie->core_rst);
1478
1479	return ret;
1480
1481	fail_phy:
1482	reset_control_assert(rstc: pcie->core_apb_rst);
1483	fail_core_apb_rst:
1484	clk_disable_unprepare(clk: pcie->core_clk);
1485	fail_core_clk:
1486	regulator_disable(regulator: pcie->pex_ctl_supply);
1487	fail_reg_en:
1488	tegra_pcie_disable_slot_regulators(pcie);
1489	fail_slot_reg_en:
1490	if (pcie->enable_ext_refclk)
1491	tegra_pcie_bpmp_set_pll_state(pcie, enable: false);
1492	fail_pll_init:
1493	tegra_pcie_bpmp_set_ctrl_state(pcie, enable: false);
1494
1495	return ret;
1496	}
1497
1498	static void tegra_pcie_unconfig_controller(struct tegra_pcie_dw *pcie)
1499	{
1500	int ret;
1501
1502	ret = reset_control_assert(rstc: pcie->core_rst);
1503	if (ret)
1504	dev_err(pcie->dev, "Failed to assert \"core\" reset: %d\n", ret);
1505
1506	tegra_pcie_disable_phy(pcie);
1507
1508	ret = reset_control_assert(rstc: pcie->core_apb_rst);
1509	if (ret)
1510	dev_err(pcie->dev, "Failed to assert APB reset: %d\n", ret);
1511
1512	clk_disable_unprepare(clk: pcie->core_clk);
1513
1514	ret = regulator_disable(regulator: pcie->pex_ctl_supply);
1515	if (ret)
1516	dev_err(pcie->dev, "Failed to disable regulator: %d\n", ret);
1517
1518	tegra_pcie_disable_slot_regulators(pcie);
1519
1520	if (pcie->enable_ext_refclk) {
1521	ret = tegra_pcie_bpmp_set_pll_state(pcie, enable: false);
1522	if (ret)
1523	dev_err(pcie->dev, "Failed to deinit UPHY: %d\n", ret);
1524	}
1525
1526	ret = tegra_pcie_bpmp_set_ctrl_state(pcie, enable: false);
1527	if (ret)
1528	dev_err(pcie->dev, "Failed to disable controller %d: %d\n",
1529	pcie->cid, ret);
1530	}
1531
1532	static int tegra_pcie_init_controller(struct tegra_pcie_dw *pcie)
1533	{
1534	struct dw_pcie *pci = &pcie->pci;
1535	struct dw_pcie_rp *pp = &pci->pp;
1536	int ret;
1537
1538	ret = tegra_pcie_config_controller(pcie, en_hw_hot_rst: false);
1539	if (ret < 0)
1540	return ret;
1541
1542	pp->ops = &tegra_pcie_dw_host_ops;
1543
1544	ret = dw_pcie_host_init(pp);
1545	if (ret < 0) {
1546	dev_err(pcie->dev, "Failed to add PCIe port: %d\n", ret);
1547	goto fail_host_init;
1548	}
1549
1550	return 0;
1551
1552	fail_host_init:
1553	tegra_pcie_unconfig_controller(pcie);
1554	return ret;
1555	}
1556
1557	static int tegra_pcie_try_link_l2(struct tegra_pcie_dw *pcie)
1558	{
1559	u32 val;
1560
1561	if (!tegra_pcie_dw_link_up(pci: &pcie->pci))
1562	return 0;
1563
1564	val = appl_readl(pcie, APPL_RADM_STATUS);
1565	val |= APPL_PM_XMT_TURNOFF_STATE;
1566	appl_writel(pcie, value: val, APPL_RADM_STATUS);
1567
1568	return readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG, val,
1569	val & APPL_DEBUG_PM_LINKST_IN_L2_LAT,
1570	1, PME_ACK_TIMEOUT);
1571	}
1572
1573	static void tegra_pcie_dw_pme_turnoff(struct tegra_pcie_dw *pcie)
1574	{
1575	u32 data;
1576	int err;
1577
1578	if (!tegra_pcie_dw_link_up(pci: &pcie->pci)) {
1579	dev_dbg(pcie->dev, "PCIe link is not up...!\n");
1580	return;
1581	}
1582
1583	/*
1584	* PCIe controller exits from L2 only if reset is applied, so
1585	* controller doesn't handle interrupts. But in cases where
1586	* L2 entry fails, PERST# is asserted which can trigger surprise
1587	* link down AER. However this function call happens in
1588	* suspend_noirq(), so AER interrupt will not be processed.
1589	* Disable all interrupts to avoid such a scenario.
1590	*/
1591	appl_writel(pcie, value: 0x0, APPL_INTR_EN_L0_0);
1592
1593	if (tegra_pcie_try_link_l2(pcie)) {
1594	dev_info(pcie->dev, "Link didn't transition to L2 state\n");
1595	/*
1596	* TX lane clock freq will reset to Gen1 only if link is in L2
1597	* or detect state.
1598	* So apply pex_rst to end point to force RP to go into detect
1599	* state
1600	*/
1601	data = appl_readl(pcie, APPL_PINMUX);
1602	data &= ~APPL_PINMUX_PEX_RST;
1603	appl_writel(pcie, value: data, APPL_PINMUX);
1604
1605	/*
1606	* Some cards do not go to detect state even after de-asserting
1607	* PERST#. So, de-assert LTSSM to bring link to detect state.
1608	*/
1609	data = readl(addr: pcie->appl_base + APPL_CTRL);
1610	data &= ~APPL_CTRL_LTSSM_EN;
1611	writel(val: data, addr: pcie->appl_base + APPL_CTRL);
1612
1613	err = readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG,
1614	data,
1615	((data &
1616	APPL_DEBUG_LTSSM_STATE_MASK) >>
1617	APPL_DEBUG_LTSSM_STATE_SHIFT) ==
1618	LTSSM_STATE_PRE_DETECT,
1619	1, LTSSM_TIMEOUT);
1620	if (err)
1621	dev_info(pcie->dev, "Link didn't go to detect state\n");
1622	}
1623	/*
1624	* DBI registers may not be accessible after this as PLL-E would be
1625	* down depending on how CLKREQ is pulled by end point
1626	*/
1627	data = appl_readl(pcie, APPL_PINMUX);
1628	data |= (APPL_PINMUX_CLKREQ_OVERRIDE_EN | APPL_PINMUX_CLKREQ_OVERRIDE);
1629	/* Cut REFCLK to slot */
1630	data |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;
1631	data &= ~APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;
1632	appl_writel(pcie, value: data, APPL_PINMUX);
1633	}
1634
1635	static void tegra_pcie_deinit_controller(struct tegra_pcie_dw *pcie)
1636	{
1637	tegra_pcie_downstream_dev_to_D0(pcie);
1638	dw_pcie_host_deinit(pp: &pcie->pci.pp);
1639	tegra_pcie_dw_pme_turnoff(pcie);
1640	tegra_pcie_unconfig_controller(pcie);
1641	}
1642
1643	static int tegra_pcie_config_rp(struct tegra_pcie_dw *pcie)
1644	{
1645	struct device *dev = pcie->dev;
1646	int ret;
1647
1648	pm_runtime_enable(dev);
1649
1650	ret = pm_runtime_get_sync(dev);
1651	if (ret < 0) {
1652	dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n",
1653	ret);
1654	goto fail_pm_get_sync;
1655	}
1656
1657	ret = pinctrl_pm_select_default_state(dev);
1658	if (ret < 0) {
1659	dev_err(dev, "Failed to configure sideband pins: %d\n", ret);
1660	goto fail_pm_get_sync;
1661	}
1662
1663	ret = tegra_pcie_init_controller(pcie);
1664	if (ret < 0) {
1665	dev_err(dev, "Failed to initialize controller: %d\n", ret);
1666	goto fail_pm_get_sync;
1667	}
1668
1669	pcie->link_state = tegra_pcie_dw_link_up(pci: &pcie->pci);
1670	if (!pcie->link_state) {
1671	ret = -ENOMEDIUM;
1672	goto fail_host_init;
1673	}
1674
1675	init_debugfs(pcie);
1676
1677	return ret;
1678
1679	fail_host_init:
1680	tegra_pcie_deinit_controller(pcie);
1681	fail_pm_get_sync:
1682	pm_runtime_put_sync(dev);
1683	pm_runtime_disable(dev);
1684	return ret;
1685	}
1686
1687	static void pex_ep_event_pex_rst_assert(struct tegra_pcie_dw *pcie)
1688	{
1689	u32 val;
1690	int ret;
1691
1692	if (pcie->ep_state == EP_STATE_DISABLED)
1693	return;
1694
1695	/* Disable LTSSM */
1696	val = appl_readl(pcie, APPL_CTRL);
1697	val &= ~APPL_CTRL_LTSSM_EN;
1698	appl_writel(pcie, value: val, APPL_CTRL);
1699
1700	ret = readl_poll_timeout(pcie->appl_base + APPL_DEBUG, val,
1701	((val & APPL_DEBUG_LTSSM_STATE_MASK) >>
1702	APPL_DEBUG_LTSSM_STATE_SHIFT) ==
1703	LTSSM_STATE_PRE_DETECT,
1704	1, LTSSM_TIMEOUT);
1705	if (ret)
1706	dev_err(pcie->dev, "Failed to go Detect state: %d\n", ret);
1707
1708	reset_control_assert(rstc: pcie->core_rst);
1709
1710	tegra_pcie_disable_phy(pcie);
1711
1712	reset_control_assert(rstc: pcie->core_apb_rst);
1713
1714	clk_disable_unprepare(clk: pcie->core_clk);
1715
1716	pm_runtime_put_sync(dev: pcie->dev);
1717
1718	if (pcie->enable_ext_refclk) {
1719	ret = tegra_pcie_bpmp_set_pll_state(pcie, enable: false);
1720	if (ret)
1721	dev_err(pcie->dev, "Failed to turn off UPHY: %d\n",
1722	ret);
1723	}
1724
1725	ret = tegra_pcie_bpmp_set_pll_state(pcie, enable: false);
1726	if (ret)
1727	dev_err(pcie->dev, "Failed to turn off UPHY: %d\n", ret);
1728
1729	pcie->ep_state = EP_STATE_DISABLED;
1730	dev_dbg(pcie->dev, "Uninitialization of endpoint is completed\n");
1731	}
1732
1733	static void pex_ep_event_pex_rst_deassert(struct tegra_pcie_dw *pcie)
1734	{
1735	struct dw_pcie *pci = &pcie->pci;
1736	struct dw_pcie_ep *ep = &pci->ep;
1737	struct device *dev = pcie->dev;
1738	u32 val;
1739	int ret;
1740	u16 val_16;
1741
1742	if (pcie->ep_state == EP_STATE_ENABLED)
1743	return;
1744
1745	ret = pm_runtime_resume_and_get(dev);
1746	if (ret < 0) {
1747	dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n",
1748	ret);
1749	return;
1750	}
1751
1752	ret = tegra_pcie_bpmp_set_ctrl_state(pcie, enable: true);
1753	if (ret) {
1754	dev_err(pcie->dev, "Failed to enable controller %u: %d\n",
1755	pcie->cid, ret);
1756	goto fail_set_ctrl_state;
1757	}
1758
1759	if (pcie->enable_ext_refclk) {
1760	ret = tegra_pcie_bpmp_set_pll_state(pcie, enable: true);
1761	if (ret) {
1762	dev_err(dev, "Failed to init UPHY for PCIe EP: %d\n",
1763	ret);
1764	goto fail_pll_init;
1765	}
1766	}
1767
1768	ret = clk_prepare_enable(clk: pcie->core_clk);
1769	if (ret) {
1770	dev_err(dev, "Failed to enable core clock: %d\n", ret);
1771	goto fail_core_clk_enable;
1772	}
1773
1774	ret = reset_control_deassert(rstc: pcie->core_apb_rst);
1775	if (ret) {
1776	dev_err(dev, "Failed to deassert core APB reset: %d\n", ret);
1777	goto fail_core_apb_rst;
1778	}
1779
1780	ret = tegra_pcie_enable_phy(pcie);
1781	if (ret) {
1782	dev_err(dev, "Failed to enable PHY: %d\n", ret);
1783	goto fail_phy;
1784	}
1785
1786	/* Perform cleanup that requires refclk */
1787	pci_epc_deinit_notify(epc: pcie->pci.ep.epc);
1788	dw_pcie_ep_cleanup(ep: &pcie->pci.ep);
1789
1790	/* Clear any stale interrupt statuses */
1791	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L0);
1792	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
1793	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
1794	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
1795	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
1796	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
1797	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
1798	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
1799	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
1800	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
1801	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
1802	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
1803	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
1804	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
1805	appl_writel(pcie, value: 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
1806
1807	/* configure this core for EP mode operation */
1808	val = appl_readl(pcie, APPL_DM_TYPE);
1809	val &= ~APPL_DM_TYPE_MASK;
1810	val |= APPL_DM_TYPE_EP;
1811	appl_writel(pcie, value: val, APPL_DM_TYPE);
1812
1813	appl_writel(pcie, value: 0x0, APPL_CFG_SLCG_OVERRIDE);
1814
1815	val = appl_readl(pcie, APPL_CTRL);
1816	val |= APPL_CTRL_SYS_PRE_DET_STATE;
1817	val |= APPL_CTRL_HW_HOT_RST_EN;
1818	appl_writel(pcie, value: val, APPL_CTRL);
1819
1820	val = appl_readl(pcie, APPL_CFG_MISC);
1821	val |= APPL_CFG_MISC_SLV_EP_MODE;
1822	val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT);
1823	appl_writel(pcie, value: val, APPL_CFG_MISC);
1824
1825	val = appl_readl(pcie, APPL_PINMUX);
1826	val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;
1827	val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;
1828	appl_writel(pcie, value: val, APPL_PINMUX);
1829
1830	appl_writel(pcie, value: pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK,
1831	APPL_CFG_BASE_ADDR);
1832
1833	appl_writel(pcie, value: pcie->atu_dma_res->start &
1834	APPL_CFG_IATU_DMA_BASE_ADDR_MASK,
1835	APPL_CFG_IATU_DMA_BASE_ADDR);
1836
1837	val = appl_readl(pcie, APPL_INTR_EN_L0_0);
1838	val |= APPL_INTR_EN_L0_0_SYS_INTR_EN;
1839	val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN;
1840	val |= APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN;
1841	appl_writel(pcie, value: val, APPL_INTR_EN_L0_0);
1842
1843	val = appl_readl(pcie, APPL_INTR_EN_L1_0_0);
1844	val |= APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN;
1845	val |= APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN;
1846	appl_writel(pcie, value: val, APPL_INTR_EN_L1_0_0);
1847
1848	reset_control_deassert(rstc: pcie->core_rst);
1849
1850	if (pcie->update_fc_fixup) {
1851	val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF);
1852	val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT;
1853	dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val);
1854	}
1855
1856	config_gen3_gen4_eq_presets(pcie);
1857
1858	init_host_aspm(pcie);
1859
1860	/* Disable ASPM-L1SS advertisement if there is no CLKREQ routing */
1861	if (!pcie->supports_clkreq) {
1862	disable_aspm_l11(pcie);
1863	disable_aspm_l12(pcie);
1864	}
1865
1866	if (!pcie->of_data->has_l1ss_exit_fix) {
1867	val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
1868	val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL;
1869	dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
1870	}
1871
1872	pcie->pcie_cap_base = dw_pcie_find_capability(pci: &pcie->pci,
1873	PCI_CAP_ID_EXP);
1874
1875	/* Clear Slot Clock Configuration bit if SRNS configuration */
1876	if (pcie->enable_srns) {
1877	val_16 = dw_pcie_readw_dbi(pci, reg: pcie->pcie_cap_base +
1878	PCI_EXP_LNKSTA);
1879	val_16 &= ~PCI_EXP_LNKSTA_SLC;
1880	dw_pcie_writew_dbi(pci, reg: pcie->pcie_cap_base + PCI_EXP_LNKSTA,
1881	val: val_16);
1882	}
1883
1884	clk_set_rate(clk: pcie->core_clk, GEN4_CORE_CLK_FREQ);
1885
1886	val = (ep->msi_mem_phys & MSIX_ADDR_MATCH_LOW_OFF_MASK);
1887	val |= MSIX_ADDR_MATCH_LOW_OFF_EN;
1888	dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_LOW_OFF, val);
1889	val = (upper_32_bits(ep->msi_mem_phys) & MSIX_ADDR_MATCH_HIGH_OFF_MASK);
1890	dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_HIGH_OFF, val);
1891
1892	ret = dw_pcie_ep_init_registers(ep);
1893	if (ret) {
1894	dev_err(dev, "Failed to complete initialization: %d\n", ret);
1895	goto fail_init_complete;
1896	}
1897
1898	pci_epc_init_notify(epc: ep->epc);
1899
1900	/* Program the private control to allow sending LTR upstream */
1901	if (pcie->of_data->has_ltr_req_fix) {
1902	val = appl_readl(pcie, APPL_LTR_MSG_2);
1903	val |= APPL_LTR_MSG_2_LTR_MSG_REQ_STATE;
1904	appl_writel(pcie, value: val, APPL_LTR_MSG_2);
1905	}
1906
1907	/* Enable LTSSM */
1908	val = appl_readl(pcie, APPL_CTRL);
1909	val |= APPL_CTRL_LTSSM_EN;
1910	appl_writel(pcie, value: val, APPL_CTRL);
1911
1912	pcie->ep_state = EP_STATE_ENABLED;
1913	dev_dbg(dev, "Initialization of endpoint is completed\n");
1914
1915	return;
1916
1917	fail_init_complete:
1918	reset_control_assert(rstc: pcie->core_rst);
1919	tegra_pcie_disable_phy(pcie);
1920	fail_phy:
1921	reset_control_assert(rstc: pcie->core_apb_rst);
1922	fail_core_apb_rst:
1923	clk_disable_unprepare(clk: pcie->core_clk);
1924	fail_core_clk_enable:
1925	tegra_pcie_bpmp_set_pll_state(pcie, enable: false);
1926	fail_pll_init:
1927	tegra_pcie_bpmp_set_ctrl_state(pcie, enable: false);
1928	fail_set_ctrl_state:
1929	pm_runtime_put_sync(dev);
1930	}
1931
1932	static irqreturn_t tegra_pcie_ep_pex_rst_irq(int irq, void *arg)
1933	{
1934	struct tegra_pcie_dw *pcie = arg;
1935
1936	if (gpiod_get_value(desc: pcie->pex_rst_gpiod))
1937	pex_ep_event_pex_rst_assert(pcie);
1938	else
1939	pex_ep_event_pex_rst_deassert(pcie);
1940
1941	return IRQ_HANDLED;
1942	}
1943
1944	static int tegra_pcie_ep_raise_intx_irq(struct tegra_pcie_dw *pcie, u16 irq)
1945	{
1946	/* Tegra194 supports only INTA */
1947	if (irq > 1)
1948	return -EINVAL;
1949
1950	appl_writel(pcie, value: 1, APPL_LEGACY_INTX);
1951	usleep_range(min: 1000, max: 2000);
1952	appl_writel(pcie, value: 0, APPL_LEGACY_INTX);
1953	return 0;
1954	}
1955
1956	static int tegra_pcie_ep_raise_msi_irq(struct tegra_pcie_dw *pcie, u16 irq)
1957	{
1958	if (unlikely(irq > 31))
1959	return -EINVAL;
1960
1961	appl_writel(pcie, BIT(irq), APPL_MSI_CTRL_1);
1962
1963	return 0;
1964	}
1965
1966	static int tegra_pcie_ep_raise_msix_irq(struct tegra_pcie_dw *pcie, u16 irq)
1967	{
1968	struct dw_pcie_ep *ep = &pcie->pci.ep;
1969
1970	writel(val: irq, addr: ep->msi_mem);
1971
1972	return 0;
1973	}
1974
1975	static int tegra_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
1976	unsigned int type, u16 interrupt_num)
1977	{
1978	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
1979	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1980
1981	switch (type) {
1982	case PCI_IRQ_INTX:
1983	return tegra_pcie_ep_raise_intx_irq(pcie, irq: interrupt_num);
1984
1985	case PCI_IRQ_MSI:
1986	return tegra_pcie_ep_raise_msi_irq(pcie, irq: interrupt_num);
1987
1988	case PCI_IRQ_MSIX:
1989	return tegra_pcie_ep_raise_msix_irq(pcie, irq: interrupt_num);
1990
1991	default:
1992	dev_err(pci->dev, "Unknown IRQ type\n");
1993	return -EPERM;
1994	}
1995
1996	return 0;
1997	}
1998
1999	static const struct pci_epc_features tegra_pcie_epc_features = {
2000	.linkup_notifier = true,
2001	.msi_capable = false,
2002	.msix_capable = false,
2003	.bar[BAR_0] = { .type = BAR_FIXED, .fixed_size = SZ_1M,
2004	.only_64bit = true, },
2005	.bar[BAR_1] = { .type = BAR_RESERVED, },
2006	.bar[BAR_2] = { .type = BAR_RESERVED, },
2007	.bar[BAR_3] = { .type = BAR_RESERVED, },
2008	.bar[BAR_4] = { .type = BAR_RESERVED, },
2009	.bar[BAR_5] = { .type = BAR_RESERVED, },
2010	.align = SZ_64K,
2011	};
2012
2013	static const struct pci_epc_features*
2014	tegra_pcie_ep_get_features(struct dw_pcie_ep *ep)
2015	{
2016	return &tegra_pcie_epc_features;
2017	}
2018
2019	static const struct dw_pcie_ep_ops pcie_ep_ops = {
2020	.raise_irq = tegra_pcie_ep_raise_irq,
2021	.get_features = tegra_pcie_ep_get_features,
2022	};
2023
2024	static int tegra_pcie_config_ep(struct tegra_pcie_dw *pcie,
2025	struct platform_device *pdev)
2026	{
2027	struct dw_pcie *pci = &pcie->pci;
2028	struct device *dev = pcie->dev;
2029	struct dw_pcie_ep *ep;
2030	char *name;
2031	int ret;
2032
2033	ep = &pci->ep;
2034	ep->ops = &pcie_ep_ops;
2035
2036	ep->page_size = SZ_64K;
2037
2038	ret = gpiod_set_debounce(desc: pcie->pex_rst_gpiod, PERST_DEBOUNCE_TIME);
2039	if (ret < 0) {
2040	dev_err(dev, "Failed to set PERST GPIO debounce time: %d\n",
2041	ret);
2042	return ret;
2043	}
2044
2045	ret = gpiod_to_irq(desc: pcie->pex_rst_gpiod);
2046	if (ret < 0) {
2047	dev_err(dev, "Failed to get IRQ for PERST GPIO: %d\n", ret);
2048	return ret;
2049	}
2050	pcie->pex_rst_irq = (unsigned int)ret;
2051
2052	name = devm_kasprintf(dev, GFP_KERNEL, fmt: "tegra_pcie_%u_pex_rst_irq",
2053	pcie->cid);
2054	if (!name) {
2055	dev_err(dev, "Failed to create PERST IRQ string\n");
2056	return -ENOMEM;
2057	}
2058
2059	irq_set_status_flags(irq: pcie->pex_rst_irq, set: IRQ_NOAUTOEN);
2060
2061	pcie->ep_state = EP_STATE_DISABLED;
2062
2063	ret = devm_request_threaded_irq(dev, irq: pcie->pex_rst_irq, NULL,
2064	thread_fn: tegra_pcie_ep_pex_rst_irq,
2065	IRQF_TRIGGER_RISING |
2066	IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
2067	devname: name, dev_id: (void *)pcie);
2068	if (ret < 0) {
2069	dev_err(dev, "Failed to request IRQ for PERST: %d\n", ret);
2070	return ret;
2071	}
2072
2073	pm_runtime_enable(dev);
2074
2075	ret = dw_pcie_ep_init(ep);
2076	if (ret) {
2077	dev_err(dev, "Failed to initialize DWC Endpoint subsystem: %d\n",
2078	ret);
2079	pm_runtime_disable(dev);
2080	return ret;
2081	}
2082
2083	return 0;
2084	}
2085
2086	static int tegra_pcie_dw_probe(struct platform_device *pdev)
2087	{
2088	const struct tegra_pcie_dw_of_data *data;
2089	struct device *dev = &pdev->dev;
2090	struct resource *atu_dma_res;
2091	struct tegra_pcie_dw *pcie;
2092	struct dw_pcie_rp *pp;
2093	struct dw_pcie *pci;
2094	struct phy **phys;
2095	char *name;
2096	int ret;
2097	u32 i;
2098
2099	data = of_device_get_match_data(dev);
2100
2101	pcie = devm_kzalloc(dev, size: sizeof(*pcie), GFP_KERNEL);
2102	if (!pcie)
2103	return -ENOMEM;
2104
2105	pci = &pcie->pci;
2106	pci->dev = &pdev->dev;
2107	pci->ops = &tegra_dw_pcie_ops;
2108	pcie->dev = &pdev->dev;
2109	pcie->of_data = (struct tegra_pcie_dw_of_data *)data;
2110	pci->n_fts[0] = pcie->of_data->n_fts[0];
2111	pci->n_fts[1] = pcie->of_data->n_fts[1];
2112	pp = &pci->pp;
2113	pp->num_vectors = MAX_MSI_IRQS;
2114
2115	ret = tegra_pcie_dw_parse_dt(pcie);
2116	if (ret < 0) {
2117	const char *level = KERN_ERR;
2118
2119	if (ret == -EPROBE_DEFER)
2120	level = KERN_DEBUG;
2121
2122	dev_printk(level, dev,
2123	dev_fmt("Failed to parse device tree: %d\n"),
2124	ret);
2125	return ret;
2126	}
2127
2128	ret = tegra_pcie_get_slot_regulators(pcie);
2129	if (ret < 0) {
2130	const char *level = KERN_ERR;
2131
2132	if (ret == -EPROBE_DEFER)
2133	level = KERN_DEBUG;
2134
2135	dev_printk(level, dev,
2136	dev_fmt("Failed to get slot regulators: %d\n"),
2137	ret);
2138	return ret;
2139	}
2140
2141	if (pcie->pex_refclk_sel_gpiod)
2142	gpiod_set_value(desc: pcie->pex_refclk_sel_gpiod, value: 1);
2143
2144	pcie->pex_ctl_supply = devm_regulator_get(dev, id: "vddio-pex-ctl");
2145	if (IS_ERR(ptr: pcie->pex_ctl_supply)) {
2146	ret = PTR_ERR(ptr: pcie->pex_ctl_supply);
2147	if (ret != -EPROBE_DEFER)
2148	dev_err(dev, "Failed to get regulator: %ld\n",
2149	PTR_ERR(pcie->pex_ctl_supply));
2150	return ret;
2151	}
2152
2153	pcie->core_clk = devm_clk_get(dev, id: "core");
2154	if (IS_ERR(ptr: pcie->core_clk)) {
2155	dev_err(dev, "Failed to get core clock: %ld\n",
2156	PTR_ERR(pcie->core_clk));
2157	return PTR_ERR(ptr: pcie->core_clk);
2158	}
2159
2160	pcie->appl_res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
2161	"appl");
2162	if (!pcie->appl_res) {
2163	dev_err(dev, "Failed to find \"appl\" region\n");
2164	return -ENODEV;
2165	}
2166
2167	pcie->appl_base = devm_ioremap_resource(dev, res: pcie->appl_res);
2168	if (IS_ERR(ptr: pcie->appl_base))
2169	return PTR_ERR(ptr: pcie->appl_base);
2170
2171	pcie->core_apb_rst = devm_reset_control_get(dev, id: "apb");
2172	if (IS_ERR(ptr: pcie->core_apb_rst)) {
2173	dev_err(dev, "Failed to get APB reset: %ld\n",
2174	PTR_ERR(pcie->core_apb_rst));
2175	return PTR_ERR(ptr: pcie->core_apb_rst);
2176	}
2177
2178	phys = devm_kcalloc(dev, n: pcie->phy_count, size: sizeof(*phys), GFP_KERNEL);
2179	if (!phys)
2180	return -ENOMEM;
2181
2182	for (i = 0; i < pcie->phy_count; i++) {
2183	name = kasprintf(GFP_KERNEL, fmt: "p2u-%u", i);
2184	if (!name) {
2185	dev_err(dev, "Failed to create P2U string\n");
2186	return -ENOMEM;
2187	}
2188	phys[i] = devm_phy_get(dev, string: name);
2189	kfree(objp: name);
2190	if (IS_ERR(ptr: phys[i])) {
2191	ret = PTR_ERR(ptr: phys[i]);
2192	if (ret != -EPROBE_DEFER)
2193	dev_err(dev, "Failed to get PHY: %d\n", ret);
2194	return ret;
2195	}
2196	}
2197
2198	pcie->phys = phys;
2199
2200	atu_dma_res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
2201	"atu_dma");
2202	if (!atu_dma_res) {
2203	dev_err(dev, "Failed to find \"atu_dma\" region\n");
2204	return -ENODEV;
2205	}
2206	pcie->atu_dma_res = atu_dma_res;
2207
2208	pci->atu_size = resource_size(res: atu_dma_res);
2209	pci->atu_base = devm_ioremap_resource(dev, res: atu_dma_res);
2210	if (IS_ERR(ptr: pci->atu_base))
2211	return PTR_ERR(ptr: pci->atu_base);
2212
2213	pcie->core_rst = devm_reset_control_get(dev, id: "core");
2214	if (IS_ERR(ptr: pcie->core_rst)) {
2215	dev_err(dev, "Failed to get core reset: %ld\n",
2216	PTR_ERR(pcie->core_rst));
2217	return PTR_ERR(ptr: pcie->core_rst);
2218	}
2219
2220	pp->irq = platform_get_irq_byname(pdev, "intr");
2221	if (pp->irq < 0)
2222	return pp->irq;
2223
2224	pcie->bpmp = tegra_bpmp_get(dev);
2225	if (IS_ERR(ptr: pcie->bpmp))
2226	return PTR_ERR(ptr: pcie->bpmp);
2227
2228	platform_set_drvdata(pdev, data: pcie);
2229
2230	pcie->icc_path = devm_of_icc_get(dev: &pdev->dev, name: "write");
2231	ret = PTR_ERR_OR_ZERO(ptr: pcie->icc_path);
2232	if (ret) {
2233	tegra_bpmp_put(bpmp: pcie->bpmp);
2234	dev_err_probe(dev: &pdev->dev, err: ret, fmt: "failed to get write interconnect\n");
2235	return ret;
2236	}
2237
2238	switch (pcie->of_data->mode) {
2239	case DW_PCIE_RC_TYPE:
2240	ret = devm_request_irq(dev, irq: pp->irq, handler: tegra_pcie_rp_irq_handler,
2241	IRQF_SHARED, devname: "tegra-pcie-intr", dev_id: pcie);
2242	if (ret) {
2243	dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq,
2244	ret);
2245	goto fail;
2246	}
2247
2248	ret = tegra_pcie_config_rp(pcie);
2249	if (ret && ret != -ENOMEDIUM)
2250	goto fail;
2251	else
2252	return 0;
2253	break;
2254
2255	case DW_PCIE_EP_TYPE:
2256	ret = devm_request_threaded_irq(dev, irq: pp->irq,
2257	handler: tegra_pcie_ep_hard_irq,
2258	thread_fn: tegra_pcie_ep_irq_thread,
2259	IRQF_SHARED | IRQF_ONESHOT,
2260	devname: "tegra-pcie-ep-intr", dev_id: pcie);
2261	if (ret) {
2262	dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq,
2263	ret);
2264	goto fail;
2265	}
2266
2267	ret = tegra_pcie_config_ep(pcie, pdev);
2268	if (ret < 0)
2269	goto fail;
2270	else
2271	return 0;
2272	break;
2273
2274	default:
2275	dev_err(dev, "Invalid PCIe device type %d\n",
2276	pcie->of_data->mode);
2277	ret = -EINVAL;
2278	}
2279
2280	fail:
2281	tegra_bpmp_put(bpmp: pcie->bpmp);
2282	return ret;
2283	}
2284
2285	static void tegra_pcie_dw_remove(struct platform_device *pdev)
2286	{
2287	struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev);
2288
2289	if (pcie->of_data->mode == DW_PCIE_RC_TYPE) {
2290	if (!pcie->link_state)
2291	return;
2292
2293	debugfs_remove_recursive(dentry: pcie->debugfs);
2294	tegra_pcie_deinit_controller(pcie);
2295	pm_runtime_put_sync(dev: pcie->dev);
2296	} else {
2297	disable_irq(irq: pcie->pex_rst_irq);
2298	pex_ep_event_pex_rst_assert(pcie);
2299	}
2300
2301	pm_runtime_disable(dev: pcie->dev);
2302	tegra_bpmp_put(bpmp: pcie->bpmp);
2303	if (pcie->pex_refclk_sel_gpiod)
2304	gpiod_set_value(desc: pcie->pex_refclk_sel_gpiod, value: 0);
2305	}
2306
2307	static int tegra_pcie_dw_suspend_late(struct device *dev)
2308	{
2309	struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2310	u32 val;
2311
2312	if (pcie->of_data->mode == DW_PCIE_EP_TYPE) {
2313	dev_err(dev, "Failed to Suspend as Tegra PCIe is in EP mode\n");
2314	return -EPERM;
2315	}
2316
2317	if (!pcie->link_state)
2318	return 0;
2319
2320	/* Enable HW_HOT_RST mode */
2321	if (!pcie->of_data->has_sbr_reset_fix) {
2322	val = appl_readl(pcie, APPL_CTRL);
2323	val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
2324	APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
2325	val |= APPL_CTRL_HW_HOT_RST_EN;
2326	appl_writel(pcie, value: val, APPL_CTRL);
2327	}
2328
2329	return 0;
2330	}
2331
2332	static int tegra_pcie_dw_suspend_noirq(struct device *dev)
2333	{
2334	struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2335
2336	if (!pcie->link_state)
2337	return 0;
2338
2339	tegra_pcie_downstream_dev_to_D0(pcie);
2340	tegra_pcie_dw_pme_turnoff(pcie);
2341	tegra_pcie_unconfig_controller(pcie);
2342
2343	return 0;
2344	}
2345
2346	static int tegra_pcie_dw_resume_noirq(struct device *dev)
2347	{
2348	struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2349	int ret;
2350
2351	if (!pcie->link_state)
2352	return 0;
2353
2354	ret = tegra_pcie_config_controller(pcie, en_hw_hot_rst: true);
2355	if (ret < 0)
2356	return ret;
2357
2358	ret = tegra_pcie_dw_host_init(pp: &pcie->pci.pp);
2359	if (ret < 0) {
2360	dev_err(dev, "Failed to init host: %d\n", ret);
2361	goto fail_host_init;
2362	}
2363
2364	dw_pcie_setup_rc(pp: &pcie->pci.pp);
2365
2366	ret = tegra_pcie_dw_start_link(pci: &pcie->pci);
2367	if (ret < 0)
2368	goto fail_host_init;
2369
2370	return 0;
2371
2372	fail_host_init:
2373	tegra_pcie_unconfig_controller(pcie);
2374	return ret;
2375	}
2376
2377	static int tegra_pcie_dw_resume_early(struct device *dev)
2378	{
2379	struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2380	u32 val;
2381
2382	if (pcie->of_data->mode == DW_PCIE_EP_TYPE) {
2383	dev_err(dev, "Suspend is not supported in EP mode");
2384	return -ENOTSUPP;
2385	}
2386
2387	if (!pcie->link_state)
2388	return 0;
2389
2390	/* Disable HW_HOT_RST mode */
2391	if (!pcie->of_data->has_sbr_reset_fix) {
2392	val = appl_readl(pcie, APPL_CTRL);
2393	val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
2394	APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
2395	val |= APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST <<
2396	APPL_CTRL_HW_HOT_RST_MODE_SHIFT;
2397	val &= ~APPL_CTRL_HW_HOT_RST_EN;
2398	appl_writel(pcie, value: val, APPL_CTRL);
2399	}
2400
2401	return 0;
2402	}
2403
2404	static void tegra_pcie_dw_shutdown(struct platform_device *pdev)
2405	{
2406	struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev);
2407
2408	if (pcie->of_data->mode == DW_PCIE_RC_TYPE) {
2409	if (!pcie->link_state)
2410	return;
2411
2412	debugfs_remove_recursive(dentry: pcie->debugfs);
2413	tegra_pcie_downstream_dev_to_D0(pcie);
2414
2415	disable_irq(irq: pcie->pci.pp.irq);
2416	if (IS_ENABLED(CONFIG_PCI_MSI))
2417	disable_irq(irq: pcie->pci.pp.msi_irq[0]);
2418
2419	tegra_pcie_dw_pme_turnoff(pcie);
2420	tegra_pcie_unconfig_controller(pcie);
2421	pm_runtime_put_sync(dev: pcie->dev);
2422	} else {
2423	disable_irq(irq: pcie->pex_rst_irq);
2424	pex_ep_event_pex_rst_assert(pcie);
2425	}
2426	}
2427
2428	static const struct tegra_pcie_dw_of_data tegra194_pcie_dw_rc_of_data = {
2429	.version = TEGRA194_DWC_IP_VER,
2430	.mode = DW_PCIE_RC_TYPE,
2431	.cdm_chk_int_en_bit = BIT(19),
2432	/* Gen4 - 5, 6, 8 and 9 presets enabled */
2433	.gen4_preset_vec = 0x360,
2434	.n_fts = { 52, 52 },
2435	};
2436
2437	static const struct tegra_pcie_dw_of_data tegra194_pcie_dw_ep_of_data = {
2438	.version = TEGRA194_DWC_IP_VER,
2439	.mode = DW_PCIE_EP_TYPE,
2440	.cdm_chk_int_en_bit = BIT(19),
2441	/* Gen4 - 5, 6, 8 and 9 presets enabled */
2442	.gen4_preset_vec = 0x360,
2443	.n_fts = { 52, 52 },
2444	};
2445
2446	static const struct tegra_pcie_dw_of_data tegra234_pcie_dw_rc_of_data = {
2447	.version = TEGRA234_DWC_IP_VER,
2448	.mode = DW_PCIE_RC_TYPE,
2449	.has_msix_doorbell_access_fix = true,
2450	.has_sbr_reset_fix = true,
2451	.has_l1ss_exit_fix = true,
2452	.cdm_chk_int_en_bit = BIT(18),
2453	/* Gen4 - 6, 8 and 9 presets enabled */
2454	.gen4_preset_vec = 0x340,
2455	.n_fts = { 52, 80 },
2456	};
2457
2458	static const struct tegra_pcie_dw_of_data tegra234_pcie_dw_ep_of_data = {
2459	.version = TEGRA234_DWC_IP_VER,
2460	.mode = DW_PCIE_EP_TYPE,
2461	.has_l1ss_exit_fix = true,
2462	.has_ltr_req_fix = true,
2463	.cdm_chk_int_en_bit = BIT(18),
2464	/* Gen4 - 6, 8 and 9 presets enabled */
2465	.gen4_preset_vec = 0x340,
2466	.n_fts = { 52, 80 },
2467	};
2468
2469	static const struct of_device_id tegra_pcie_dw_of_match[] = {
2470	{
2471	.compatible = "nvidia,tegra194-pcie",
2472	.data = &tegra194_pcie_dw_rc_of_data,
2473	},
2474	{
2475	.compatible = "nvidia,tegra194-pcie-ep",
2476	.data = &tegra194_pcie_dw_ep_of_data,
2477	},
2478	{
2479	.compatible = "nvidia,tegra234-pcie",
2480	.data = &tegra234_pcie_dw_rc_of_data,
2481	},
2482	{
2483	.compatible = "nvidia,tegra234-pcie-ep",
2484	.data = &tegra234_pcie_dw_ep_of_data,
2485	},
2486	{}
2487	};
2488
2489	static const struct dev_pm_ops tegra_pcie_dw_pm_ops = {
2490	.suspend_late = tegra_pcie_dw_suspend_late,
2491	.suspend_noirq = tegra_pcie_dw_suspend_noirq,
2492	.resume_noirq = tegra_pcie_dw_resume_noirq,
2493	.resume_early = tegra_pcie_dw_resume_early,
2494	};
2495
2496	static struct platform_driver tegra_pcie_dw_driver = {
2497	.probe = tegra_pcie_dw_probe,
2498	.remove = tegra_pcie_dw_remove,
2499	.shutdown = tegra_pcie_dw_shutdown,
2500	.driver = {
2501	.name = "tegra194-pcie",
2502	.pm = &tegra_pcie_dw_pm_ops,
2503	.of_match_table = tegra_pcie_dw_of_match,
2504	},
2505	};
2506	module_platform_driver(tegra_pcie_dw_driver);
2507
2508	MODULE_DEVICE_TABLE(of, tegra_pcie_dw_of_match);
2509
2510	MODULE_AUTHOR("Vidya Sagar <vidyas@nvidia.com>");
2511	MODULE_DESCRIPTION("NVIDIA PCIe host controller driver");
2512	MODULE_LICENSE("GPL v2");
2513