1 | // SPDX-License-Identifier: GPL-2.0+ |
2 | /* Copyright (C) 2009 - 2019 Broadcom */ |
3 | |
4 | #include <linux/bitfield.h> |
5 | #include <linux/bitops.h> |
6 | #include <linux/clk.h> |
7 | #include <linux/compiler.h> |
8 | #include <linux/delay.h> |
9 | #include <linux/init.h> |
10 | #include <linux/interrupt.h> |
11 | #include <linux/io.h> |
12 | #include <linux/iopoll.h> |
13 | #include <linux/ioport.h> |
14 | #include <linux/irqchip/chained_irq.h> |
15 | #include <linux/irqdomain.h> |
16 | #include <linux/kernel.h> |
17 | #include <linux/list.h> |
18 | #include <linux/log2.h> |
19 | #include <linux/module.h> |
20 | #include <linux/msi.h> |
21 | #include <linux/of_address.h> |
22 | #include <linux/of_irq.h> |
23 | #include <linux/of_pci.h> |
24 | #include <linux/of_platform.h> |
25 | #include <linux/pci.h> |
26 | #include <linux/pci-ecam.h> |
27 | #include <linux/printk.h> |
28 | #include <linux/regulator/consumer.h> |
29 | #include <linux/reset.h> |
30 | #include <linux/sizes.h> |
31 | #include <linux/slab.h> |
32 | #include <linux/string.h> |
33 | #include <linux/types.h> |
34 | |
35 | #include "../pci.h" |
36 | |
37 | /* BRCM_PCIE_CAP_REGS - Offset for the mandatory capability config regs */ |
38 | #define BRCM_PCIE_CAP_REGS 0x00ac |
39 | |
40 | /* Broadcom STB PCIe Register Offsets */ |
41 | #define PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1 0x0188 |
42 | #define PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1_ENDIAN_MODE_BAR2_MASK 0xc |
43 | #define PCIE_RC_CFG_VENDOR_SPCIFIC_REG1_LITTLE_ENDIAN 0x0 |
44 | |
45 | #define PCIE_RC_CFG_PRIV1_ID_VAL3 0x043c |
46 | #define PCIE_RC_CFG_PRIV1_ID_VAL3_CLASS_CODE_MASK 0xffffff |
47 | |
48 | #define PCIE_RC_CFG_PRIV1_LINK_CAPABILITY 0x04dc |
49 | #define PCIE_RC_CFG_PRIV1_LINK_CAPABILITY_ASPM_SUPPORT_MASK 0xc00 |
50 | |
51 | #define PCIE_RC_CFG_PRIV1_ROOT_CAP 0x4f8 |
52 | #define PCIE_RC_CFG_PRIV1_ROOT_CAP_L1SS_MODE_MASK 0xf8 |
53 | |
54 | #define PCIE_RC_DL_MDIO_ADDR 0x1100 |
55 | #define PCIE_RC_DL_MDIO_WR_DATA 0x1104 |
56 | #define PCIE_RC_DL_MDIO_RD_DATA 0x1108 |
57 | |
58 | #define PCIE_MISC_MISC_CTRL 0x4008 |
59 | #define PCIE_MISC_MISC_CTRL_PCIE_RCB_64B_MODE_MASK 0x80 |
60 | #define PCIE_MISC_MISC_CTRL_PCIE_RCB_MPS_MODE_MASK 0x400 |
61 | #define PCIE_MISC_MISC_CTRL_SCB_ACCESS_EN_MASK 0x1000 |
62 | #define PCIE_MISC_MISC_CTRL_CFG_READ_UR_MODE_MASK 0x2000 |
63 | #define PCIE_MISC_MISC_CTRL_MAX_BURST_SIZE_MASK 0x300000 |
64 | |
65 | #define PCIE_MISC_MISC_CTRL_SCB0_SIZE_MASK 0xf8000000 |
66 | #define PCIE_MISC_MISC_CTRL_SCB1_SIZE_MASK 0x07c00000 |
67 | #define PCIE_MISC_MISC_CTRL_SCB2_SIZE_MASK 0x0000001f |
68 | #define SCB_SIZE_MASK(x) PCIE_MISC_MISC_CTRL_SCB ## x ## _SIZE_MASK |
69 | |
70 | #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LO 0x400c |
71 | #define PCIE_MEM_WIN0_LO(win) \ |
72 | PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LO + ((win) * 8) |
73 | |
74 | #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_HI 0x4010 |
75 | #define PCIE_MEM_WIN0_HI(win) \ |
76 | PCIE_MISC_CPU_2_PCIE_MEM_WIN0_HI + ((win) * 8) |
77 | |
78 | #define PCIE_MISC_RC_BAR1_CONFIG_LO 0x402c |
79 | #define PCIE_MISC_RC_BAR1_CONFIG_LO_SIZE_MASK 0x1f |
80 | |
81 | #define PCIE_MISC_RC_BAR2_CONFIG_LO 0x4034 |
82 | #define PCIE_MISC_RC_BAR2_CONFIG_LO_SIZE_MASK 0x1f |
83 | #define PCIE_MISC_RC_BAR2_CONFIG_HI 0x4038 |
84 | |
85 | #define PCIE_MISC_RC_BAR3_CONFIG_LO 0x403c |
86 | #define PCIE_MISC_RC_BAR3_CONFIG_LO_SIZE_MASK 0x1f |
87 | |
88 | #define PCIE_MISC_MSI_BAR_CONFIG_LO 0x4044 |
89 | #define PCIE_MISC_MSI_BAR_CONFIG_HI 0x4048 |
90 | |
91 | #define PCIE_MISC_MSI_DATA_CONFIG 0x404c |
92 | #define PCIE_MISC_MSI_DATA_CONFIG_VAL_32 0xffe06540 |
93 | #define PCIE_MISC_MSI_DATA_CONFIG_VAL_8 0xfff86540 |
94 | |
95 | #define PCIE_MISC_PCIE_CTRL 0x4064 |
96 | #define PCIE_MISC_PCIE_CTRL_PCIE_L23_REQUEST_MASK 0x1 |
97 | #define PCIE_MISC_PCIE_CTRL_PCIE_PERSTB_MASK 0x4 |
98 | |
99 | #define PCIE_MISC_PCIE_STATUS 0x4068 |
100 | #define PCIE_MISC_PCIE_STATUS_PCIE_PORT_MASK 0x80 |
101 | #define PCIE_MISC_PCIE_STATUS_PCIE_DL_ACTIVE_MASK 0x20 |
102 | #define PCIE_MISC_PCIE_STATUS_PCIE_PHYLINKUP_MASK 0x10 |
103 | #define PCIE_MISC_PCIE_STATUS_PCIE_LINK_IN_L23_MASK 0x40 |
104 | |
105 | #define PCIE_MISC_REVISION 0x406c |
106 | #define BRCM_PCIE_HW_REV_33 0x0303 |
107 | #define BRCM_PCIE_HW_REV_3_20 0x0320 |
108 | |
109 | #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT 0x4070 |
110 | #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_LIMIT_MASK 0xfff00000 |
111 | #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_BASE_MASK 0xfff0 |
112 | #define PCIE_MEM_WIN0_BASE_LIMIT(win) \ |
113 | PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT + ((win) * 4) |
114 | |
115 | #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI 0x4080 |
116 | #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI_BASE_MASK 0xff |
117 | #define PCIE_MEM_WIN0_BASE_HI(win) \ |
118 | PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI + ((win) * 8) |
119 | |
120 | #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI 0x4084 |
121 | #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI_LIMIT_MASK 0xff |
122 | #define PCIE_MEM_WIN0_LIMIT_HI(win) \ |
123 | PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI + ((win) * 8) |
124 | |
125 | #define PCIE_MISC_HARD_PCIE_HARD_DEBUG 0x4204 |
126 | #define PCIE_MISC_HARD_PCIE_HARD_DEBUG_CLKREQ_DEBUG_ENABLE_MASK 0x2 |
127 | #define PCIE_MISC_HARD_PCIE_HARD_DEBUG_L1SS_ENABLE_MASK 0x200000 |
128 | #define PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK 0x08000000 |
129 | #define PCIE_BMIPS_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK 0x00800000 |
130 | #define PCIE_CLKREQ_MASK \ |
131 | (PCIE_MISC_HARD_PCIE_HARD_DEBUG_CLKREQ_DEBUG_ENABLE_MASK | \ |
132 | PCIE_MISC_HARD_PCIE_HARD_DEBUG_L1SS_ENABLE_MASK) |
133 | |
134 | #define PCIE_INTR2_CPU_BASE 0x4300 |
135 | #define PCIE_MSI_INTR2_BASE 0x4500 |
136 | /* Offsets from PCIE_INTR2_CPU_BASE and PCIE_MSI_INTR2_BASE */ |
137 | #define MSI_INT_STATUS 0x0 |
138 | #define MSI_INT_CLR 0x8 |
139 | #define MSI_INT_MASK_SET 0x10 |
140 | #define MSI_INT_MASK_CLR 0x14 |
141 | |
142 | #define PCIE_EXT_CFG_DATA 0x8000 |
143 | #define PCIE_EXT_CFG_INDEX 0x9000 |
144 | |
145 | #define PCIE_RGR1_SW_INIT_1_PERST_MASK 0x1 |
146 | #define PCIE_RGR1_SW_INIT_1_PERST_SHIFT 0x0 |
147 | |
148 | #define RGR1_SW_INIT_1_INIT_GENERIC_MASK 0x2 |
149 | #define RGR1_SW_INIT_1_INIT_GENERIC_SHIFT 0x1 |
150 | #define RGR1_SW_INIT_1_INIT_7278_MASK 0x1 |
151 | #define RGR1_SW_INIT_1_INIT_7278_SHIFT 0x0 |
152 | |
153 | /* PCIe parameters */ |
154 | #define BRCM_NUM_PCIE_OUT_WINS 0x4 |
155 | #define BRCM_INT_PCI_MSI_NR 32 |
156 | #define BRCM_INT_PCI_MSI_LEGACY_NR 8 |
157 | #define BRCM_INT_PCI_MSI_SHIFT 0 |
158 | #define BRCM_INT_PCI_MSI_MASK GENMASK(BRCM_INT_PCI_MSI_NR - 1, 0) |
159 | #define BRCM_INT_PCI_MSI_LEGACY_MASK GENMASK(31, \ |
160 | 32 - BRCM_INT_PCI_MSI_LEGACY_NR) |
161 | |
162 | /* MSI target addresses */ |
163 | #define BRCM_MSI_TARGET_ADDR_LT_4GB 0x0fffffffcULL |
164 | #define BRCM_MSI_TARGET_ADDR_GT_4GB 0xffffffffcULL |
165 | |
166 | /* MDIO registers */ |
167 | #define MDIO_PORT0 0x0 |
168 | #define MDIO_DATA_MASK 0x7fffffff |
169 | #define MDIO_PORT_MASK 0xf0000 |
170 | #define MDIO_REGAD_MASK 0xffff |
171 | #define MDIO_CMD_MASK 0xfff00000 |
172 | #define MDIO_CMD_READ 0x1 |
173 | #define MDIO_CMD_WRITE 0x0 |
174 | #define MDIO_DATA_DONE_MASK 0x80000000 |
175 | #define MDIO_RD_DONE(x) (((x) & MDIO_DATA_DONE_MASK) ? 1 : 0) |
176 | #define MDIO_WT_DONE(x) (((x) & MDIO_DATA_DONE_MASK) ? 0 : 1) |
177 | #define SSC_REGS_ADDR 0x1100 |
178 | #define SET_ADDR_OFFSET 0x1f |
179 | #define SSC_CNTL_OFFSET 0x2 |
180 | #define SSC_CNTL_OVRD_EN_MASK 0x8000 |
181 | #define SSC_CNTL_OVRD_VAL_MASK 0x4000 |
182 | #define SSC_STATUS_OFFSET 0x1 |
183 | #define SSC_STATUS_SSC_MASK 0x400 |
184 | #define SSC_STATUS_PLL_LOCK_MASK 0x800 |
185 | #define PCIE_BRCM_MAX_MEMC 3 |
186 | |
187 | #define IDX_ADDR(pcie) (pcie->reg_offsets[EXT_CFG_INDEX]) |
188 | #define DATA_ADDR(pcie) (pcie->reg_offsets[EXT_CFG_DATA]) |
189 | #define PCIE_RGR1_SW_INIT_1(pcie) (pcie->reg_offsets[RGR1_SW_INIT_1]) |
190 | |
191 | /* Rescal registers */ |
192 | #define PCIE_DVT_PMU_PCIE_PHY_CTRL 0xc700 |
193 | #define PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_NFLDS 0x3 |
194 | #define PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_DIG_RESET_MASK 0x4 |
195 | #define PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_DIG_RESET_SHIFT 0x2 |
196 | #define PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_RESET_MASK 0x2 |
197 | #define PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_RESET_SHIFT 0x1 |
198 | #define PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_PWRDN_MASK 0x1 |
199 | #define PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_PWRDN_SHIFT 0x0 |
200 | |
201 | /* Forward declarations */ |
202 | struct brcm_pcie; |
203 | |
204 | enum { |
205 | RGR1_SW_INIT_1, |
206 | EXT_CFG_INDEX, |
207 | EXT_CFG_DATA, |
208 | }; |
209 | |
210 | enum { |
211 | RGR1_SW_INIT_1_INIT_MASK, |
212 | RGR1_SW_INIT_1_INIT_SHIFT, |
213 | }; |
214 | |
215 | enum pcie_type { |
216 | GENERIC, |
217 | BCM7425, |
218 | BCM7435, |
219 | BCM4908, |
220 | BCM7278, |
221 | BCM2711, |
222 | }; |
223 | |
224 | struct pcie_cfg_data { |
225 | const int *offsets; |
226 | const enum pcie_type type; |
227 | void (*perst_set)(struct brcm_pcie *pcie, u32 val); |
228 | void (*bridge_sw_init_set)(struct brcm_pcie *pcie, u32 val); |
229 | }; |
230 | |
231 | struct subdev_regulators { |
232 | unsigned int num_supplies; |
233 | struct regulator_bulk_data supplies[]; |
234 | }; |
235 | |
236 | struct brcm_msi { |
237 | struct device *dev; |
238 | void __iomem *base; |
239 | struct device_node *np; |
240 | struct irq_domain *msi_domain; |
241 | struct irq_domain *inner_domain; |
242 | struct mutex lock; /* guards the alloc/free operations */ |
243 | u64 target_addr; |
244 | int irq; |
245 | DECLARE_BITMAP(used, BRCM_INT_PCI_MSI_NR); |
246 | bool legacy; |
247 | /* Some chips have MSIs in bits [31..24] of a shared register. */ |
248 | int legacy_shift; |
249 | int nr; /* No. of MSI available, depends on chip */ |
250 | /* This is the base pointer for interrupt status/set/clr regs */ |
251 | void __iomem *intr_base; |
252 | }; |
253 | |
254 | /* Internal PCIe Host Controller Information.*/ |
255 | struct brcm_pcie { |
256 | struct device *dev; |
257 | void __iomem *base; |
258 | struct clk *clk; |
259 | struct device_node *np; |
260 | bool ssc; |
261 | int gen; |
262 | u64 msi_target_addr; |
263 | struct brcm_msi *msi; |
264 | const int *reg_offsets; |
265 | enum pcie_type type; |
266 | struct reset_control *rescal; |
267 | struct reset_control *perst_reset; |
268 | int num_memc; |
269 | u64 memc_size[PCIE_BRCM_MAX_MEMC]; |
270 | u32 hw_rev; |
271 | void (*perst_set)(struct brcm_pcie *pcie, u32 val); |
272 | void (*bridge_sw_init_set)(struct brcm_pcie *pcie, u32 val); |
273 | struct subdev_regulators *sr; |
274 | bool ep_wakeup_capable; |
275 | }; |
276 | |
277 | static inline bool is_bmips(const struct brcm_pcie *pcie) |
278 | { |
279 | return pcie->type == BCM7435 || pcie->type == BCM7425; |
280 | } |
281 | |
282 | /* |
283 | * This is to convert the size of the inbound "BAR" region to the |
284 | * non-linear values of PCIE_X_MISC_RC_BAR[123]_CONFIG_LO.SIZE |
285 | */ |
286 | static int brcm_pcie_encode_ibar_size(u64 size) |
287 | { |
288 | int log2_in = ilog2(size); |
289 | |
290 | if (log2_in >= 12 && log2_in <= 15) |
291 | /* Covers 4KB to 32KB (inclusive) */ |
292 | return (log2_in - 12) + 0x1c; |
293 | else if (log2_in >= 16 && log2_in <= 35) |
294 | /* Covers 64KB to 32GB, (inclusive) */ |
295 | return log2_in - 15; |
296 | /* Something is awry so disable */ |
297 | return 0; |
298 | } |
299 | |
300 | static u32 brcm_pcie_mdio_form_pkt(int port, int regad, int cmd) |
301 | { |
302 | u32 pkt = 0; |
303 | |
304 | pkt |= FIELD_PREP(MDIO_PORT_MASK, port); |
305 | pkt |= FIELD_PREP(MDIO_REGAD_MASK, regad); |
306 | pkt |= FIELD_PREP(MDIO_CMD_MASK, cmd); |
307 | |
308 | return pkt; |
309 | } |
310 | |
311 | /* negative return value indicates error */ |
312 | static int brcm_pcie_mdio_read(void __iomem *base, u8 port, u8 regad, u32 *val) |
313 | { |
314 | u32 data; |
315 | int err; |
316 | |
317 | writel(val: brcm_pcie_mdio_form_pkt(port, regad, MDIO_CMD_READ), |
318 | addr: base + PCIE_RC_DL_MDIO_ADDR); |
319 | readl(addr: base + PCIE_RC_DL_MDIO_ADDR); |
320 | err = readl_poll_timeout_atomic(base + PCIE_RC_DL_MDIO_RD_DATA, data, |
321 | MDIO_RD_DONE(data), 10, 100); |
322 | *val = FIELD_GET(MDIO_DATA_MASK, data); |
323 | |
324 | return err; |
325 | } |
326 | |
327 | /* negative return value indicates error */ |
328 | static int brcm_pcie_mdio_write(void __iomem *base, u8 port, |
329 | u8 regad, u16 wrdata) |
330 | { |
331 | u32 data; |
332 | int err; |
333 | |
334 | writel(val: brcm_pcie_mdio_form_pkt(port, regad, MDIO_CMD_WRITE), |
335 | addr: base + PCIE_RC_DL_MDIO_ADDR); |
336 | readl(addr: base + PCIE_RC_DL_MDIO_ADDR); |
337 | writel(MDIO_DATA_DONE_MASK | wrdata, addr: base + PCIE_RC_DL_MDIO_WR_DATA); |
338 | |
339 | err = readl_poll_timeout_atomic(base + PCIE_RC_DL_MDIO_WR_DATA, data, |
340 | MDIO_WT_DONE(data), 10, 100); |
341 | return err; |
342 | } |
343 | |
344 | /* |
345 | * Configures device for Spread Spectrum Clocking (SSC) mode; a negative |
346 | * return value indicates error. |
347 | */ |
348 | static int brcm_pcie_set_ssc(struct brcm_pcie *pcie) |
349 | { |
350 | int pll, ssc; |
351 | int ret; |
352 | u32 tmp; |
353 | |
354 | ret = brcm_pcie_mdio_write(base: pcie->base, MDIO_PORT0, SET_ADDR_OFFSET, |
355 | SSC_REGS_ADDR); |
356 | if (ret < 0) |
357 | return ret; |
358 | |
359 | ret = brcm_pcie_mdio_read(base: pcie->base, MDIO_PORT0, |
360 | SSC_CNTL_OFFSET, val: &tmp); |
361 | if (ret < 0) |
362 | return ret; |
363 | |
364 | u32p_replace_bits(p: &tmp, val: 1, SSC_CNTL_OVRD_EN_MASK); |
365 | u32p_replace_bits(p: &tmp, val: 1, SSC_CNTL_OVRD_VAL_MASK); |
366 | ret = brcm_pcie_mdio_write(base: pcie->base, MDIO_PORT0, |
367 | SSC_CNTL_OFFSET, wrdata: tmp); |
368 | if (ret < 0) |
369 | return ret; |
370 | |
371 | usleep_range(min: 1000, max: 2000); |
372 | ret = brcm_pcie_mdio_read(base: pcie->base, MDIO_PORT0, |
373 | SSC_STATUS_OFFSET, val: &tmp); |
374 | if (ret < 0) |
375 | return ret; |
376 | |
377 | ssc = FIELD_GET(SSC_STATUS_SSC_MASK, tmp); |
378 | pll = FIELD_GET(SSC_STATUS_PLL_LOCK_MASK, tmp); |
379 | |
380 | return ssc && pll ? 0 : -EIO; |
381 | } |
382 | |
383 | /* Limits operation to a specific generation (1, 2, or 3) */ |
384 | static void brcm_pcie_set_gen(struct brcm_pcie *pcie, int gen) |
385 | { |
386 | u16 lnkctl2 = readw(addr: pcie->base + BRCM_PCIE_CAP_REGS + PCI_EXP_LNKCTL2); |
387 | u32 lnkcap = readl(addr: pcie->base + BRCM_PCIE_CAP_REGS + PCI_EXP_LNKCAP); |
388 | |
389 | lnkcap = (lnkcap & ~PCI_EXP_LNKCAP_SLS) | gen; |
390 | writel(val: lnkcap, addr: pcie->base + BRCM_PCIE_CAP_REGS + PCI_EXP_LNKCAP); |
391 | |
392 | lnkctl2 = (lnkctl2 & ~0xf) | gen; |
393 | writew(val: lnkctl2, addr: pcie->base + BRCM_PCIE_CAP_REGS + PCI_EXP_LNKCTL2); |
394 | } |
395 | |
396 | static void brcm_pcie_set_outbound_win(struct brcm_pcie *pcie, |
397 | unsigned int win, u64 cpu_addr, |
398 | u64 pcie_addr, u64 size) |
399 | { |
400 | u32 cpu_addr_mb_high, limit_addr_mb_high; |
401 | phys_addr_t cpu_addr_mb, limit_addr_mb; |
402 | int high_addr_shift; |
403 | u32 tmp; |
404 | |
405 | /* Set the base of the pcie_addr window */ |
406 | writel(lower_32_bits(pcie_addr), addr: pcie->base + PCIE_MEM_WIN0_LO(win)); |
407 | writel(upper_32_bits(pcie_addr), addr: pcie->base + PCIE_MEM_WIN0_HI(win)); |
408 | |
409 | /* Write the addr base & limit lower bits (in MBs) */ |
410 | cpu_addr_mb = cpu_addr / SZ_1M; |
411 | limit_addr_mb = (cpu_addr + size - 1) / SZ_1M; |
412 | |
413 | tmp = readl(addr: pcie->base + PCIE_MEM_WIN0_BASE_LIMIT(win)); |
414 | u32p_replace_bits(p: &tmp, val: cpu_addr_mb, |
415 | PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_BASE_MASK); |
416 | u32p_replace_bits(p: &tmp, val: limit_addr_mb, |
417 | PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_LIMIT_MASK); |
418 | writel(val: tmp, addr: pcie->base + PCIE_MEM_WIN0_BASE_LIMIT(win)); |
419 | |
420 | if (is_bmips(pcie)) |
421 | return; |
422 | |
423 | /* Write the cpu & limit addr upper bits */ |
424 | high_addr_shift = |
425 | HWEIGHT32(PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_BASE_MASK); |
426 | |
427 | cpu_addr_mb_high = cpu_addr_mb >> high_addr_shift; |
428 | tmp = readl(addr: pcie->base + PCIE_MEM_WIN0_BASE_HI(win)); |
429 | u32p_replace_bits(p: &tmp, val: cpu_addr_mb_high, |
430 | PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI_BASE_MASK); |
431 | writel(val: tmp, addr: pcie->base + PCIE_MEM_WIN0_BASE_HI(win)); |
432 | |
433 | limit_addr_mb_high = limit_addr_mb >> high_addr_shift; |
434 | tmp = readl(addr: pcie->base + PCIE_MEM_WIN0_LIMIT_HI(win)); |
435 | u32p_replace_bits(p: &tmp, val: limit_addr_mb_high, |
436 | PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI_LIMIT_MASK); |
437 | writel(val: tmp, addr: pcie->base + PCIE_MEM_WIN0_LIMIT_HI(win)); |
438 | } |
439 | |
440 | static struct irq_chip brcm_msi_irq_chip = { |
441 | .name = "BRCM STB PCIe MSI" , |
442 | .irq_ack = irq_chip_ack_parent, |
443 | .irq_mask = pci_msi_mask_irq, |
444 | .irq_unmask = pci_msi_unmask_irq, |
445 | }; |
446 | |
447 | static struct msi_domain_info brcm_msi_domain_info = { |
448 | .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | |
449 | MSI_FLAG_MULTI_PCI_MSI), |
450 | .chip = &brcm_msi_irq_chip, |
451 | }; |
452 | |
453 | static void brcm_pcie_msi_isr(struct irq_desc *desc) |
454 | { |
455 | struct irq_chip *chip = irq_desc_get_chip(desc); |
456 | unsigned long status; |
457 | struct brcm_msi *msi; |
458 | struct device *dev; |
459 | u32 bit; |
460 | |
461 | chained_irq_enter(chip, desc); |
462 | msi = irq_desc_get_handler_data(desc); |
463 | dev = msi->dev; |
464 | |
465 | status = readl(addr: msi->intr_base + MSI_INT_STATUS); |
466 | status >>= msi->legacy_shift; |
467 | |
468 | for_each_set_bit(bit, &status, msi->nr) { |
469 | int ret; |
470 | ret = generic_handle_domain_irq(domain: msi->inner_domain, hwirq: bit); |
471 | if (ret) |
472 | dev_dbg(dev, "unexpected MSI\n" ); |
473 | } |
474 | |
475 | chained_irq_exit(chip, desc); |
476 | } |
477 | |
478 | static void brcm_msi_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) |
479 | { |
480 | struct brcm_msi *msi = irq_data_get_irq_chip_data(d: data); |
481 | |
482 | msg->address_lo = lower_32_bits(msi->target_addr); |
483 | msg->address_hi = upper_32_bits(msi->target_addr); |
484 | msg->data = (0xffff & PCIE_MISC_MSI_DATA_CONFIG_VAL_32) | data->hwirq; |
485 | } |
486 | |
487 | static int brcm_msi_set_affinity(struct irq_data *irq_data, |
488 | const struct cpumask *mask, bool force) |
489 | { |
490 | return -EINVAL; |
491 | } |
492 | |
493 | static void brcm_msi_ack_irq(struct irq_data *data) |
494 | { |
495 | struct brcm_msi *msi = irq_data_get_irq_chip_data(d: data); |
496 | const int shift_amt = data->hwirq + msi->legacy_shift; |
497 | |
498 | writel(val: 1 << shift_amt, addr: msi->intr_base + MSI_INT_CLR); |
499 | } |
500 | |
501 | |
502 | static struct irq_chip brcm_msi_bottom_irq_chip = { |
503 | .name = "BRCM STB MSI" , |
504 | .irq_compose_msi_msg = brcm_msi_compose_msi_msg, |
505 | .irq_set_affinity = brcm_msi_set_affinity, |
506 | .irq_ack = brcm_msi_ack_irq, |
507 | }; |
508 | |
509 | static int brcm_msi_alloc(struct brcm_msi *msi, unsigned int nr_irqs) |
510 | { |
511 | int hwirq; |
512 | |
513 | mutex_lock(&msi->lock); |
514 | hwirq = bitmap_find_free_region(bitmap: msi->used, bits: msi->nr, |
515 | order_base_2(nr_irqs)); |
516 | mutex_unlock(lock: &msi->lock); |
517 | |
518 | return hwirq; |
519 | } |
520 | |
521 | static void brcm_msi_free(struct brcm_msi *msi, unsigned long hwirq, |
522 | unsigned int nr_irqs) |
523 | { |
524 | mutex_lock(&msi->lock); |
525 | bitmap_release_region(bitmap: msi->used, pos: hwirq, order_base_2(nr_irqs)); |
526 | mutex_unlock(lock: &msi->lock); |
527 | } |
528 | |
529 | static int brcm_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, |
530 | unsigned int nr_irqs, void *args) |
531 | { |
532 | struct brcm_msi *msi = domain->host_data; |
533 | int hwirq, i; |
534 | |
535 | hwirq = brcm_msi_alloc(msi, nr_irqs); |
536 | |
537 | if (hwirq < 0) |
538 | return hwirq; |
539 | |
540 | for (i = 0; i < nr_irqs; i++) |
541 | irq_domain_set_info(domain, virq: virq + i, hwirq: hwirq + i, |
542 | chip: &brcm_msi_bottom_irq_chip, chip_data: domain->host_data, |
543 | handler: handle_edge_irq, NULL, NULL); |
544 | return 0; |
545 | } |
546 | |
547 | static void brcm_irq_domain_free(struct irq_domain *domain, |
548 | unsigned int virq, unsigned int nr_irqs) |
549 | { |
550 | struct irq_data *d = irq_domain_get_irq_data(domain, virq); |
551 | struct brcm_msi *msi = irq_data_get_irq_chip_data(d); |
552 | |
553 | brcm_msi_free(msi, hwirq: d->hwirq, nr_irqs); |
554 | } |
555 | |
556 | static const struct irq_domain_ops msi_domain_ops = { |
557 | .alloc = brcm_irq_domain_alloc, |
558 | .free = brcm_irq_domain_free, |
559 | }; |
560 | |
561 | static int brcm_allocate_domains(struct brcm_msi *msi) |
562 | { |
563 | struct fwnode_handle *fwnode = of_node_to_fwnode(node: msi->np); |
564 | struct device *dev = msi->dev; |
565 | |
566 | msi->inner_domain = irq_domain_add_linear(NULL, size: msi->nr, ops: &msi_domain_ops, host_data: msi); |
567 | if (!msi->inner_domain) { |
568 | dev_err(dev, "failed to create IRQ domain\n" ); |
569 | return -ENOMEM; |
570 | } |
571 | |
572 | msi->msi_domain = pci_msi_create_irq_domain(fwnode, |
573 | info: &brcm_msi_domain_info, |
574 | parent: msi->inner_domain); |
575 | if (!msi->msi_domain) { |
576 | dev_err(dev, "failed to create MSI domain\n" ); |
577 | irq_domain_remove(host: msi->inner_domain); |
578 | return -ENOMEM; |
579 | } |
580 | |
581 | return 0; |
582 | } |
583 | |
584 | static void brcm_free_domains(struct brcm_msi *msi) |
585 | { |
586 | irq_domain_remove(host: msi->msi_domain); |
587 | irq_domain_remove(host: msi->inner_domain); |
588 | } |
589 | |
590 | static void brcm_msi_remove(struct brcm_pcie *pcie) |
591 | { |
592 | struct brcm_msi *msi = pcie->msi; |
593 | |
594 | if (!msi) |
595 | return; |
596 | irq_set_chained_handler_and_data(irq: msi->irq, NULL, NULL); |
597 | brcm_free_domains(msi); |
598 | } |
599 | |
600 | static void brcm_msi_set_regs(struct brcm_msi *msi) |
601 | { |
602 | u32 val = msi->legacy ? BRCM_INT_PCI_MSI_LEGACY_MASK : |
603 | BRCM_INT_PCI_MSI_MASK; |
604 | |
605 | writel(val, addr: msi->intr_base + MSI_INT_MASK_CLR); |
606 | writel(val, addr: msi->intr_base + MSI_INT_CLR); |
607 | |
608 | /* |
609 | * The 0 bit of PCIE_MISC_MSI_BAR_CONFIG_LO is repurposed to MSI |
610 | * enable, which we set to 1. |
611 | */ |
612 | writel(lower_32_bits(msi->target_addr) | 0x1, |
613 | addr: msi->base + PCIE_MISC_MSI_BAR_CONFIG_LO); |
614 | writel(upper_32_bits(msi->target_addr), |
615 | addr: msi->base + PCIE_MISC_MSI_BAR_CONFIG_HI); |
616 | |
617 | val = msi->legacy ? PCIE_MISC_MSI_DATA_CONFIG_VAL_8 : PCIE_MISC_MSI_DATA_CONFIG_VAL_32; |
618 | writel(val, addr: msi->base + PCIE_MISC_MSI_DATA_CONFIG); |
619 | } |
620 | |
621 | static int brcm_pcie_enable_msi(struct brcm_pcie *pcie) |
622 | { |
623 | struct brcm_msi *msi; |
624 | int irq, ret; |
625 | struct device *dev = pcie->dev; |
626 | |
627 | irq = irq_of_parse_and_map(node: dev->of_node, index: 1); |
628 | if (irq <= 0) { |
629 | dev_err(dev, "cannot map MSI interrupt\n" ); |
630 | return -ENODEV; |
631 | } |
632 | |
633 | msi = devm_kzalloc(dev, size: sizeof(struct brcm_msi), GFP_KERNEL); |
634 | if (!msi) |
635 | return -ENOMEM; |
636 | |
637 | mutex_init(&msi->lock); |
638 | msi->dev = dev; |
639 | msi->base = pcie->base; |
640 | msi->np = pcie->np; |
641 | msi->target_addr = pcie->msi_target_addr; |
642 | msi->irq = irq; |
643 | msi->legacy = pcie->hw_rev < BRCM_PCIE_HW_REV_33; |
644 | |
645 | /* |
646 | * Sanity check to make sure that the 'used' bitmap in struct brcm_msi |
647 | * is large enough. |
648 | */ |
649 | BUILD_BUG_ON(BRCM_INT_PCI_MSI_LEGACY_NR > BRCM_INT_PCI_MSI_NR); |
650 | |
651 | if (msi->legacy) { |
652 | msi->intr_base = msi->base + PCIE_INTR2_CPU_BASE; |
653 | msi->nr = BRCM_INT_PCI_MSI_LEGACY_NR; |
654 | msi->legacy_shift = 24; |
655 | } else { |
656 | msi->intr_base = msi->base + PCIE_MSI_INTR2_BASE; |
657 | msi->nr = BRCM_INT_PCI_MSI_NR; |
658 | msi->legacy_shift = 0; |
659 | } |
660 | |
661 | ret = brcm_allocate_domains(msi); |
662 | if (ret) |
663 | return ret; |
664 | |
665 | irq_set_chained_handler_and_data(irq: msi->irq, handle: brcm_pcie_msi_isr, data: msi); |
666 | |
667 | brcm_msi_set_regs(msi); |
668 | pcie->msi = msi; |
669 | |
670 | return 0; |
671 | } |
672 | |
673 | /* The controller is capable of serving in both RC and EP roles */ |
674 | static bool brcm_pcie_rc_mode(struct brcm_pcie *pcie) |
675 | { |
676 | void __iomem *base = pcie->base; |
677 | u32 val = readl(addr: base + PCIE_MISC_PCIE_STATUS); |
678 | |
679 | return !!FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_PORT_MASK, val); |
680 | } |
681 | |
682 | static bool brcm_pcie_link_up(struct brcm_pcie *pcie) |
683 | { |
684 | u32 val = readl(addr: pcie->base + PCIE_MISC_PCIE_STATUS); |
685 | u32 dla = FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_DL_ACTIVE_MASK, val); |
686 | u32 plu = FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_PHYLINKUP_MASK, val); |
687 | |
688 | return dla && plu; |
689 | } |
690 | |
691 | static void __iomem *brcm_pcie_map_bus(struct pci_bus *bus, |
692 | unsigned int devfn, int where) |
693 | { |
694 | struct brcm_pcie *pcie = bus->sysdata; |
695 | void __iomem *base = pcie->base; |
696 | int idx; |
697 | |
698 | /* Accesses to the RC go right to the RC registers if !devfn */ |
699 | if (pci_is_root_bus(pbus: bus)) |
700 | return devfn ? NULL : base + PCIE_ECAM_REG(where); |
701 | |
702 | /* An access to our HW w/o link-up will cause a CPU Abort */ |
703 | if (!brcm_pcie_link_up(pcie)) |
704 | return NULL; |
705 | |
706 | /* For devices, write to the config space index register */ |
707 | idx = PCIE_ECAM_OFFSET(bus->number, devfn, 0); |
708 | writel(val: idx, addr: pcie->base + PCIE_EXT_CFG_INDEX); |
709 | return base + PCIE_EXT_CFG_DATA + PCIE_ECAM_REG(where); |
710 | } |
711 | |
712 | static void __iomem *brcm7425_pcie_map_bus(struct pci_bus *bus, |
713 | unsigned int devfn, int where) |
714 | { |
715 | struct brcm_pcie *pcie = bus->sysdata; |
716 | void __iomem *base = pcie->base; |
717 | int idx; |
718 | |
719 | /* Accesses to the RC go right to the RC registers if !devfn */ |
720 | if (pci_is_root_bus(pbus: bus)) |
721 | return devfn ? NULL : base + PCIE_ECAM_REG(where); |
722 | |
723 | /* An access to our HW w/o link-up will cause a CPU Abort */ |
724 | if (!brcm_pcie_link_up(pcie)) |
725 | return NULL; |
726 | |
727 | /* For devices, write to the config space index register */ |
728 | idx = PCIE_ECAM_OFFSET(bus->number, devfn, where); |
729 | writel(val: idx, addr: base + IDX_ADDR(pcie)); |
730 | return base + DATA_ADDR(pcie); |
731 | } |
732 | |
733 | static void brcm_pcie_bridge_sw_init_set_generic(struct brcm_pcie *pcie, u32 val) |
734 | { |
735 | u32 tmp, mask = RGR1_SW_INIT_1_INIT_GENERIC_MASK; |
736 | u32 shift = RGR1_SW_INIT_1_INIT_GENERIC_SHIFT; |
737 | |
738 | tmp = readl(addr: pcie->base + PCIE_RGR1_SW_INIT_1(pcie)); |
739 | tmp = (tmp & ~mask) | ((val << shift) & mask); |
740 | writel(val: tmp, addr: pcie->base + PCIE_RGR1_SW_INIT_1(pcie)); |
741 | } |
742 | |
743 | static void brcm_pcie_bridge_sw_init_set_7278(struct brcm_pcie *pcie, u32 val) |
744 | { |
745 | u32 tmp, mask = RGR1_SW_INIT_1_INIT_7278_MASK; |
746 | u32 shift = RGR1_SW_INIT_1_INIT_7278_SHIFT; |
747 | |
748 | tmp = readl(addr: pcie->base + PCIE_RGR1_SW_INIT_1(pcie)); |
749 | tmp = (tmp & ~mask) | ((val << shift) & mask); |
750 | writel(val: tmp, addr: pcie->base + PCIE_RGR1_SW_INIT_1(pcie)); |
751 | } |
752 | |
753 | static void brcm_pcie_perst_set_4908(struct brcm_pcie *pcie, u32 val) |
754 | { |
755 | if (WARN_ONCE(!pcie->perst_reset, "missing PERST# reset controller\n" )) |
756 | return; |
757 | |
758 | if (val) |
759 | reset_control_assert(rstc: pcie->perst_reset); |
760 | else |
761 | reset_control_deassert(rstc: pcie->perst_reset); |
762 | } |
763 | |
764 | static void brcm_pcie_perst_set_7278(struct brcm_pcie *pcie, u32 val) |
765 | { |
766 | u32 tmp; |
767 | |
768 | /* Perst bit has moved and assert value is 0 */ |
769 | tmp = readl(addr: pcie->base + PCIE_MISC_PCIE_CTRL); |
770 | u32p_replace_bits(p: &tmp, val: !val, PCIE_MISC_PCIE_CTRL_PCIE_PERSTB_MASK); |
771 | writel(val: tmp, addr: pcie->base + PCIE_MISC_PCIE_CTRL); |
772 | } |
773 | |
774 | static void brcm_pcie_perst_set_generic(struct brcm_pcie *pcie, u32 val) |
775 | { |
776 | u32 tmp; |
777 | |
778 | tmp = readl(addr: pcie->base + PCIE_RGR1_SW_INIT_1(pcie)); |
779 | u32p_replace_bits(p: &tmp, val, PCIE_RGR1_SW_INIT_1_PERST_MASK); |
780 | writel(val: tmp, addr: pcie->base + PCIE_RGR1_SW_INIT_1(pcie)); |
781 | } |
782 | |
783 | static int brcm_pcie_get_rc_bar2_size_and_offset(struct brcm_pcie *pcie, |
784 | u64 *rc_bar2_size, |
785 | u64 *rc_bar2_offset) |
786 | { |
787 | struct pci_host_bridge *bridge = pci_host_bridge_from_priv(priv: pcie); |
788 | struct resource_entry *entry; |
789 | struct device *dev = pcie->dev; |
790 | u64 lowest_pcie_addr = ~(u64)0; |
791 | int ret, i = 0; |
792 | u64 size = 0; |
793 | |
794 | resource_list_for_each_entry(entry, &bridge->dma_ranges) { |
795 | u64 pcie_beg = entry->res->start - entry->offset; |
796 | |
797 | size += entry->res->end - entry->res->start + 1; |
798 | if (pcie_beg < lowest_pcie_addr) |
799 | lowest_pcie_addr = pcie_beg; |
800 | } |
801 | |
802 | if (lowest_pcie_addr == ~(u64)0) { |
803 | dev_err(dev, "DT node has no dma-ranges\n" ); |
804 | return -EINVAL; |
805 | } |
806 | |
807 | ret = of_property_read_variable_u64_array(np: pcie->np, propname: "brcm,scb-sizes" , out_values: pcie->memc_size, sz_min: 1, |
808 | PCIE_BRCM_MAX_MEMC); |
809 | |
810 | if (ret <= 0) { |
811 | /* Make an educated guess */ |
812 | pcie->num_memc = 1; |
813 | pcie->memc_size[0] = 1ULL << fls64(x: size - 1); |
814 | } else { |
815 | pcie->num_memc = ret; |
816 | } |
817 | |
818 | /* Each memc is viewed through a "port" that is a power of 2 */ |
819 | for (i = 0, size = 0; i < pcie->num_memc; i++) |
820 | size += pcie->memc_size[i]; |
821 | |
822 | /* System memory starts at this address in PCIe-space */ |
823 | *rc_bar2_offset = lowest_pcie_addr; |
824 | /* The sum of all memc views must also be a power of 2 */ |
825 | *rc_bar2_size = 1ULL << fls64(x: size - 1); |
826 | |
827 | /* |
828 | * We validate the inbound memory view even though we should trust |
829 | * whatever the device-tree provides. This is because of an HW issue on |
830 | * early Raspberry Pi 4's revisions (bcm2711). It turns out its |
831 | * firmware has to dynamically edit dma-ranges due to a bug on the |
832 | * PCIe controller integration, which prohibits any access above the |
833 | * lower 3GB of memory. Given this, we decided to keep the dma-ranges |
834 | * in check, avoiding hard to debug device-tree related issues in the |
835 | * future: |
836 | * |
837 | * The PCIe host controller by design must set the inbound viewport to |
838 | * be a contiguous arrangement of all of the system's memory. In |
839 | * addition, its size mut be a power of two. To further complicate |
840 | * matters, the viewport must start on a pcie-address that is aligned |
841 | * on a multiple of its size. If a portion of the viewport does not |
842 | * represent system memory -- e.g. 3GB of memory requires a 4GB |
843 | * viewport -- we can map the outbound memory in or after 3GB and even |
844 | * though the viewport will overlap the outbound memory the controller |
845 | * will know to send outbound memory downstream and everything else |
846 | * upstream. |
847 | * |
848 | * For example: |
849 | * |
850 | * - The best-case scenario, memory up to 3GB, is to place the inbound |
851 | * region in the first 4GB of pcie-space, as some legacy devices can |
852 | * only address 32bits. We would also like to put the MSI under 4GB |
853 | * as well, since some devices require a 32bit MSI target address. |
854 | * |
855 | * - If the system memory is 4GB or larger we cannot start the inbound |
856 | * region at location 0 (since we have to allow some space for |
857 | * outbound memory @ 3GB). So instead it will start at the 1x |
858 | * multiple of its size |
859 | */ |
860 | if (!*rc_bar2_size || (*rc_bar2_offset & (*rc_bar2_size - 1)) || |
861 | (*rc_bar2_offset < SZ_4G && *rc_bar2_offset > SZ_2G)) { |
862 | dev_err(dev, "Invalid rc_bar2_offset/size: size 0x%llx, off 0x%llx\n" , |
863 | *rc_bar2_size, *rc_bar2_offset); |
864 | return -EINVAL; |
865 | } |
866 | |
867 | return 0; |
868 | } |
869 | |
870 | static int brcm_pcie_setup(struct brcm_pcie *pcie) |
871 | { |
872 | u64 rc_bar2_offset, rc_bar2_size; |
873 | void __iomem *base = pcie->base; |
874 | struct pci_host_bridge *bridge; |
875 | struct resource_entry *entry; |
876 | u32 tmp, burst, aspm_support; |
877 | int num_out_wins = 0; |
878 | int ret, memc; |
879 | |
880 | /* Reset the bridge */ |
881 | pcie->bridge_sw_init_set(pcie, 1); |
882 | |
883 | /* Ensure that PERST# is asserted; some bootloaders may deassert it. */ |
884 | if (pcie->type == BCM2711) |
885 | pcie->perst_set(pcie, 1); |
886 | |
887 | usleep_range(min: 100, max: 200); |
888 | |
889 | /* Take the bridge out of reset */ |
890 | pcie->bridge_sw_init_set(pcie, 0); |
891 | |
892 | tmp = readl(addr: base + PCIE_MISC_HARD_PCIE_HARD_DEBUG); |
893 | if (is_bmips(pcie)) |
894 | tmp &= ~PCIE_BMIPS_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK; |
895 | else |
896 | tmp &= ~PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK; |
897 | writel(val: tmp, addr: base + PCIE_MISC_HARD_PCIE_HARD_DEBUG); |
898 | /* Wait for SerDes to be stable */ |
899 | usleep_range(min: 100, max: 200); |
900 | |
901 | /* |
902 | * SCB_MAX_BURST_SIZE is a two bit field. For GENERIC chips it |
903 | * is encoded as 0=128, 1=256, 2=512, 3=Rsvd, for BCM7278 it |
904 | * is encoded as 0=Rsvd, 1=128, 2=256, 3=512. |
905 | */ |
906 | if (is_bmips(pcie)) |
907 | burst = 0x1; /* 256 bytes */ |
908 | else if (pcie->type == BCM2711) |
909 | burst = 0x0; /* 128 bytes */ |
910 | else if (pcie->type == BCM7278) |
911 | burst = 0x3; /* 512 bytes */ |
912 | else |
913 | burst = 0x2; /* 512 bytes */ |
914 | |
915 | /* |
916 | * Set SCB_MAX_BURST_SIZE, CFG_READ_UR_MODE, SCB_ACCESS_EN, |
917 | * RCB_MPS_MODE, RCB_64B_MODE |
918 | */ |
919 | tmp = readl(addr: base + PCIE_MISC_MISC_CTRL); |
920 | u32p_replace_bits(p: &tmp, val: 1, PCIE_MISC_MISC_CTRL_SCB_ACCESS_EN_MASK); |
921 | u32p_replace_bits(p: &tmp, val: 1, PCIE_MISC_MISC_CTRL_CFG_READ_UR_MODE_MASK); |
922 | u32p_replace_bits(p: &tmp, val: burst, PCIE_MISC_MISC_CTRL_MAX_BURST_SIZE_MASK); |
923 | u32p_replace_bits(p: &tmp, val: 1, PCIE_MISC_MISC_CTRL_PCIE_RCB_MPS_MODE_MASK); |
924 | u32p_replace_bits(p: &tmp, val: 1, PCIE_MISC_MISC_CTRL_PCIE_RCB_64B_MODE_MASK); |
925 | writel(val: tmp, addr: base + PCIE_MISC_MISC_CTRL); |
926 | |
927 | ret = brcm_pcie_get_rc_bar2_size_and_offset(pcie, rc_bar2_size: &rc_bar2_size, |
928 | rc_bar2_offset: &rc_bar2_offset); |
929 | if (ret) |
930 | return ret; |
931 | |
932 | tmp = lower_32_bits(rc_bar2_offset); |
933 | u32p_replace_bits(p: &tmp, val: brcm_pcie_encode_ibar_size(size: rc_bar2_size), |
934 | PCIE_MISC_RC_BAR2_CONFIG_LO_SIZE_MASK); |
935 | writel(val: tmp, addr: base + PCIE_MISC_RC_BAR2_CONFIG_LO); |
936 | writel(upper_32_bits(rc_bar2_offset), |
937 | addr: base + PCIE_MISC_RC_BAR2_CONFIG_HI); |
938 | |
939 | tmp = readl(addr: base + PCIE_MISC_MISC_CTRL); |
940 | for (memc = 0; memc < pcie->num_memc; memc++) { |
941 | u32 scb_size_val = ilog2(pcie->memc_size[memc]) - 15; |
942 | |
943 | if (memc == 0) |
944 | u32p_replace_bits(p: &tmp, val: scb_size_val, SCB_SIZE_MASK(0)); |
945 | else if (memc == 1) |
946 | u32p_replace_bits(p: &tmp, val: scb_size_val, SCB_SIZE_MASK(1)); |
947 | else if (memc == 2) |
948 | u32p_replace_bits(p: &tmp, val: scb_size_val, SCB_SIZE_MASK(2)); |
949 | } |
950 | writel(val: tmp, addr: base + PCIE_MISC_MISC_CTRL); |
951 | |
952 | /* |
953 | * We ideally want the MSI target address to be located in the 32bit |
954 | * addressable memory area. Some devices might depend on it. This is |
955 | * possible either when the inbound window is located above the lower |
956 | * 4GB or when the inbound area is smaller than 4GB (taking into |
957 | * account the rounding-up we're forced to perform). |
958 | */ |
959 | if (rc_bar2_offset >= SZ_4G || (rc_bar2_size + rc_bar2_offset) < SZ_4G) |
960 | pcie->msi_target_addr = BRCM_MSI_TARGET_ADDR_LT_4GB; |
961 | else |
962 | pcie->msi_target_addr = BRCM_MSI_TARGET_ADDR_GT_4GB; |
963 | |
964 | if (!brcm_pcie_rc_mode(pcie)) { |
965 | dev_err(pcie->dev, "PCIe RC controller misconfigured as Endpoint\n" ); |
966 | return -EINVAL; |
967 | } |
968 | |
969 | /* disable the PCIe->GISB memory window (RC_BAR1) */ |
970 | tmp = readl(addr: base + PCIE_MISC_RC_BAR1_CONFIG_LO); |
971 | tmp &= ~PCIE_MISC_RC_BAR1_CONFIG_LO_SIZE_MASK; |
972 | writel(val: tmp, addr: base + PCIE_MISC_RC_BAR1_CONFIG_LO); |
973 | |
974 | /* disable the PCIe->SCB memory window (RC_BAR3) */ |
975 | tmp = readl(addr: base + PCIE_MISC_RC_BAR3_CONFIG_LO); |
976 | tmp &= ~PCIE_MISC_RC_BAR3_CONFIG_LO_SIZE_MASK; |
977 | writel(val: tmp, addr: base + PCIE_MISC_RC_BAR3_CONFIG_LO); |
978 | |
979 | /* Don't advertise L0s capability if 'aspm-no-l0s' */ |
980 | aspm_support = PCIE_LINK_STATE_L1; |
981 | if (!of_property_read_bool(np: pcie->np, propname: "aspm-no-l0s" )) |
982 | aspm_support |= PCIE_LINK_STATE_L0S; |
983 | tmp = readl(addr: base + PCIE_RC_CFG_PRIV1_LINK_CAPABILITY); |
984 | u32p_replace_bits(p: &tmp, val: aspm_support, |
985 | PCIE_RC_CFG_PRIV1_LINK_CAPABILITY_ASPM_SUPPORT_MASK); |
986 | writel(val: tmp, addr: base + PCIE_RC_CFG_PRIV1_LINK_CAPABILITY); |
987 | |
988 | /* |
989 | * For config space accesses on the RC, show the right class for |
990 | * a PCIe-PCIe bridge (the default setting is to be EP mode). |
991 | */ |
992 | tmp = readl(addr: base + PCIE_RC_CFG_PRIV1_ID_VAL3); |
993 | u32p_replace_bits(p: &tmp, val: 0x060400, |
994 | PCIE_RC_CFG_PRIV1_ID_VAL3_CLASS_CODE_MASK); |
995 | writel(val: tmp, addr: base + PCIE_RC_CFG_PRIV1_ID_VAL3); |
996 | |
997 | bridge = pci_host_bridge_from_priv(priv: pcie); |
998 | resource_list_for_each_entry(entry, &bridge->windows) { |
999 | struct resource *res = entry->res; |
1000 | |
1001 | if (resource_type(res) != IORESOURCE_MEM) |
1002 | continue; |
1003 | |
1004 | if (num_out_wins >= BRCM_NUM_PCIE_OUT_WINS) { |
1005 | dev_err(pcie->dev, "too many outbound wins\n" ); |
1006 | return -EINVAL; |
1007 | } |
1008 | |
1009 | if (is_bmips(pcie)) { |
1010 | u64 start = res->start; |
1011 | unsigned int j, nwins = resource_size(res) / SZ_128M; |
1012 | |
1013 | /* bmips PCIe outbound windows have a 128MB max size */ |
1014 | if (nwins > BRCM_NUM_PCIE_OUT_WINS) |
1015 | nwins = BRCM_NUM_PCIE_OUT_WINS; |
1016 | for (j = 0; j < nwins; j++, start += SZ_128M) |
1017 | brcm_pcie_set_outbound_win(pcie, win: j, cpu_addr: start, |
1018 | pcie_addr: start - entry->offset, |
1019 | SZ_128M); |
1020 | break; |
1021 | } |
1022 | brcm_pcie_set_outbound_win(pcie, win: num_out_wins, cpu_addr: res->start, |
1023 | pcie_addr: res->start - entry->offset, |
1024 | size: resource_size(res)); |
1025 | num_out_wins++; |
1026 | } |
1027 | |
1028 | /* PCIe->SCB endian mode for BAR */ |
1029 | tmp = readl(addr: base + PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1); |
1030 | u32p_replace_bits(p: &tmp, PCIE_RC_CFG_VENDOR_SPCIFIC_REG1_LITTLE_ENDIAN, |
1031 | PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1_ENDIAN_MODE_BAR2_MASK); |
1032 | writel(val: tmp, addr: base + PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1); |
1033 | |
1034 | return 0; |
1035 | } |
1036 | |
1037 | /* |
1038 | * This extends the timeout period for an access to an internal bus. This |
1039 | * access timeout may occur during L1SS sleep periods, even without the |
1040 | * presence of a PCIe access. |
1041 | */ |
1042 | static void brcm_extend_rbus_timeout(struct brcm_pcie *pcie) |
1043 | { |
1044 | /* TIMEOUT register is two registers before RGR1_SW_INIT_1 */ |
1045 | const unsigned int REG_OFFSET = PCIE_RGR1_SW_INIT_1(pcie) - 8; |
1046 | u32 timeout_us = 4000000; /* 4 seconds, our setting for L1SS */ |
1047 | |
1048 | /* Each unit in timeout register is 1/216,000,000 seconds */ |
1049 | writel(val: 216 * timeout_us, addr: pcie->base + REG_OFFSET); |
1050 | } |
1051 | |
1052 | static void brcm_config_clkreq(struct brcm_pcie *pcie) |
1053 | { |
1054 | static const char err_msg[] = "invalid 'brcm,clkreq-mode' DT string\n" ; |
1055 | const char *mode = "default" ; |
1056 | u32 clkreq_cntl; |
1057 | int ret, tmp; |
1058 | |
1059 | ret = of_property_read_string(np: pcie->np, propname: "brcm,clkreq-mode" , out_string: &mode); |
1060 | if (ret && ret != -EINVAL) { |
1061 | dev_err(pcie->dev, err_msg); |
1062 | mode = "safe" ; |
1063 | } |
1064 | |
1065 | /* Start out assuming safe mode (both mode bits cleared) */ |
1066 | clkreq_cntl = readl(addr: pcie->base + PCIE_MISC_HARD_PCIE_HARD_DEBUG); |
1067 | clkreq_cntl &= ~PCIE_CLKREQ_MASK; |
1068 | |
1069 | if (strcmp(mode, "no-l1ss" ) == 0) { |
1070 | /* |
1071 | * "no-l1ss" -- Provides Clock Power Management, L0s, and |
1072 | * L1, but cannot provide L1 substate (L1SS) power |
1073 | * savings. If the downstream device connected to the RC is |
1074 | * L1SS capable AND the OS enables L1SS, all PCIe traffic |
1075 | * may abruptly halt, potentially hanging the system. |
1076 | */ |
1077 | clkreq_cntl |= PCIE_MISC_HARD_PCIE_HARD_DEBUG_CLKREQ_DEBUG_ENABLE_MASK; |
1078 | /* |
1079 | * We want to un-advertise L1 substates because if the OS |
1080 | * tries to configure the controller into using L1 substate |
1081 | * power savings it may fail or hang when the RC HW is in |
1082 | * "no-l1ss" mode. |
1083 | */ |
1084 | tmp = readl(addr: pcie->base + PCIE_RC_CFG_PRIV1_ROOT_CAP); |
1085 | u32p_replace_bits(p: &tmp, val: 2, PCIE_RC_CFG_PRIV1_ROOT_CAP_L1SS_MODE_MASK); |
1086 | writel(val: tmp, addr: pcie->base + PCIE_RC_CFG_PRIV1_ROOT_CAP); |
1087 | |
1088 | } else if (strcmp(mode, "default" ) == 0) { |
1089 | /* |
1090 | * "default" -- Provides L0s, L1, and L1SS, but not |
1091 | * compliant to provide Clock Power Management; |
1092 | * specifically, may not be able to meet the Tclron max |
1093 | * timing of 400ns as specified in "Dynamic Clock Control", |
1094 | * section 3.2.5.2.2 of the PCIe spec. This situation is |
1095 | * atypical and should happen only with older devices. |
1096 | */ |
1097 | clkreq_cntl |= PCIE_MISC_HARD_PCIE_HARD_DEBUG_L1SS_ENABLE_MASK; |
1098 | brcm_extend_rbus_timeout(pcie); |
1099 | |
1100 | } else { |
1101 | /* |
1102 | * "safe" -- No power savings; refclk is driven by RC |
1103 | * unconditionally. |
1104 | */ |
1105 | if (strcmp(mode, "safe" ) != 0) |
1106 | dev_err(pcie->dev, err_msg); |
1107 | mode = "safe" ; |
1108 | } |
1109 | writel(val: clkreq_cntl, addr: pcie->base + PCIE_MISC_HARD_PCIE_HARD_DEBUG); |
1110 | |
1111 | dev_info(pcie->dev, "clkreq-mode set to %s\n" , mode); |
1112 | } |
1113 | |
1114 | static int brcm_pcie_start_link(struct brcm_pcie *pcie) |
1115 | { |
1116 | struct device *dev = pcie->dev; |
1117 | void __iomem *base = pcie->base; |
1118 | u16 nlw, cls, lnksta; |
1119 | bool ssc_good = false; |
1120 | int ret, i; |
1121 | |
1122 | /* Unassert the fundamental reset */ |
1123 | pcie->perst_set(pcie, 0); |
1124 | |
1125 | /* |
1126 | * Wait for 100ms after PERST# deassertion; see PCIe CEM specification |
1127 | * sections 2.2, PCIe r5.0, 6.6.1. |
1128 | */ |
1129 | msleep(msecs: 100); |
1130 | |
1131 | /* |
1132 | * Give the RC/EP even more time to wake up, before trying to |
1133 | * configure RC. Intermittently check status for link-up, up to a |
1134 | * total of 100ms. |
1135 | */ |
1136 | for (i = 0; i < 100 && !brcm_pcie_link_up(pcie); i += 5) |
1137 | msleep(msecs: 5); |
1138 | |
1139 | if (!brcm_pcie_link_up(pcie)) { |
1140 | dev_err(dev, "link down\n" ); |
1141 | return -ENODEV; |
1142 | } |
1143 | |
1144 | brcm_config_clkreq(pcie); |
1145 | |
1146 | if (pcie->gen) |
1147 | brcm_pcie_set_gen(pcie, gen: pcie->gen); |
1148 | |
1149 | if (pcie->ssc) { |
1150 | ret = brcm_pcie_set_ssc(pcie); |
1151 | if (ret == 0) |
1152 | ssc_good = true; |
1153 | else |
1154 | dev_err(dev, "failed attempt to enter ssc mode\n" ); |
1155 | } |
1156 | |
1157 | lnksta = readw(addr: base + BRCM_PCIE_CAP_REGS + PCI_EXP_LNKSTA); |
1158 | cls = FIELD_GET(PCI_EXP_LNKSTA_CLS, lnksta); |
1159 | nlw = FIELD_GET(PCI_EXP_LNKSTA_NLW, lnksta); |
1160 | dev_info(dev, "link up, %s x%u %s\n" , |
1161 | pci_speed_string(pcie_link_speed[cls]), nlw, |
1162 | ssc_good ? "(SSC)" : "(!SSC)" ); |
1163 | |
1164 | return 0; |
1165 | } |
1166 | |
1167 | static const char * const supplies[] = { |
1168 | "vpcie3v3" , |
1169 | "vpcie3v3aux" , |
1170 | "vpcie12v" , |
1171 | }; |
1172 | |
1173 | static void *alloc_subdev_regulators(struct device *dev) |
1174 | { |
1175 | const size_t size = sizeof(struct subdev_regulators) + |
1176 | sizeof(struct regulator_bulk_data) * ARRAY_SIZE(supplies); |
1177 | struct subdev_regulators *sr; |
1178 | int i; |
1179 | |
1180 | sr = devm_kzalloc(dev, size, GFP_KERNEL); |
1181 | if (sr) { |
1182 | sr->num_supplies = ARRAY_SIZE(supplies); |
1183 | for (i = 0; i < ARRAY_SIZE(supplies); i++) |
1184 | sr->supplies[i].supply = supplies[i]; |
1185 | } |
1186 | |
1187 | return sr; |
1188 | } |
1189 | |
1190 | static int brcm_pcie_add_bus(struct pci_bus *bus) |
1191 | { |
1192 | struct brcm_pcie *pcie = bus->sysdata; |
1193 | struct device *dev = &bus->dev; |
1194 | struct subdev_regulators *sr; |
1195 | int ret; |
1196 | |
1197 | if (!bus->parent || !pci_is_root_bus(pbus: bus->parent)) |
1198 | return 0; |
1199 | |
1200 | if (dev->of_node) { |
1201 | sr = alloc_subdev_regulators(dev); |
1202 | if (!sr) { |
1203 | dev_info(dev, "Can't allocate regulators for downstream device\n" ); |
1204 | goto no_regulators; |
1205 | } |
1206 | |
1207 | pcie->sr = sr; |
1208 | |
1209 | ret = regulator_bulk_get(dev, num_consumers: sr->num_supplies, consumers: sr->supplies); |
1210 | if (ret) { |
1211 | dev_info(dev, "No regulators for downstream device\n" ); |
1212 | goto no_regulators; |
1213 | } |
1214 | |
1215 | ret = regulator_bulk_enable(num_consumers: sr->num_supplies, consumers: sr->supplies); |
1216 | if (ret) { |
1217 | dev_err(dev, "Can't enable regulators for downstream device\n" ); |
1218 | regulator_bulk_free(num_consumers: sr->num_supplies, consumers: sr->supplies); |
1219 | pcie->sr = NULL; |
1220 | } |
1221 | } |
1222 | |
1223 | no_regulators: |
1224 | brcm_pcie_start_link(pcie); |
1225 | return 0; |
1226 | } |
1227 | |
1228 | static void brcm_pcie_remove_bus(struct pci_bus *bus) |
1229 | { |
1230 | struct brcm_pcie *pcie = bus->sysdata; |
1231 | struct subdev_regulators *sr = pcie->sr; |
1232 | struct device *dev = &bus->dev; |
1233 | |
1234 | if (!sr) |
1235 | return; |
1236 | |
1237 | if (regulator_bulk_disable(num_consumers: sr->num_supplies, consumers: sr->supplies)) |
1238 | dev_err(dev, "Failed to disable regulators for downstream device\n" ); |
1239 | regulator_bulk_free(num_consumers: sr->num_supplies, consumers: sr->supplies); |
1240 | pcie->sr = NULL; |
1241 | } |
1242 | |
1243 | /* L23 is a low-power PCIe link state */ |
1244 | static void brcm_pcie_enter_l23(struct brcm_pcie *pcie) |
1245 | { |
1246 | void __iomem *base = pcie->base; |
1247 | int l23, i; |
1248 | u32 tmp; |
1249 | |
1250 | /* Assert request for L23 */ |
1251 | tmp = readl(addr: base + PCIE_MISC_PCIE_CTRL); |
1252 | u32p_replace_bits(p: &tmp, val: 1, PCIE_MISC_PCIE_CTRL_PCIE_L23_REQUEST_MASK); |
1253 | writel(val: tmp, addr: base + PCIE_MISC_PCIE_CTRL); |
1254 | |
1255 | /* Wait up to 36 msec for L23 */ |
1256 | tmp = readl(addr: base + PCIE_MISC_PCIE_STATUS); |
1257 | l23 = FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_LINK_IN_L23_MASK, tmp); |
1258 | for (i = 0; i < 15 && !l23; i++) { |
1259 | usleep_range(min: 2000, max: 2400); |
1260 | tmp = readl(addr: base + PCIE_MISC_PCIE_STATUS); |
1261 | l23 = FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_LINK_IN_L23_MASK, |
1262 | tmp); |
1263 | } |
1264 | |
1265 | if (!l23) |
1266 | dev_err(pcie->dev, "failed to enter low-power link state\n" ); |
1267 | } |
1268 | |
1269 | static int brcm_phy_cntl(struct brcm_pcie *pcie, const int start) |
1270 | { |
1271 | static const u32 shifts[PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_NFLDS] = { |
1272 | PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_PWRDN_SHIFT, |
1273 | PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_RESET_SHIFT, |
1274 | PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_DIG_RESET_SHIFT,}; |
1275 | static const u32 masks[PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_NFLDS] = { |
1276 | PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_PWRDN_MASK, |
1277 | PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_RESET_MASK, |
1278 | PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_DIG_RESET_MASK,}; |
1279 | const int beg = start ? 0 : PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_NFLDS - 1; |
1280 | const int end = start ? PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_NFLDS : -1; |
1281 | u32 tmp, combined_mask = 0; |
1282 | u32 val; |
1283 | void __iomem *base = pcie->base; |
1284 | int i, ret; |
1285 | |
1286 | for (i = beg; i != end; start ? i++ : i--) { |
1287 | val = start ? BIT_MASK(shifts[i]) : 0; |
1288 | tmp = readl(addr: base + PCIE_DVT_PMU_PCIE_PHY_CTRL); |
1289 | tmp = (tmp & ~masks[i]) | (val & masks[i]); |
1290 | writel(val: tmp, addr: base + PCIE_DVT_PMU_PCIE_PHY_CTRL); |
1291 | usleep_range(min: 50, max: 200); |
1292 | combined_mask |= masks[i]; |
1293 | } |
1294 | |
1295 | tmp = readl(addr: base + PCIE_DVT_PMU_PCIE_PHY_CTRL); |
1296 | val = start ? combined_mask : 0; |
1297 | |
1298 | ret = (tmp & combined_mask) == val ? 0 : -EIO; |
1299 | if (ret) |
1300 | dev_err(pcie->dev, "failed to %s phy\n" , (start ? "start" : "stop" )); |
1301 | |
1302 | return ret; |
1303 | } |
1304 | |
1305 | static inline int brcm_phy_start(struct brcm_pcie *pcie) |
1306 | { |
1307 | return pcie->rescal ? brcm_phy_cntl(pcie, start: 1) : 0; |
1308 | } |
1309 | |
1310 | static inline int brcm_phy_stop(struct brcm_pcie *pcie) |
1311 | { |
1312 | return pcie->rescal ? brcm_phy_cntl(pcie, start: 0) : 0; |
1313 | } |
1314 | |
1315 | static void brcm_pcie_turn_off(struct brcm_pcie *pcie) |
1316 | { |
1317 | void __iomem *base = pcie->base; |
1318 | int tmp; |
1319 | |
1320 | if (brcm_pcie_link_up(pcie)) |
1321 | brcm_pcie_enter_l23(pcie); |
1322 | /* Assert fundamental reset */ |
1323 | pcie->perst_set(pcie, 1); |
1324 | |
1325 | /* Deassert request for L23 in case it was asserted */ |
1326 | tmp = readl(addr: base + PCIE_MISC_PCIE_CTRL); |
1327 | u32p_replace_bits(p: &tmp, val: 0, PCIE_MISC_PCIE_CTRL_PCIE_L23_REQUEST_MASK); |
1328 | writel(val: tmp, addr: base + PCIE_MISC_PCIE_CTRL); |
1329 | |
1330 | /* Turn off SerDes */ |
1331 | tmp = readl(addr: base + PCIE_MISC_HARD_PCIE_HARD_DEBUG); |
1332 | u32p_replace_bits(p: &tmp, val: 1, PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK); |
1333 | writel(val: tmp, addr: base + PCIE_MISC_HARD_PCIE_HARD_DEBUG); |
1334 | |
1335 | /* Shutdown PCIe bridge */ |
1336 | pcie->bridge_sw_init_set(pcie, 1); |
1337 | } |
1338 | |
1339 | static int pci_dev_may_wakeup(struct pci_dev *dev, void *data) |
1340 | { |
1341 | bool *ret = data; |
1342 | |
1343 | if (device_may_wakeup(dev: &dev->dev)) { |
1344 | *ret = true; |
1345 | dev_info(&dev->dev, "Possible wake-up device; regulators will not be disabled\n" ); |
1346 | } |
1347 | return (int) *ret; |
1348 | } |
1349 | |
1350 | static int brcm_pcie_suspend_noirq(struct device *dev) |
1351 | { |
1352 | struct brcm_pcie *pcie = dev_get_drvdata(dev); |
1353 | struct pci_host_bridge *bridge = pci_host_bridge_from_priv(priv: pcie); |
1354 | int ret; |
1355 | |
1356 | brcm_pcie_turn_off(pcie); |
1357 | /* |
1358 | * If brcm_phy_stop() returns an error, just dev_err(). If we |
1359 | * return the error it will cause the suspend to fail and this is a |
1360 | * forgivable offense that will probably be erased on resume. |
1361 | */ |
1362 | if (brcm_phy_stop(pcie)) |
1363 | dev_err(dev, "Could not stop phy for suspend\n" ); |
1364 | |
1365 | ret = reset_control_rearm(rstc: pcie->rescal); |
1366 | if (ret) { |
1367 | dev_err(dev, "Could not rearm rescal reset\n" ); |
1368 | return ret; |
1369 | } |
1370 | |
1371 | if (pcie->sr) { |
1372 | /* |
1373 | * Now turn off the regulators, but if at least one |
1374 | * downstream device is enabled as a wake-up source, do not |
1375 | * turn off regulators. |
1376 | */ |
1377 | pcie->ep_wakeup_capable = false; |
1378 | pci_walk_bus(top: bridge->bus, cb: pci_dev_may_wakeup, |
1379 | userdata: &pcie->ep_wakeup_capable); |
1380 | if (!pcie->ep_wakeup_capable) { |
1381 | ret = regulator_bulk_disable(num_consumers: pcie->sr->num_supplies, |
1382 | consumers: pcie->sr->supplies); |
1383 | if (ret) { |
1384 | dev_err(dev, "Could not turn off regulators\n" ); |
1385 | reset_control_reset(rstc: pcie->rescal); |
1386 | return ret; |
1387 | } |
1388 | } |
1389 | } |
1390 | clk_disable_unprepare(clk: pcie->clk); |
1391 | |
1392 | return 0; |
1393 | } |
1394 | |
1395 | static int brcm_pcie_resume_noirq(struct device *dev) |
1396 | { |
1397 | struct brcm_pcie *pcie = dev_get_drvdata(dev); |
1398 | void __iomem *base; |
1399 | u32 tmp; |
1400 | int ret; |
1401 | |
1402 | base = pcie->base; |
1403 | ret = clk_prepare_enable(clk: pcie->clk); |
1404 | if (ret) |
1405 | return ret; |
1406 | |
1407 | ret = reset_control_reset(rstc: pcie->rescal); |
1408 | if (ret) |
1409 | goto err_disable_clk; |
1410 | |
1411 | ret = brcm_phy_start(pcie); |
1412 | if (ret) |
1413 | goto err_reset; |
1414 | |
1415 | /* Take bridge out of reset so we can access the SERDES reg */ |
1416 | pcie->bridge_sw_init_set(pcie, 0); |
1417 | |
1418 | /* SERDES_IDDQ = 0 */ |
1419 | tmp = readl(addr: base + PCIE_MISC_HARD_PCIE_HARD_DEBUG); |
1420 | u32p_replace_bits(p: &tmp, val: 0, PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK); |
1421 | writel(val: tmp, addr: base + PCIE_MISC_HARD_PCIE_HARD_DEBUG); |
1422 | |
1423 | /* wait for serdes to be stable */ |
1424 | udelay(100); |
1425 | |
1426 | ret = brcm_pcie_setup(pcie); |
1427 | if (ret) |
1428 | goto err_reset; |
1429 | |
1430 | if (pcie->sr) { |
1431 | if (pcie->ep_wakeup_capable) { |
1432 | /* |
1433 | * We are resuming from a suspend. In the suspend we |
1434 | * did not disable the power supplies, so there is |
1435 | * no need to enable them (and falsely increase their |
1436 | * usage count). |
1437 | */ |
1438 | pcie->ep_wakeup_capable = false; |
1439 | } else { |
1440 | ret = regulator_bulk_enable(num_consumers: pcie->sr->num_supplies, |
1441 | consumers: pcie->sr->supplies); |
1442 | if (ret) { |
1443 | dev_err(dev, "Could not turn on regulators\n" ); |
1444 | goto err_reset; |
1445 | } |
1446 | } |
1447 | } |
1448 | |
1449 | ret = brcm_pcie_start_link(pcie); |
1450 | if (ret) |
1451 | goto err_regulator; |
1452 | |
1453 | if (pcie->msi) |
1454 | brcm_msi_set_regs(msi: pcie->msi); |
1455 | |
1456 | return 0; |
1457 | |
1458 | err_regulator: |
1459 | if (pcie->sr) |
1460 | regulator_bulk_disable(num_consumers: pcie->sr->num_supplies, consumers: pcie->sr->supplies); |
1461 | err_reset: |
1462 | reset_control_rearm(rstc: pcie->rescal); |
1463 | err_disable_clk: |
1464 | clk_disable_unprepare(clk: pcie->clk); |
1465 | return ret; |
1466 | } |
1467 | |
1468 | static void __brcm_pcie_remove(struct brcm_pcie *pcie) |
1469 | { |
1470 | brcm_msi_remove(pcie); |
1471 | brcm_pcie_turn_off(pcie); |
1472 | if (brcm_phy_stop(pcie)) |
1473 | dev_err(pcie->dev, "Could not stop phy\n" ); |
1474 | if (reset_control_rearm(rstc: pcie->rescal)) |
1475 | dev_err(pcie->dev, "Could not rearm rescal reset\n" ); |
1476 | clk_disable_unprepare(clk: pcie->clk); |
1477 | } |
1478 | |
1479 | static void brcm_pcie_remove(struct platform_device *pdev) |
1480 | { |
1481 | struct brcm_pcie *pcie = platform_get_drvdata(pdev); |
1482 | struct pci_host_bridge *bridge = pci_host_bridge_from_priv(priv: pcie); |
1483 | |
1484 | pci_stop_root_bus(bus: bridge->bus); |
1485 | pci_remove_root_bus(bus: bridge->bus); |
1486 | __brcm_pcie_remove(pcie); |
1487 | } |
1488 | |
1489 | static const int pcie_offsets[] = { |
1490 | [RGR1_SW_INIT_1] = 0x9210, |
1491 | [EXT_CFG_INDEX] = 0x9000, |
1492 | [EXT_CFG_DATA] = 0x9004, |
1493 | }; |
1494 | |
1495 | static const int pcie_offsets_bmips_7425[] = { |
1496 | [RGR1_SW_INIT_1] = 0x8010, |
1497 | [EXT_CFG_INDEX] = 0x8300, |
1498 | [EXT_CFG_DATA] = 0x8304, |
1499 | }; |
1500 | |
1501 | static const struct pcie_cfg_data generic_cfg = { |
1502 | .offsets = pcie_offsets, |
1503 | .type = GENERIC, |
1504 | .perst_set = brcm_pcie_perst_set_generic, |
1505 | .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, |
1506 | }; |
1507 | |
1508 | static const struct pcie_cfg_data bcm7425_cfg = { |
1509 | .offsets = pcie_offsets_bmips_7425, |
1510 | .type = BCM7425, |
1511 | .perst_set = brcm_pcie_perst_set_generic, |
1512 | .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, |
1513 | }; |
1514 | |
1515 | static const struct pcie_cfg_data bcm7435_cfg = { |
1516 | .offsets = pcie_offsets, |
1517 | .type = BCM7435, |
1518 | .perst_set = brcm_pcie_perst_set_generic, |
1519 | .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, |
1520 | }; |
1521 | |
1522 | static const struct pcie_cfg_data bcm4908_cfg = { |
1523 | .offsets = pcie_offsets, |
1524 | .type = BCM4908, |
1525 | .perst_set = brcm_pcie_perst_set_4908, |
1526 | .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, |
1527 | }; |
1528 | |
1529 | static const int pcie_offset_bcm7278[] = { |
1530 | [RGR1_SW_INIT_1] = 0xc010, |
1531 | [EXT_CFG_INDEX] = 0x9000, |
1532 | [EXT_CFG_DATA] = 0x9004, |
1533 | }; |
1534 | |
1535 | static const struct pcie_cfg_data bcm7278_cfg = { |
1536 | .offsets = pcie_offset_bcm7278, |
1537 | .type = BCM7278, |
1538 | .perst_set = brcm_pcie_perst_set_7278, |
1539 | .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_7278, |
1540 | }; |
1541 | |
1542 | static const struct pcie_cfg_data bcm2711_cfg = { |
1543 | .offsets = pcie_offsets, |
1544 | .type = BCM2711, |
1545 | .perst_set = brcm_pcie_perst_set_generic, |
1546 | .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, |
1547 | }; |
1548 | |
1549 | static const struct of_device_id brcm_pcie_match[] = { |
1550 | { .compatible = "brcm,bcm2711-pcie" , .data = &bcm2711_cfg }, |
1551 | { .compatible = "brcm,bcm4908-pcie" , .data = &bcm4908_cfg }, |
1552 | { .compatible = "brcm,bcm7211-pcie" , .data = &generic_cfg }, |
1553 | { .compatible = "brcm,bcm7278-pcie" , .data = &bcm7278_cfg }, |
1554 | { .compatible = "brcm,bcm7216-pcie" , .data = &bcm7278_cfg }, |
1555 | { .compatible = "brcm,bcm7445-pcie" , .data = &generic_cfg }, |
1556 | { .compatible = "brcm,bcm7435-pcie" , .data = &bcm7435_cfg }, |
1557 | { .compatible = "brcm,bcm7425-pcie" , .data = &bcm7425_cfg }, |
1558 | {}, |
1559 | }; |
1560 | |
1561 | static struct pci_ops brcm_pcie_ops = { |
1562 | .map_bus = brcm_pcie_map_bus, |
1563 | .read = pci_generic_config_read, |
1564 | .write = pci_generic_config_write, |
1565 | .add_bus = brcm_pcie_add_bus, |
1566 | .remove_bus = brcm_pcie_remove_bus, |
1567 | }; |
1568 | |
1569 | static struct pci_ops brcm7425_pcie_ops = { |
1570 | .map_bus = brcm7425_pcie_map_bus, |
1571 | .read = pci_generic_config_read32, |
1572 | .write = pci_generic_config_write32, |
1573 | .add_bus = brcm_pcie_add_bus, |
1574 | .remove_bus = brcm_pcie_remove_bus, |
1575 | }; |
1576 | |
1577 | static int brcm_pcie_probe(struct platform_device *pdev) |
1578 | { |
1579 | struct device_node *np = pdev->dev.of_node, *msi_np; |
1580 | struct pci_host_bridge *bridge; |
1581 | const struct pcie_cfg_data *data; |
1582 | struct brcm_pcie *pcie; |
1583 | int ret; |
1584 | |
1585 | bridge = devm_pci_alloc_host_bridge(dev: &pdev->dev, priv: sizeof(*pcie)); |
1586 | if (!bridge) |
1587 | return -ENOMEM; |
1588 | |
1589 | data = of_device_get_match_data(dev: &pdev->dev); |
1590 | if (!data) { |
1591 | pr_err("failed to look up compatible string\n" ); |
1592 | return -EINVAL; |
1593 | } |
1594 | |
1595 | pcie = pci_host_bridge_priv(bridge); |
1596 | pcie->dev = &pdev->dev; |
1597 | pcie->np = np; |
1598 | pcie->reg_offsets = data->offsets; |
1599 | pcie->type = data->type; |
1600 | pcie->perst_set = data->perst_set; |
1601 | pcie->bridge_sw_init_set = data->bridge_sw_init_set; |
1602 | |
1603 | pcie->base = devm_platform_ioremap_resource(pdev, index: 0); |
1604 | if (IS_ERR(ptr: pcie->base)) |
1605 | return PTR_ERR(ptr: pcie->base); |
1606 | |
1607 | pcie->clk = devm_clk_get_optional(dev: &pdev->dev, id: "sw_pcie" ); |
1608 | if (IS_ERR(ptr: pcie->clk)) |
1609 | return PTR_ERR(ptr: pcie->clk); |
1610 | |
1611 | ret = of_pci_get_max_link_speed(node: np); |
1612 | pcie->gen = (ret < 0) ? 0 : ret; |
1613 | |
1614 | pcie->ssc = of_property_read_bool(np, propname: "brcm,enable-ssc" ); |
1615 | |
1616 | ret = clk_prepare_enable(clk: pcie->clk); |
1617 | if (ret) { |
1618 | dev_err(&pdev->dev, "could not enable clock\n" ); |
1619 | return ret; |
1620 | } |
1621 | pcie->rescal = devm_reset_control_get_optional_shared(dev: &pdev->dev, id: "rescal" ); |
1622 | if (IS_ERR(ptr: pcie->rescal)) { |
1623 | clk_disable_unprepare(clk: pcie->clk); |
1624 | return PTR_ERR(ptr: pcie->rescal); |
1625 | } |
1626 | pcie->perst_reset = devm_reset_control_get_optional_exclusive(dev: &pdev->dev, id: "perst" ); |
1627 | if (IS_ERR(ptr: pcie->perst_reset)) { |
1628 | clk_disable_unprepare(clk: pcie->clk); |
1629 | return PTR_ERR(ptr: pcie->perst_reset); |
1630 | } |
1631 | |
1632 | ret = reset_control_reset(rstc: pcie->rescal); |
1633 | if (ret) |
1634 | dev_err(&pdev->dev, "failed to deassert 'rescal'\n" ); |
1635 | |
1636 | ret = brcm_phy_start(pcie); |
1637 | if (ret) { |
1638 | reset_control_rearm(rstc: pcie->rescal); |
1639 | clk_disable_unprepare(clk: pcie->clk); |
1640 | return ret; |
1641 | } |
1642 | |
1643 | ret = brcm_pcie_setup(pcie); |
1644 | if (ret) |
1645 | goto fail; |
1646 | |
1647 | pcie->hw_rev = readl(addr: pcie->base + PCIE_MISC_REVISION); |
1648 | if (pcie->type == BCM4908 && pcie->hw_rev >= BRCM_PCIE_HW_REV_3_20) { |
1649 | dev_err(pcie->dev, "hardware revision with unsupported PERST# setup\n" ); |
1650 | ret = -ENODEV; |
1651 | goto fail; |
1652 | } |
1653 | |
1654 | msi_np = of_parse_phandle(np: pcie->np, phandle_name: "msi-parent" , index: 0); |
1655 | if (pci_msi_enabled() && msi_np == pcie->np) { |
1656 | ret = brcm_pcie_enable_msi(pcie); |
1657 | if (ret) { |
1658 | dev_err(pcie->dev, "probe of internal MSI failed" ); |
1659 | goto fail; |
1660 | } |
1661 | } |
1662 | |
1663 | bridge->ops = pcie->type == BCM7425 ? &brcm7425_pcie_ops : &brcm_pcie_ops; |
1664 | bridge->sysdata = pcie; |
1665 | |
1666 | platform_set_drvdata(pdev, data: pcie); |
1667 | |
1668 | ret = pci_host_probe(bridge); |
1669 | if (!ret && !brcm_pcie_link_up(pcie)) |
1670 | ret = -ENODEV; |
1671 | |
1672 | if (ret) { |
1673 | brcm_pcie_remove(pdev); |
1674 | return ret; |
1675 | } |
1676 | |
1677 | return 0; |
1678 | |
1679 | fail: |
1680 | __brcm_pcie_remove(pcie); |
1681 | return ret; |
1682 | } |
1683 | |
1684 | MODULE_DEVICE_TABLE(of, brcm_pcie_match); |
1685 | |
1686 | static const struct dev_pm_ops brcm_pcie_pm_ops = { |
1687 | .suspend_noirq = brcm_pcie_suspend_noirq, |
1688 | .resume_noirq = brcm_pcie_resume_noirq, |
1689 | }; |
1690 | |
1691 | static struct platform_driver brcm_pcie_driver = { |
1692 | .probe = brcm_pcie_probe, |
1693 | .remove_new = brcm_pcie_remove, |
1694 | .driver = { |
1695 | .name = "brcm-pcie" , |
1696 | .of_match_table = brcm_pcie_match, |
1697 | .pm = &brcm_pcie_pm_ops, |
1698 | }, |
1699 | }; |
1700 | module_platform_driver(brcm_pcie_driver); |
1701 | |
1702 | MODULE_LICENSE("GPL" ); |
1703 | MODULE_DESCRIPTION("Broadcom STB PCIe RC driver" ); |
1704 | MODULE_AUTHOR("Broadcom" ); |
1705 | |