1 | // SPDX-License-Identifier: GPL-2.0+ |
2 | /* |
3 | * Rockchip AXI PCIe host controller driver |
4 | * |
5 | * Copyright (c) 2016 Rockchip, Inc. |
6 | * |
7 | * Author: Shawn Lin <shawn.lin@rock-chips.com> |
8 | * Wenrui Li <wenrui.li@rock-chips.com> |
9 | * |
10 | * Bits taken from Synopsys DesignWare Host controller driver and |
11 | * ARM PCI Host generic driver. |
12 | */ |
13 | |
14 | #include <linux/bitrev.h> |
15 | #include <linux/clk.h> |
16 | #include <linux/delay.h> |
17 | #include <linux/gpio/consumer.h> |
18 | #include <linux/init.h> |
19 | #include <linux/interrupt.h> |
20 | #include <linux/iopoll.h> |
21 | #include <linux/irq.h> |
22 | #include <linux/irqchip/chained_irq.h> |
23 | #include <linux/irqdomain.h> |
24 | #include <linux/kernel.h> |
25 | #include <linux/mfd/syscon.h> |
26 | #include <linux/module.h> |
27 | #include <linux/of.h> |
28 | #include <linux/of_pci.h> |
29 | #include <linux/pci.h> |
30 | #include <linux/pci_ids.h> |
31 | #include <linux/phy/phy.h> |
32 | #include <linux/platform_device.h> |
33 | #include <linux/reset.h> |
34 | #include <linux/regmap.h> |
35 | |
36 | #include "../pci.h" |
37 | #include "pcie-rockchip.h" |
38 | |
39 | static void rockchip_pcie_enable_bw_int(struct rockchip_pcie *rockchip) |
40 | { |
41 | u32 status; |
42 | |
43 | status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); |
44 | status |= (PCI_EXP_LNKCTL_LBMIE | PCI_EXP_LNKCTL_LABIE); |
45 | rockchip_pcie_write(rockchip, val: status, PCIE_RC_CONFIG_LCS); |
46 | } |
47 | |
48 | static void rockchip_pcie_clr_bw_int(struct rockchip_pcie *rockchip) |
49 | { |
50 | u32 status; |
51 | |
52 | status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); |
53 | status |= (PCI_EXP_LNKSTA_LBMS | PCI_EXP_LNKSTA_LABS) << 16; |
54 | rockchip_pcie_write(rockchip, val: status, PCIE_RC_CONFIG_LCS); |
55 | } |
56 | |
57 | static void rockchip_pcie_update_txcredit_mui(struct rockchip_pcie *rockchip) |
58 | { |
59 | u32 val; |
60 | |
61 | /* Update Tx credit maximum update interval */ |
62 | val = rockchip_pcie_read(rockchip, PCIE_CORE_TXCREDIT_CFG1); |
63 | val &= ~PCIE_CORE_TXCREDIT_CFG1_MUI_MASK; |
64 | val |= PCIE_CORE_TXCREDIT_CFG1_MUI_ENCODE(24000); /* ns */ |
65 | rockchip_pcie_write(rockchip, val, PCIE_CORE_TXCREDIT_CFG1); |
66 | } |
67 | |
68 | static int rockchip_pcie_valid_device(struct rockchip_pcie *rockchip, |
69 | struct pci_bus *bus, int dev) |
70 | { |
71 | /* |
72 | * Access only one slot on each root port. |
73 | * Do not read more than one device on the bus directly attached |
74 | * to RC's downstream side. |
75 | */ |
76 | if (pci_is_root_bus(pbus: bus) || pci_is_root_bus(pbus: bus->parent)) |
77 | return dev == 0; |
78 | |
79 | return 1; |
80 | } |
81 | |
82 | static u8 rockchip_pcie_lane_map(struct rockchip_pcie *rockchip) |
83 | { |
84 | u32 val; |
85 | u8 map; |
86 | |
87 | if (rockchip->legacy_phy) |
88 | return GENMASK(MAX_LANE_NUM - 1, 0); |
89 | |
90 | val = rockchip_pcie_read(rockchip, PCIE_CORE_LANE_MAP); |
91 | map = val & PCIE_CORE_LANE_MAP_MASK; |
92 | |
93 | /* The link may be using a reverse-indexed mapping. */ |
94 | if (val & PCIE_CORE_LANE_MAP_REVERSE) |
95 | map = bitrev8(map) >> 4; |
96 | |
97 | return map; |
98 | } |
99 | |
100 | static int rockchip_pcie_rd_own_conf(struct rockchip_pcie *rockchip, |
101 | int where, int size, u32 *val) |
102 | { |
103 | void __iomem *addr; |
104 | |
105 | addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + where; |
106 | |
107 | if (!IS_ALIGNED((uintptr_t)addr, size)) { |
108 | *val = 0; |
109 | return PCIBIOS_BAD_REGISTER_NUMBER; |
110 | } |
111 | |
112 | if (size == 4) { |
113 | *val = readl(addr); |
114 | } else if (size == 2) { |
115 | *val = readw(addr); |
116 | } else if (size == 1) { |
117 | *val = readb(addr); |
118 | } else { |
119 | *val = 0; |
120 | return PCIBIOS_BAD_REGISTER_NUMBER; |
121 | } |
122 | return PCIBIOS_SUCCESSFUL; |
123 | } |
124 | |
125 | static int rockchip_pcie_wr_own_conf(struct rockchip_pcie *rockchip, |
126 | int where, int size, u32 val) |
127 | { |
128 | u32 mask, tmp, offset; |
129 | void __iomem *addr; |
130 | |
131 | offset = where & ~0x3; |
132 | addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + offset; |
133 | |
134 | if (size == 4) { |
135 | writel(val, addr); |
136 | return PCIBIOS_SUCCESSFUL; |
137 | } |
138 | |
139 | mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8)); |
140 | |
141 | /* |
142 | * N.B. This read/modify/write isn't safe in general because it can |
143 | * corrupt RW1C bits in adjacent registers. But the hardware |
144 | * doesn't support smaller writes. |
145 | */ |
146 | tmp = readl(addr) & mask; |
147 | tmp |= val << ((where & 0x3) * 8); |
148 | writel(val: tmp, addr); |
149 | |
150 | return PCIBIOS_SUCCESSFUL; |
151 | } |
152 | |
153 | static int rockchip_pcie_rd_other_conf(struct rockchip_pcie *rockchip, |
154 | struct pci_bus *bus, u32 devfn, |
155 | int where, int size, u32 *val) |
156 | { |
157 | void __iomem *addr; |
158 | |
159 | addr = rockchip->reg_base + PCIE_ECAM_OFFSET(bus->number, devfn, where); |
160 | |
161 | if (!IS_ALIGNED((uintptr_t)addr, size)) { |
162 | *val = 0; |
163 | return PCIBIOS_BAD_REGISTER_NUMBER; |
164 | } |
165 | |
166 | if (pci_is_root_bus(pbus: bus->parent)) |
167 | rockchip_pcie_cfg_configuration_accesses(rockchip, |
168 | AXI_WRAPPER_TYPE0_CFG); |
169 | else |
170 | rockchip_pcie_cfg_configuration_accesses(rockchip, |
171 | AXI_WRAPPER_TYPE1_CFG); |
172 | |
173 | if (size == 4) { |
174 | *val = readl(addr); |
175 | } else if (size == 2) { |
176 | *val = readw(addr); |
177 | } else if (size == 1) { |
178 | *val = readb(addr); |
179 | } else { |
180 | *val = 0; |
181 | return PCIBIOS_BAD_REGISTER_NUMBER; |
182 | } |
183 | return PCIBIOS_SUCCESSFUL; |
184 | } |
185 | |
186 | static int rockchip_pcie_wr_other_conf(struct rockchip_pcie *rockchip, |
187 | struct pci_bus *bus, u32 devfn, |
188 | int where, int size, u32 val) |
189 | { |
190 | void __iomem *addr; |
191 | |
192 | addr = rockchip->reg_base + PCIE_ECAM_OFFSET(bus->number, devfn, where); |
193 | |
194 | if (!IS_ALIGNED((uintptr_t)addr, size)) |
195 | return PCIBIOS_BAD_REGISTER_NUMBER; |
196 | |
197 | if (pci_is_root_bus(pbus: bus->parent)) |
198 | rockchip_pcie_cfg_configuration_accesses(rockchip, |
199 | AXI_WRAPPER_TYPE0_CFG); |
200 | else |
201 | rockchip_pcie_cfg_configuration_accesses(rockchip, |
202 | AXI_WRAPPER_TYPE1_CFG); |
203 | |
204 | if (size == 4) |
205 | writel(val, addr); |
206 | else if (size == 2) |
207 | writew(val, addr); |
208 | else if (size == 1) |
209 | writeb(val, addr); |
210 | else |
211 | return PCIBIOS_BAD_REGISTER_NUMBER; |
212 | |
213 | return PCIBIOS_SUCCESSFUL; |
214 | } |
215 | |
216 | static int rockchip_pcie_rd_conf(struct pci_bus *bus, u32 devfn, int where, |
217 | int size, u32 *val) |
218 | { |
219 | struct rockchip_pcie *rockchip = bus->sysdata; |
220 | |
221 | if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn))) |
222 | return PCIBIOS_DEVICE_NOT_FOUND; |
223 | |
224 | if (pci_is_root_bus(pbus: bus)) |
225 | return rockchip_pcie_rd_own_conf(rockchip, where, size, val); |
226 | |
227 | return rockchip_pcie_rd_other_conf(rockchip, bus, devfn, where, size, |
228 | val); |
229 | } |
230 | |
231 | static int rockchip_pcie_wr_conf(struct pci_bus *bus, u32 devfn, |
232 | int where, int size, u32 val) |
233 | { |
234 | struct rockchip_pcie *rockchip = bus->sysdata; |
235 | |
236 | if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn))) |
237 | return PCIBIOS_DEVICE_NOT_FOUND; |
238 | |
239 | if (pci_is_root_bus(pbus: bus)) |
240 | return rockchip_pcie_wr_own_conf(rockchip, where, size, val); |
241 | |
242 | return rockchip_pcie_wr_other_conf(rockchip, bus, devfn, where, size, |
243 | val); |
244 | } |
245 | |
246 | static struct pci_ops rockchip_pcie_ops = { |
247 | .read = rockchip_pcie_rd_conf, |
248 | .write = rockchip_pcie_wr_conf, |
249 | }; |
250 | |
251 | static void rockchip_pcie_set_power_limit(struct rockchip_pcie *rockchip) |
252 | { |
253 | int curr; |
254 | u32 status, scale, power; |
255 | |
256 | if (IS_ERR(ptr: rockchip->vpcie3v3)) |
257 | return; |
258 | |
259 | /* |
260 | * Set RC's captured slot power limit and scale if |
261 | * vpcie3v3 available. The default values are both zero |
262 | * which means the software should set these two according |
263 | * to the actual power supply. |
264 | */ |
265 | curr = regulator_get_current_limit(regulator: rockchip->vpcie3v3); |
266 | if (curr <= 0) |
267 | return; |
268 | |
269 | scale = 3; /* 0.001x */ |
270 | curr = curr / 1000; /* convert to mA */ |
271 | power = (curr * 3300) / 1000; /* milliwatt */ |
272 | while (power > PCIE_RC_CONFIG_DCR_CSPL_LIMIT) { |
273 | if (!scale) { |
274 | dev_warn(rockchip->dev, "invalid power supply\n" ); |
275 | return; |
276 | } |
277 | scale--; |
278 | power = power / 10; |
279 | } |
280 | |
281 | status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_DCR); |
282 | status |= (power << PCIE_RC_CONFIG_DCR_CSPL_SHIFT) | |
283 | (scale << PCIE_RC_CONFIG_DCR_CPLS_SHIFT); |
284 | rockchip_pcie_write(rockchip, val: status, PCIE_RC_CONFIG_DCR); |
285 | } |
286 | |
287 | /** |
288 | * rockchip_pcie_host_init_port - Initialize hardware |
289 | * @rockchip: PCIe port information |
290 | */ |
291 | static int rockchip_pcie_host_init_port(struct rockchip_pcie *rockchip) |
292 | { |
293 | struct device *dev = rockchip->dev; |
294 | int err, i = MAX_LANE_NUM; |
295 | u32 status; |
296 | |
297 | gpiod_set_value_cansleep(desc: rockchip->ep_gpio, value: 0); |
298 | |
299 | err = rockchip_pcie_init_port(rockchip); |
300 | if (err) |
301 | return err; |
302 | |
303 | /* Fix the transmitted FTS count desired to exit from L0s. */ |
304 | status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL_PLC1); |
305 | status = (status & ~PCIE_CORE_CTRL_PLC1_FTS_MASK) | |
306 | (PCIE_CORE_CTRL_PLC1_FTS_CNT << PCIE_CORE_CTRL_PLC1_FTS_SHIFT); |
307 | rockchip_pcie_write(rockchip, val: status, PCIE_CORE_CTRL_PLC1); |
308 | |
309 | rockchip_pcie_set_power_limit(rockchip); |
310 | |
311 | /* Set RC's clock architecture as common clock */ |
312 | status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); |
313 | status |= PCI_EXP_LNKSTA_SLC << 16; |
314 | rockchip_pcie_write(rockchip, val: status, PCIE_RC_CONFIG_LCS); |
315 | |
316 | /* Set RC's RCB to 128 */ |
317 | status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); |
318 | status |= PCI_EXP_LNKCTL_RCB; |
319 | rockchip_pcie_write(rockchip, val: status, PCIE_RC_CONFIG_LCS); |
320 | |
321 | /* Enable Gen1 training */ |
322 | rockchip_pcie_write(rockchip, PCIE_CLIENT_LINK_TRAIN_ENABLE, |
323 | PCIE_CLIENT_CONFIG); |
324 | |
325 | gpiod_set_value_cansleep(desc: rockchip->ep_gpio, value: 1); |
326 | |
327 | /* 500ms timeout value should be enough for Gen1/2 training */ |
328 | err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_BASIC_STATUS1, |
329 | status, PCIE_LINK_UP(status), 20, |
330 | 500 * USEC_PER_MSEC); |
331 | if (err) { |
332 | dev_err(dev, "PCIe link training gen1 timeout!\n" ); |
333 | goto err_power_off_phy; |
334 | } |
335 | |
336 | if (rockchip->link_gen == 2) { |
337 | /* |
338 | * Enable retrain for gen2. This should be configured only after |
339 | * gen1 finished. |
340 | */ |
341 | status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); |
342 | status |= PCI_EXP_LNKCTL_RL; |
343 | rockchip_pcie_write(rockchip, val: status, PCIE_RC_CONFIG_LCS); |
344 | |
345 | err = readl_poll_timeout(rockchip->apb_base + PCIE_CORE_CTRL, |
346 | status, PCIE_LINK_IS_GEN2(status), 20, |
347 | 500 * USEC_PER_MSEC); |
348 | if (err) |
349 | dev_dbg(dev, "PCIe link training gen2 timeout, fall back to gen1!\n" ); |
350 | } |
351 | |
352 | /* Check the final link width from negotiated lane counter from MGMT */ |
353 | status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL); |
354 | status = 0x1 << ((status & PCIE_CORE_PL_CONF_LANE_MASK) >> |
355 | PCIE_CORE_PL_CONF_LANE_SHIFT); |
356 | dev_dbg(dev, "current link width is x%d\n" , status); |
357 | |
358 | /* Power off unused lane(s) */ |
359 | rockchip->lanes_map = rockchip_pcie_lane_map(rockchip); |
360 | for (i = 0; i < MAX_LANE_NUM; i++) { |
361 | if (!(rockchip->lanes_map & BIT(i))) { |
362 | dev_dbg(dev, "idling lane %d\n" , i); |
363 | phy_power_off(phy: rockchip->phys[i]); |
364 | } |
365 | } |
366 | |
367 | rockchip_pcie_write(rockchip, ROCKCHIP_VENDOR_ID, |
368 | PCIE_CORE_CONFIG_VENDOR); |
369 | rockchip_pcie_write(rockchip, |
370 | PCI_CLASS_BRIDGE_PCI_NORMAL << 8, |
371 | PCIE_RC_CONFIG_RID_CCR); |
372 | |
373 | /* Clear THP cap's next cap pointer to remove L1 substate cap */ |
374 | status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_THP_CAP); |
375 | status &= ~PCIE_RC_CONFIG_THP_CAP_NEXT_MASK; |
376 | rockchip_pcie_write(rockchip, val: status, PCIE_RC_CONFIG_THP_CAP); |
377 | |
378 | /* Clear L0s from RC's link cap */ |
379 | if (of_property_read_bool(np: dev->of_node, propname: "aspm-no-l0s" )) { |
380 | status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LINK_CAP); |
381 | status &= ~PCIE_RC_CONFIG_LINK_CAP_L0S; |
382 | rockchip_pcie_write(rockchip, val: status, PCIE_RC_CONFIG_LINK_CAP); |
383 | } |
384 | |
385 | status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_DCSR); |
386 | status &= ~PCIE_RC_CONFIG_DCSR_MPS_MASK; |
387 | status |= PCIE_RC_CONFIG_DCSR_MPS_256; |
388 | rockchip_pcie_write(rockchip, val: status, PCIE_RC_CONFIG_DCSR); |
389 | |
390 | return 0; |
391 | err_power_off_phy: |
392 | while (i--) |
393 | phy_power_off(phy: rockchip->phys[i]); |
394 | i = MAX_LANE_NUM; |
395 | while (i--) |
396 | phy_exit(phy: rockchip->phys[i]); |
397 | return err; |
398 | } |
399 | |
400 | static irqreturn_t rockchip_pcie_subsys_irq_handler(int irq, void *arg) |
401 | { |
402 | struct rockchip_pcie *rockchip = arg; |
403 | struct device *dev = rockchip->dev; |
404 | u32 reg; |
405 | u32 sub_reg; |
406 | |
407 | reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS); |
408 | if (reg & PCIE_CLIENT_INT_LOCAL) { |
409 | dev_dbg(dev, "local interrupt received\n" ); |
410 | sub_reg = rockchip_pcie_read(rockchip, PCIE_CORE_INT_STATUS); |
411 | if (sub_reg & PCIE_CORE_INT_PRFPE) |
412 | dev_dbg(dev, "parity error detected while reading from the PNP receive FIFO RAM\n" ); |
413 | |
414 | if (sub_reg & PCIE_CORE_INT_CRFPE) |
415 | dev_dbg(dev, "parity error detected while reading from the Completion Receive FIFO RAM\n" ); |
416 | |
417 | if (sub_reg & PCIE_CORE_INT_RRPE) |
418 | dev_dbg(dev, "parity error detected while reading from replay buffer RAM\n" ); |
419 | |
420 | if (sub_reg & PCIE_CORE_INT_PRFO) |
421 | dev_dbg(dev, "overflow occurred in the PNP receive FIFO\n" ); |
422 | |
423 | if (sub_reg & PCIE_CORE_INT_CRFO) |
424 | dev_dbg(dev, "overflow occurred in the completion receive FIFO\n" ); |
425 | |
426 | if (sub_reg & PCIE_CORE_INT_RT) |
427 | dev_dbg(dev, "replay timer timed out\n" ); |
428 | |
429 | if (sub_reg & PCIE_CORE_INT_RTR) |
430 | dev_dbg(dev, "replay timer rolled over after 4 transmissions of the same TLP\n" ); |
431 | |
432 | if (sub_reg & PCIE_CORE_INT_PE) |
433 | dev_dbg(dev, "phy error detected on receive side\n" ); |
434 | |
435 | if (sub_reg & PCIE_CORE_INT_MTR) |
436 | dev_dbg(dev, "malformed TLP received from the link\n" ); |
437 | |
438 | if (sub_reg & PCIE_CORE_INT_UCR) |
439 | dev_dbg(dev, "malformed TLP received from the link\n" ); |
440 | |
441 | if (sub_reg & PCIE_CORE_INT_FCE) |
442 | dev_dbg(dev, "an error was observed in the flow control advertisements from the other side\n" ); |
443 | |
444 | if (sub_reg & PCIE_CORE_INT_CT) |
445 | dev_dbg(dev, "a request timed out waiting for completion\n" ); |
446 | |
447 | if (sub_reg & PCIE_CORE_INT_UTC) |
448 | dev_dbg(dev, "unmapped TC error\n" ); |
449 | |
450 | if (sub_reg & PCIE_CORE_INT_MMVC) |
451 | dev_dbg(dev, "MSI mask register changes\n" ); |
452 | |
453 | rockchip_pcie_write(rockchip, val: sub_reg, PCIE_CORE_INT_STATUS); |
454 | } else if (reg & PCIE_CLIENT_INT_PHY) { |
455 | dev_dbg(dev, "phy link changes\n" ); |
456 | rockchip_pcie_update_txcredit_mui(rockchip); |
457 | rockchip_pcie_clr_bw_int(rockchip); |
458 | } |
459 | |
460 | rockchip_pcie_write(rockchip, val: reg & PCIE_CLIENT_INT_LOCAL, |
461 | PCIE_CLIENT_INT_STATUS); |
462 | |
463 | return IRQ_HANDLED; |
464 | } |
465 | |
466 | static irqreturn_t rockchip_pcie_client_irq_handler(int irq, void *arg) |
467 | { |
468 | struct rockchip_pcie *rockchip = arg; |
469 | struct device *dev = rockchip->dev; |
470 | u32 reg; |
471 | |
472 | reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS); |
473 | if (reg & PCIE_CLIENT_INT_LEGACY_DONE) |
474 | dev_dbg(dev, "legacy done interrupt received\n" ); |
475 | |
476 | if (reg & PCIE_CLIENT_INT_MSG) |
477 | dev_dbg(dev, "message done interrupt received\n" ); |
478 | |
479 | if (reg & PCIE_CLIENT_INT_HOT_RST) |
480 | dev_dbg(dev, "hot reset interrupt received\n" ); |
481 | |
482 | if (reg & PCIE_CLIENT_INT_DPA) |
483 | dev_dbg(dev, "dpa interrupt received\n" ); |
484 | |
485 | if (reg & PCIE_CLIENT_INT_FATAL_ERR) |
486 | dev_dbg(dev, "fatal error interrupt received\n" ); |
487 | |
488 | if (reg & PCIE_CLIENT_INT_NFATAL_ERR) |
489 | dev_dbg(dev, "no fatal error interrupt received\n" ); |
490 | |
491 | if (reg & PCIE_CLIENT_INT_CORR_ERR) |
492 | dev_dbg(dev, "correctable error interrupt received\n" ); |
493 | |
494 | if (reg & PCIE_CLIENT_INT_PHY) |
495 | dev_dbg(dev, "phy interrupt received\n" ); |
496 | |
497 | rockchip_pcie_write(rockchip, val: reg & (PCIE_CLIENT_INT_LEGACY_DONE | |
498 | PCIE_CLIENT_INT_MSG | PCIE_CLIENT_INT_HOT_RST | |
499 | PCIE_CLIENT_INT_DPA | PCIE_CLIENT_INT_FATAL_ERR | |
500 | PCIE_CLIENT_INT_NFATAL_ERR | |
501 | PCIE_CLIENT_INT_CORR_ERR | |
502 | PCIE_CLIENT_INT_PHY), |
503 | PCIE_CLIENT_INT_STATUS); |
504 | |
505 | return IRQ_HANDLED; |
506 | } |
507 | |
508 | static void rockchip_pcie_intx_handler(struct irq_desc *desc) |
509 | { |
510 | struct irq_chip *chip = irq_desc_get_chip(desc); |
511 | struct rockchip_pcie *rockchip = irq_desc_get_handler_data(desc); |
512 | struct device *dev = rockchip->dev; |
513 | u32 reg; |
514 | u32 hwirq; |
515 | int ret; |
516 | |
517 | chained_irq_enter(chip, desc); |
518 | |
519 | reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS); |
520 | reg = (reg & PCIE_CLIENT_INTR_MASK) >> PCIE_CLIENT_INTR_SHIFT; |
521 | |
522 | while (reg) { |
523 | hwirq = ffs(reg) - 1; |
524 | reg &= ~BIT(hwirq); |
525 | |
526 | ret = generic_handle_domain_irq(domain: rockchip->irq_domain, hwirq); |
527 | if (ret) |
528 | dev_err(dev, "unexpected IRQ, INT%d\n" , hwirq); |
529 | } |
530 | |
531 | chained_irq_exit(chip, desc); |
532 | } |
533 | |
534 | static int rockchip_pcie_setup_irq(struct rockchip_pcie *rockchip) |
535 | { |
536 | int irq, err; |
537 | struct device *dev = rockchip->dev; |
538 | struct platform_device *pdev = to_platform_device(dev); |
539 | |
540 | irq = platform_get_irq_byname(pdev, "sys" ); |
541 | if (irq < 0) |
542 | return irq; |
543 | |
544 | err = devm_request_irq(dev, irq, handler: rockchip_pcie_subsys_irq_handler, |
545 | IRQF_SHARED, devname: "pcie-sys" , dev_id: rockchip); |
546 | if (err) { |
547 | dev_err(dev, "failed to request PCIe subsystem IRQ\n" ); |
548 | return err; |
549 | } |
550 | |
551 | irq = platform_get_irq_byname(pdev, "legacy" ); |
552 | if (irq < 0) |
553 | return irq; |
554 | |
555 | irq_set_chained_handler_and_data(irq, |
556 | handle: rockchip_pcie_intx_handler, |
557 | data: rockchip); |
558 | |
559 | irq = platform_get_irq_byname(pdev, "client" ); |
560 | if (irq < 0) |
561 | return irq; |
562 | |
563 | err = devm_request_irq(dev, irq, handler: rockchip_pcie_client_irq_handler, |
564 | IRQF_SHARED, devname: "pcie-client" , dev_id: rockchip); |
565 | if (err) { |
566 | dev_err(dev, "failed to request PCIe client IRQ\n" ); |
567 | return err; |
568 | } |
569 | |
570 | return 0; |
571 | } |
572 | |
573 | /** |
574 | * rockchip_pcie_parse_host_dt - Parse Device Tree |
575 | * @rockchip: PCIe port information |
576 | * |
577 | * Return: '0' on success and error value on failure |
578 | */ |
579 | static int rockchip_pcie_parse_host_dt(struct rockchip_pcie *rockchip) |
580 | { |
581 | struct device *dev = rockchip->dev; |
582 | int err; |
583 | |
584 | err = rockchip_pcie_parse_dt(rockchip); |
585 | if (err) |
586 | return err; |
587 | |
588 | rockchip->vpcie12v = devm_regulator_get_optional(dev, id: "vpcie12v" ); |
589 | if (IS_ERR(ptr: rockchip->vpcie12v)) { |
590 | if (PTR_ERR(ptr: rockchip->vpcie12v) != -ENODEV) |
591 | return PTR_ERR(ptr: rockchip->vpcie12v); |
592 | dev_info(dev, "no vpcie12v regulator found\n" ); |
593 | } |
594 | |
595 | rockchip->vpcie3v3 = devm_regulator_get_optional(dev, id: "vpcie3v3" ); |
596 | if (IS_ERR(ptr: rockchip->vpcie3v3)) { |
597 | if (PTR_ERR(ptr: rockchip->vpcie3v3) != -ENODEV) |
598 | return PTR_ERR(ptr: rockchip->vpcie3v3); |
599 | dev_info(dev, "no vpcie3v3 regulator found\n" ); |
600 | } |
601 | |
602 | rockchip->vpcie1v8 = devm_regulator_get(dev, id: "vpcie1v8" ); |
603 | if (IS_ERR(ptr: rockchip->vpcie1v8)) |
604 | return PTR_ERR(ptr: rockchip->vpcie1v8); |
605 | |
606 | rockchip->vpcie0v9 = devm_regulator_get(dev, id: "vpcie0v9" ); |
607 | if (IS_ERR(ptr: rockchip->vpcie0v9)) |
608 | return PTR_ERR(ptr: rockchip->vpcie0v9); |
609 | |
610 | return 0; |
611 | } |
612 | |
613 | static int rockchip_pcie_set_vpcie(struct rockchip_pcie *rockchip) |
614 | { |
615 | struct device *dev = rockchip->dev; |
616 | int err; |
617 | |
618 | if (!IS_ERR(ptr: rockchip->vpcie12v)) { |
619 | err = regulator_enable(regulator: rockchip->vpcie12v); |
620 | if (err) { |
621 | dev_err(dev, "fail to enable vpcie12v regulator\n" ); |
622 | goto err_out; |
623 | } |
624 | } |
625 | |
626 | if (!IS_ERR(ptr: rockchip->vpcie3v3)) { |
627 | err = regulator_enable(regulator: rockchip->vpcie3v3); |
628 | if (err) { |
629 | dev_err(dev, "fail to enable vpcie3v3 regulator\n" ); |
630 | goto err_disable_12v; |
631 | } |
632 | } |
633 | |
634 | err = regulator_enable(regulator: rockchip->vpcie1v8); |
635 | if (err) { |
636 | dev_err(dev, "fail to enable vpcie1v8 regulator\n" ); |
637 | goto err_disable_3v3; |
638 | } |
639 | |
640 | err = regulator_enable(regulator: rockchip->vpcie0v9); |
641 | if (err) { |
642 | dev_err(dev, "fail to enable vpcie0v9 regulator\n" ); |
643 | goto err_disable_1v8; |
644 | } |
645 | |
646 | return 0; |
647 | |
648 | err_disable_1v8: |
649 | regulator_disable(regulator: rockchip->vpcie1v8); |
650 | err_disable_3v3: |
651 | if (!IS_ERR(ptr: rockchip->vpcie3v3)) |
652 | regulator_disable(regulator: rockchip->vpcie3v3); |
653 | err_disable_12v: |
654 | if (!IS_ERR(ptr: rockchip->vpcie12v)) |
655 | regulator_disable(regulator: rockchip->vpcie12v); |
656 | err_out: |
657 | return err; |
658 | } |
659 | |
660 | static void rockchip_pcie_enable_interrupts(struct rockchip_pcie *rockchip) |
661 | { |
662 | rockchip_pcie_write(rockchip, val: (PCIE_CLIENT_INT_CLI << 16) & |
663 | (~PCIE_CLIENT_INT_CLI), PCIE_CLIENT_INT_MASK); |
664 | rockchip_pcie_write(rockchip, val: (u32)(~PCIE_CORE_INT), |
665 | PCIE_CORE_INT_MASK); |
666 | |
667 | rockchip_pcie_enable_bw_int(rockchip); |
668 | } |
669 | |
670 | static int rockchip_pcie_intx_map(struct irq_domain *domain, unsigned int irq, |
671 | irq_hw_number_t hwirq) |
672 | { |
673 | irq_set_chip_and_handler(irq, chip: &dummy_irq_chip, handle: handle_simple_irq); |
674 | irq_set_chip_data(irq, data: domain->host_data); |
675 | |
676 | return 0; |
677 | } |
678 | |
679 | static const struct irq_domain_ops intx_domain_ops = { |
680 | .map = rockchip_pcie_intx_map, |
681 | }; |
682 | |
683 | static int rockchip_pcie_init_irq_domain(struct rockchip_pcie *rockchip) |
684 | { |
685 | struct device *dev = rockchip->dev; |
686 | struct device_node *intc = of_get_next_child(node: dev->of_node, NULL); |
687 | |
688 | if (!intc) { |
689 | dev_err(dev, "missing child interrupt-controller node\n" ); |
690 | return -EINVAL; |
691 | } |
692 | |
693 | rockchip->irq_domain = irq_domain_add_linear(of_node: intc, PCI_NUM_INTX, |
694 | ops: &intx_domain_ops, host_data: rockchip); |
695 | of_node_put(node: intc); |
696 | if (!rockchip->irq_domain) { |
697 | dev_err(dev, "failed to get a INTx IRQ domain\n" ); |
698 | return -EINVAL; |
699 | } |
700 | |
701 | return 0; |
702 | } |
703 | |
704 | static int rockchip_pcie_prog_ob_atu(struct rockchip_pcie *rockchip, |
705 | int region_no, int type, u8 num_pass_bits, |
706 | u32 lower_addr, u32 upper_addr) |
707 | { |
708 | u32 ob_addr_0; |
709 | u32 ob_addr_1; |
710 | u32 ob_desc_0; |
711 | u32 aw_offset; |
712 | |
713 | if (region_no >= MAX_AXI_WRAPPER_REGION_NUM) |
714 | return -EINVAL; |
715 | if (num_pass_bits + 1 < 8) |
716 | return -EINVAL; |
717 | if (num_pass_bits > 63) |
718 | return -EINVAL; |
719 | if (region_no == 0) { |
720 | if (AXI_REGION_0_SIZE < (2ULL << num_pass_bits)) |
721 | return -EINVAL; |
722 | } |
723 | if (region_no != 0) { |
724 | if (AXI_REGION_SIZE < (2ULL << num_pass_bits)) |
725 | return -EINVAL; |
726 | } |
727 | |
728 | aw_offset = (region_no << OB_REG_SIZE_SHIFT); |
729 | |
730 | ob_addr_0 = num_pass_bits & PCIE_CORE_OB_REGION_ADDR0_NUM_BITS; |
731 | ob_addr_0 |= lower_addr & PCIE_CORE_OB_REGION_ADDR0_LO_ADDR; |
732 | ob_addr_1 = upper_addr; |
733 | ob_desc_0 = (1 << 23 | type); |
734 | |
735 | rockchip_pcie_write(rockchip, val: ob_addr_0, |
736 | PCIE_CORE_OB_REGION_ADDR0 + aw_offset); |
737 | rockchip_pcie_write(rockchip, val: ob_addr_1, |
738 | PCIE_CORE_OB_REGION_ADDR1 + aw_offset); |
739 | rockchip_pcie_write(rockchip, val: ob_desc_0, |
740 | PCIE_CORE_OB_REGION_DESC0 + aw_offset); |
741 | rockchip_pcie_write(rockchip, val: 0, |
742 | PCIE_CORE_OB_REGION_DESC1 + aw_offset); |
743 | |
744 | return 0; |
745 | } |
746 | |
747 | static int rockchip_pcie_prog_ib_atu(struct rockchip_pcie *rockchip, |
748 | int region_no, u8 num_pass_bits, |
749 | u32 lower_addr, u32 upper_addr) |
750 | { |
751 | u32 ib_addr_0; |
752 | u32 ib_addr_1; |
753 | u32 aw_offset; |
754 | |
755 | if (region_no > MAX_AXI_IB_ROOTPORT_REGION_NUM) |
756 | return -EINVAL; |
757 | if (num_pass_bits + 1 < MIN_AXI_ADDR_BITS_PASSED) |
758 | return -EINVAL; |
759 | if (num_pass_bits > 63) |
760 | return -EINVAL; |
761 | |
762 | aw_offset = (region_no << IB_ROOT_PORT_REG_SIZE_SHIFT); |
763 | |
764 | ib_addr_0 = num_pass_bits & PCIE_CORE_IB_REGION_ADDR0_NUM_BITS; |
765 | ib_addr_0 |= (lower_addr << 8) & PCIE_CORE_IB_REGION_ADDR0_LO_ADDR; |
766 | ib_addr_1 = upper_addr; |
767 | |
768 | rockchip_pcie_write(rockchip, val: ib_addr_0, PCIE_RP_IB_ADDR0 + aw_offset); |
769 | rockchip_pcie_write(rockchip, val: ib_addr_1, PCIE_RP_IB_ADDR1 + aw_offset); |
770 | |
771 | return 0; |
772 | } |
773 | |
774 | static int rockchip_pcie_cfg_atu(struct rockchip_pcie *rockchip) |
775 | { |
776 | struct device *dev = rockchip->dev; |
777 | struct pci_host_bridge *bridge = pci_host_bridge_from_priv(priv: rockchip); |
778 | struct resource_entry *entry; |
779 | u64 pci_addr, size; |
780 | int offset; |
781 | int err; |
782 | int reg_no; |
783 | |
784 | rockchip_pcie_cfg_configuration_accesses(rockchip, |
785 | AXI_WRAPPER_TYPE0_CFG); |
786 | entry = resource_list_first_type(list: &bridge->windows, IORESOURCE_MEM); |
787 | if (!entry) |
788 | return -ENODEV; |
789 | |
790 | size = resource_size(res: entry->res); |
791 | pci_addr = entry->res->start - entry->offset; |
792 | rockchip->msg_bus_addr = pci_addr; |
793 | |
794 | for (reg_no = 0; reg_no < (size >> 20); reg_no++) { |
795 | err = rockchip_pcie_prog_ob_atu(rockchip, region_no: reg_no + 1, |
796 | AXI_WRAPPER_MEM_WRITE, |
797 | num_pass_bits: 20 - 1, |
798 | lower_addr: pci_addr + (reg_no << 20), |
799 | upper_addr: 0); |
800 | if (err) { |
801 | dev_err(dev, "program RC mem outbound ATU failed\n" ); |
802 | return err; |
803 | } |
804 | } |
805 | |
806 | err = rockchip_pcie_prog_ib_atu(rockchip, region_no: 2, num_pass_bits: 32 - 1, lower_addr: 0x0, upper_addr: 0); |
807 | if (err) { |
808 | dev_err(dev, "program RC mem inbound ATU failed\n" ); |
809 | return err; |
810 | } |
811 | |
812 | entry = resource_list_first_type(list: &bridge->windows, IORESOURCE_IO); |
813 | if (!entry) |
814 | return -ENODEV; |
815 | |
816 | /* store the register number offset to program RC io outbound ATU */ |
817 | offset = size >> 20; |
818 | |
819 | size = resource_size(res: entry->res); |
820 | pci_addr = entry->res->start - entry->offset; |
821 | |
822 | for (reg_no = 0; reg_no < (size >> 20); reg_no++) { |
823 | err = rockchip_pcie_prog_ob_atu(rockchip, |
824 | region_no: reg_no + 1 + offset, |
825 | AXI_WRAPPER_IO_WRITE, |
826 | num_pass_bits: 20 - 1, |
827 | lower_addr: pci_addr + (reg_no << 20), |
828 | upper_addr: 0); |
829 | if (err) { |
830 | dev_err(dev, "program RC io outbound ATU failed\n" ); |
831 | return err; |
832 | } |
833 | } |
834 | |
835 | /* assign message regions */ |
836 | rockchip_pcie_prog_ob_atu(rockchip, region_no: reg_no + 1 + offset, |
837 | AXI_WRAPPER_NOR_MSG, |
838 | num_pass_bits: 20 - 1, lower_addr: 0, upper_addr: 0); |
839 | |
840 | rockchip->msg_bus_addr += ((reg_no + offset) << 20); |
841 | return err; |
842 | } |
843 | |
844 | static int rockchip_pcie_wait_l2(struct rockchip_pcie *rockchip) |
845 | { |
846 | u32 value; |
847 | int err; |
848 | |
849 | /* send PME_TURN_OFF message */ |
850 | writel(val: 0x0, addr: rockchip->msg_region + PCIE_RC_SEND_PME_OFF); |
851 | |
852 | /* read LTSSM and wait for falling into L2 link state */ |
853 | err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_DEBUG_OUT_0, |
854 | value, PCIE_LINK_IS_L2(value), 20, |
855 | jiffies_to_usecs(5 * HZ)); |
856 | if (err) { |
857 | dev_err(rockchip->dev, "PCIe link enter L2 timeout!\n" ); |
858 | return err; |
859 | } |
860 | |
861 | return 0; |
862 | } |
863 | |
864 | static int rockchip_pcie_suspend_noirq(struct device *dev) |
865 | { |
866 | struct rockchip_pcie *rockchip = dev_get_drvdata(dev); |
867 | int ret; |
868 | |
869 | /* disable core and cli int since we don't need to ack PME_ACK */ |
870 | rockchip_pcie_write(rockchip, val: (PCIE_CLIENT_INT_CLI << 16) | |
871 | PCIE_CLIENT_INT_CLI, PCIE_CLIENT_INT_MASK); |
872 | rockchip_pcie_write(rockchip, val: (u32)PCIE_CORE_INT, PCIE_CORE_INT_MASK); |
873 | |
874 | ret = rockchip_pcie_wait_l2(rockchip); |
875 | if (ret) { |
876 | rockchip_pcie_enable_interrupts(rockchip); |
877 | return ret; |
878 | } |
879 | |
880 | rockchip_pcie_deinit_phys(rockchip); |
881 | |
882 | rockchip_pcie_disable_clocks(data: rockchip); |
883 | |
884 | regulator_disable(regulator: rockchip->vpcie0v9); |
885 | |
886 | return ret; |
887 | } |
888 | |
889 | static int rockchip_pcie_resume_noirq(struct device *dev) |
890 | { |
891 | struct rockchip_pcie *rockchip = dev_get_drvdata(dev); |
892 | int err; |
893 | |
894 | err = regulator_enable(regulator: rockchip->vpcie0v9); |
895 | if (err) { |
896 | dev_err(dev, "fail to enable vpcie0v9 regulator\n" ); |
897 | return err; |
898 | } |
899 | |
900 | err = rockchip_pcie_enable_clocks(rockchip); |
901 | if (err) |
902 | goto err_disable_0v9; |
903 | |
904 | err = rockchip_pcie_host_init_port(rockchip); |
905 | if (err) |
906 | goto err_pcie_resume; |
907 | |
908 | err = rockchip_pcie_cfg_atu(rockchip); |
909 | if (err) |
910 | goto err_err_deinit_port; |
911 | |
912 | /* Need this to enter L1 again */ |
913 | rockchip_pcie_update_txcredit_mui(rockchip); |
914 | rockchip_pcie_enable_interrupts(rockchip); |
915 | |
916 | return 0; |
917 | |
918 | err_err_deinit_port: |
919 | rockchip_pcie_deinit_phys(rockchip); |
920 | err_pcie_resume: |
921 | rockchip_pcie_disable_clocks(data: rockchip); |
922 | err_disable_0v9: |
923 | regulator_disable(regulator: rockchip->vpcie0v9); |
924 | return err; |
925 | } |
926 | |
927 | static int rockchip_pcie_probe(struct platform_device *pdev) |
928 | { |
929 | struct rockchip_pcie *rockchip; |
930 | struct device *dev = &pdev->dev; |
931 | struct pci_host_bridge *bridge; |
932 | int err; |
933 | |
934 | if (!dev->of_node) |
935 | return -ENODEV; |
936 | |
937 | bridge = devm_pci_alloc_host_bridge(dev, priv: sizeof(*rockchip)); |
938 | if (!bridge) |
939 | return -ENOMEM; |
940 | |
941 | rockchip = pci_host_bridge_priv(bridge); |
942 | |
943 | platform_set_drvdata(pdev, data: rockchip); |
944 | rockchip->dev = dev; |
945 | rockchip->is_rc = true; |
946 | |
947 | err = rockchip_pcie_parse_host_dt(rockchip); |
948 | if (err) |
949 | return err; |
950 | |
951 | err = rockchip_pcie_enable_clocks(rockchip); |
952 | if (err) |
953 | return err; |
954 | |
955 | err = rockchip_pcie_set_vpcie(rockchip); |
956 | if (err) { |
957 | dev_err(dev, "failed to set vpcie regulator\n" ); |
958 | goto err_set_vpcie; |
959 | } |
960 | |
961 | err = rockchip_pcie_host_init_port(rockchip); |
962 | if (err) |
963 | goto err_vpcie; |
964 | |
965 | err = rockchip_pcie_init_irq_domain(rockchip); |
966 | if (err < 0) |
967 | goto err_deinit_port; |
968 | |
969 | err = rockchip_pcie_cfg_atu(rockchip); |
970 | if (err) |
971 | goto err_remove_irq_domain; |
972 | |
973 | rockchip->msg_region = devm_ioremap(dev, offset: rockchip->msg_bus_addr, SZ_1M); |
974 | if (!rockchip->msg_region) { |
975 | err = -ENOMEM; |
976 | goto err_remove_irq_domain; |
977 | } |
978 | |
979 | bridge->sysdata = rockchip; |
980 | bridge->ops = &rockchip_pcie_ops; |
981 | |
982 | err = rockchip_pcie_setup_irq(rockchip); |
983 | if (err) |
984 | goto err_remove_irq_domain; |
985 | |
986 | rockchip_pcie_enable_interrupts(rockchip); |
987 | |
988 | err = pci_host_probe(bridge); |
989 | if (err < 0) |
990 | goto err_remove_irq_domain; |
991 | |
992 | return 0; |
993 | |
994 | err_remove_irq_domain: |
995 | irq_domain_remove(host: rockchip->irq_domain); |
996 | err_deinit_port: |
997 | rockchip_pcie_deinit_phys(rockchip); |
998 | err_vpcie: |
999 | if (!IS_ERR(ptr: rockchip->vpcie12v)) |
1000 | regulator_disable(regulator: rockchip->vpcie12v); |
1001 | if (!IS_ERR(ptr: rockchip->vpcie3v3)) |
1002 | regulator_disable(regulator: rockchip->vpcie3v3); |
1003 | regulator_disable(regulator: rockchip->vpcie1v8); |
1004 | regulator_disable(regulator: rockchip->vpcie0v9); |
1005 | err_set_vpcie: |
1006 | rockchip_pcie_disable_clocks(data: rockchip); |
1007 | return err; |
1008 | } |
1009 | |
1010 | static void rockchip_pcie_remove(struct platform_device *pdev) |
1011 | { |
1012 | struct device *dev = &pdev->dev; |
1013 | struct rockchip_pcie *rockchip = dev_get_drvdata(dev); |
1014 | struct pci_host_bridge *bridge = pci_host_bridge_from_priv(priv: rockchip); |
1015 | |
1016 | pci_stop_root_bus(bus: bridge->bus); |
1017 | pci_remove_root_bus(bus: bridge->bus); |
1018 | irq_domain_remove(host: rockchip->irq_domain); |
1019 | |
1020 | rockchip_pcie_deinit_phys(rockchip); |
1021 | |
1022 | rockchip_pcie_disable_clocks(data: rockchip); |
1023 | |
1024 | if (!IS_ERR(ptr: rockchip->vpcie12v)) |
1025 | regulator_disable(regulator: rockchip->vpcie12v); |
1026 | if (!IS_ERR(ptr: rockchip->vpcie3v3)) |
1027 | regulator_disable(regulator: rockchip->vpcie3v3); |
1028 | regulator_disable(regulator: rockchip->vpcie1v8); |
1029 | regulator_disable(regulator: rockchip->vpcie0v9); |
1030 | } |
1031 | |
1032 | static const struct dev_pm_ops rockchip_pcie_pm_ops = { |
1033 | NOIRQ_SYSTEM_SLEEP_PM_OPS(rockchip_pcie_suspend_noirq, |
1034 | rockchip_pcie_resume_noirq) |
1035 | }; |
1036 | |
1037 | static const struct of_device_id rockchip_pcie_of_match[] = { |
1038 | { .compatible = "rockchip,rk3399-pcie" , }, |
1039 | {} |
1040 | }; |
1041 | MODULE_DEVICE_TABLE(of, rockchip_pcie_of_match); |
1042 | |
1043 | static struct platform_driver rockchip_pcie_driver = { |
1044 | .driver = { |
1045 | .name = "rockchip-pcie" , |
1046 | .of_match_table = rockchip_pcie_of_match, |
1047 | .pm = &rockchip_pcie_pm_ops, |
1048 | }, |
1049 | .probe = rockchip_pcie_probe, |
1050 | .remove_new = rockchip_pcie_remove, |
1051 | }; |
1052 | module_platform_driver(rockchip_pcie_driver); |
1053 | |
1054 | MODULE_AUTHOR("Rockchip Inc" ); |
1055 | MODULE_DESCRIPTION("Rockchip AXI PCIe driver" ); |
1056 | MODULE_LICENSE("GPL v2" ); |
1057 | |