1// SPDX-License-Identifier: GPL-2.0+
2/*
3 * PCI Express PCI Hot Plug Driver
4 *
5 * Copyright (C) 1995,2001 Compaq Computer Corporation
6 * Copyright (C) 2001 Greg Kroah-Hartman (greg@kroah.com)
7 * Copyright (C) 2001 IBM Corp.
8 * Copyright (C) 2003-2004 Intel Corporation
9 *
10 * All rights reserved.
11 *
12 * Send feedback to <greg@kroah.com>,<kristen.c.accardi@intel.com>
13 */
14
15#define dev_fmt(fmt) "pciehp: " fmt
16
17#include <linux/bitfield.h>
18#include <linux/dmi.h>
19#include <linux/kernel.h>
20#include <linux/types.h>
21#include <linux/jiffies.h>
22#include <linux/kthread.h>
23#include <linux/pci.h>
24#include <linux/pm_runtime.h>
25#include <linux/interrupt.h>
26#include <linux/slab.h>
27
28#include "../pci.h"
29#include "pciehp.h"
30
31static const struct dmi_system_id inband_presence_disabled_dmi_table[] = {
32 /*
33 * Match all Dell systems, as some Dell systems have inband
34 * presence disabled on NVMe slots (but don't support the bit to
35 * report it). Setting inband presence disabled should have no
36 * negative effect, except on broken hotplug slots that never
37 * assert presence detect--and those will still work, they will
38 * just have a bit of extra delay before being probed.
39 */
40 {
41 .ident = "Dell System",
42 .matches = {
43 DMI_MATCH(DMI_OEM_STRING, "Dell System"),
44 },
45 },
46 {}
47};
48
49static inline struct pci_dev *ctrl_dev(struct controller *ctrl)
50{
51 return ctrl->pcie->port;
52}
53
54static irqreturn_t pciehp_isr(int irq, void *dev_id);
55static irqreturn_t pciehp_ist(int irq, void *dev_id);
56static int pciehp_poll(void *data);
57
58static inline int pciehp_request_irq(struct controller *ctrl)
59{
60 int retval, irq = ctrl->pcie->irq;
61
62 if (pciehp_poll_mode) {
63 ctrl->poll_thread = kthread_run(&pciehp_poll, ctrl,
64 "pciehp_poll-%s",
65 slot_name(ctrl));
66 return PTR_ERR_OR_ZERO(ptr: ctrl->poll_thread);
67 }
68
69 /* Installs the interrupt handler */
70 retval = request_threaded_irq(irq, handler: pciehp_isr, thread_fn: pciehp_ist,
71 IRQF_SHARED, name: "pciehp", dev: ctrl);
72 if (retval)
73 ctrl_err(ctrl, "Cannot get irq %d for the hotplug controller\n",
74 irq);
75 return retval;
76}
77
78static inline void pciehp_free_irq(struct controller *ctrl)
79{
80 if (pciehp_poll_mode)
81 kthread_stop(k: ctrl->poll_thread);
82 else
83 free_irq(ctrl->pcie->irq, ctrl);
84}
85
86static int pcie_poll_cmd(struct controller *ctrl, int timeout)
87{
88 struct pci_dev *pdev = ctrl_dev(ctrl);
89 u16 slot_status;
90
91 do {
92 pcie_capability_read_word(dev: pdev, PCI_EXP_SLTSTA, val: &slot_status);
93 if (PCI_POSSIBLE_ERROR(slot_status)) {
94 ctrl_info(ctrl, "%s: no response from device\n",
95 __func__);
96 return 0;
97 }
98
99 if (slot_status & PCI_EXP_SLTSTA_CC) {
100 pcie_capability_write_word(dev: pdev, PCI_EXP_SLTSTA,
101 PCI_EXP_SLTSTA_CC);
102 ctrl->cmd_busy = 0;
103 smp_mb();
104 return 1;
105 }
106 msleep(msecs: 10);
107 timeout -= 10;
108 } while (timeout >= 0);
109 return 0; /* timeout */
110}
111
112static void pcie_wait_cmd(struct controller *ctrl)
113{
114 unsigned int msecs = pciehp_poll_mode ? 2500 : 1000;
115 unsigned long duration = msecs_to_jiffies(m: msecs);
116 unsigned long cmd_timeout = ctrl->cmd_started + duration;
117 unsigned long now, timeout;
118 int rc;
119
120 /*
121 * If the controller does not generate notifications for command
122 * completions, we never need to wait between writes.
123 */
124 if (NO_CMD_CMPL(ctrl))
125 return;
126
127 if (!ctrl->cmd_busy)
128 return;
129
130 /*
131 * Even if the command has already timed out, we want to call
132 * pcie_poll_cmd() so it can clear PCI_EXP_SLTSTA_CC.
133 */
134 now = jiffies;
135 if (time_before_eq(cmd_timeout, now))
136 timeout = 1;
137 else
138 timeout = cmd_timeout - now;
139
140 if (ctrl->slot_ctrl & PCI_EXP_SLTCTL_HPIE &&
141 ctrl->slot_ctrl & PCI_EXP_SLTCTL_CCIE)
142 rc = wait_event_timeout(ctrl->queue, !ctrl->cmd_busy, timeout);
143 else
144 rc = pcie_poll_cmd(ctrl, timeout: jiffies_to_msecs(j: timeout));
145
146 if (!rc)
147 ctrl_info(ctrl, "Timeout on hotplug command %#06x (issued %u msec ago)\n",
148 ctrl->slot_ctrl,
149 jiffies_to_msecs(jiffies - ctrl->cmd_started));
150}
151
152#define CC_ERRATUM_MASK (PCI_EXP_SLTCTL_PCC | \
153 PCI_EXP_SLTCTL_PIC | \
154 PCI_EXP_SLTCTL_AIC | \
155 PCI_EXP_SLTCTL_EIC)
156
157static void pcie_do_write_cmd(struct controller *ctrl, u16 cmd,
158 u16 mask, bool wait)
159{
160 struct pci_dev *pdev = ctrl_dev(ctrl);
161 u16 slot_ctrl_orig, slot_ctrl;
162
163 mutex_lock(&ctrl->ctrl_lock);
164
165 /*
166 * Always wait for any previous command that might still be in progress
167 */
168 pcie_wait_cmd(ctrl);
169
170 pcie_capability_read_word(dev: pdev, PCI_EXP_SLTCTL, val: &slot_ctrl);
171 if (PCI_POSSIBLE_ERROR(slot_ctrl)) {
172 ctrl_info(ctrl, "%s: no response from device\n", __func__);
173 goto out;
174 }
175
176 slot_ctrl_orig = slot_ctrl;
177 slot_ctrl &= ~mask;
178 slot_ctrl |= (cmd & mask);
179 ctrl->cmd_busy = 1;
180 smp_mb();
181 ctrl->slot_ctrl = slot_ctrl;
182 pcie_capability_write_word(dev: pdev, PCI_EXP_SLTCTL, val: slot_ctrl);
183 ctrl->cmd_started = jiffies;
184
185 /*
186 * Controllers with the Intel CF118 and similar errata advertise
187 * Command Completed support, but they only set Command Completed
188 * if we change the "Control" bits for power, power indicator,
189 * attention indicator, or interlock. If we only change the
190 * "Enable" bits, they never set the Command Completed bit.
191 */
192 if (pdev->broken_cmd_compl &&
193 (slot_ctrl_orig & CC_ERRATUM_MASK) == (slot_ctrl & CC_ERRATUM_MASK))
194 ctrl->cmd_busy = 0;
195
196 /*
197 * Optionally wait for the hardware to be ready for a new command,
198 * indicating completion of the above issued command.
199 */
200 if (wait)
201 pcie_wait_cmd(ctrl);
202
203out:
204 mutex_unlock(lock: &ctrl->ctrl_lock);
205}
206
207/**
208 * pcie_write_cmd - Issue controller command
209 * @ctrl: controller to which the command is issued
210 * @cmd: command value written to slot control register
211 * @mask: bitmask of slot control register to be modified
212 */
213static void pcie_write_cmd(struct controller *ctrl, u16 cmd, u16 mask)
214{
215 pcie_do_write_cmd(ctrl, cmd, mask, wait: true);
216}
217
218/* Same as above without waiting for the hardware to latch */
219static void pcie_write_cmd_nowait(struct controller *ctrl, u16 cmd, u16 mask)
220{
221 pcie_do_write_cmd(ctrl, cmd, mask, wait: false);
222}
223
224/**
225 * pciehp_check_link_active() - Is the link active
226 * @ctrl: PCIe hotplug controller
227 *
228 * Check whether the downstream link is currently active. Note it is
229 * possible that the card is removed immediately after this so the
230 * caller may need to take it into account.
231 *
232 * If the hotplug controller itself is not available anymore returns
233 * %-ENODEV.
234 */
235int pciehp_check_link_active(struct controller *ctrl)
236{
237 struct pci_dev *pdev = ctrl_dev(ctrl);
238 u16 lnk_status;
239 int ret;
240
241 ret = pcie_capability_read_word(dev: pdev, PCI_EXP_LNKSTA, val: &lnk_status);
242 if (ret == PCIBIOS_DEVICE_NOT_FOUND || PCI_POSSIBLE_ERROR(lnk_status))
243 return -ENODEV;
244
245 ret = !!(lnk_status & PCI_EXP_LNKSTA_DLLLA);
246 ctrl_dbg(ctrl, "%s: lnk_status = %x\n", __func__, lnk_status);
247
248 return ret;
249}
250
251static bool pci_bus_check_dev(struct pci_bus *bus, int devfn)
252{
253 u32 l;
254 int count = 0;
255 int delay = 1000, step = 20;
256 bool found = false;
257
258 do {
259 found = pci_bus_read_dev_vendor_id(bus, devfn, pl: &l, rrs_timeout: 0);
260 count++;
261
262 if (found)
263 break;
264
265 msleep(msecs: step);
266 delay -= step;
267 } while (delay > 0);
268
269 if (count > 1)
270 pr_debug("pci %04x:%02x:%02x.%d id reading try %d times with interval %d ms to get %08x\n",
271 pci_domain_nr(bus), bus->number, PCI_SLOT(devfn),
272 PCI_FUNC(devfn), count, step, l);
273
274 return found;
275}
276
277static void pcie_wait_for_presence(struct pci_dev *pdev)
278{
279 int timeout = 1250;
280 u16 slot_status;
281
282 do {
283 pcie_capability_read_word(dev: pdev, PCI_EXP_SLTSTA, val: &slot_status);
284 if (slot_status & PCI_EXP_SLTSTA_PDS)
285 return;
286 msleep(msecs: 10);
287 timeout -= 10;
288 } while (timeout > 0);
289}
290
291int pciehp_check_link_status(struct controller *ctrl)
292{
293 struct pci_dev *pdev = ctrl_dev(ctrl);
294 bool found;
295 u16 lnk_status, linksta2;
296
297 if (!pcie_wait_for_link(pdev, active: true)) {
298 ctrl_info(ctrl, "Slot(%s): No link\n", slot_name(ctrl));
299 return -1;
300 }
301
302 if (ctrl->inband_presence_disabled)
303 pcie_wait_for_presence(pdev);
304
305 found = pci_bus_check_dev(bus: ctrl->pcie->port->subordinate,
306 PCI_DEVFN(0, 0));
307
308 /* ignore link or presence changes up to this point */
309 if (found)
310 atomic_and(i: ~(PCI_EXP_SLTSTA_DLLSC | PCI_EXP_SLTSTA_PDC),
311 v: &ctrl->pending_events);
312
313 pcie_capability_read_word(dev: pdev, PCI_EXP_LNKSTA, val: &lnk_status);
314 ctrl_dbg(ctrl, "%s: lnk_status = %x\n", __func__, lnk_status);
315 if ((lnk_status & PCI_EXP_LNKSTA_LT) ||
316 !(lnk_status & PCI_EXP_LNKSTA_NLW)) {
317 ctrl_info(ctrl, "Slot(%s): Cannot train link: status %#06x\n",
318 slot_name(ctrl), lnk_status);
319 return -1;
320 }
321
322 pcie_capability_read_word(dev: pdev, PCI_EXP_LNKSTA2, val: &linksta2);
323 __pcie_update_link_speed(bus: ctrl->pcie->port->subordinate, linksta: lnk_status, linksta2);
324
325 if (!found) {
326 ctrl_info(ctrl, "Slot(%s): No device found\n",
327 slot_name(ctrl));
328 return -1;
329 }
330
331 return 0;
332}
333
334static int __pciehp_link_set(struct controller *ctrl, bool enable)
335{
336 struct pci_dev *pdev = ctrl_dev(ctrl);
337
338 pcie_capability_clear_and_set_word(dev: pdev, PCI_EXP_LNKCTL,
339 PCI_EXP_LNKCTL_LD,
340 set: enable ? 0 : PCI_EXP_LNKCTL_LD);
341
342 return 0;
343}
344
345static int pciehp_link_enable(struct controller *ctrl)
346{
347 return __pciehp_link_set(ctrl, enable: true);
348}
349
350int pciehp_get_raw_indicator_status(struct hotplug_slot *hotplug_slot,
351 u8 *status)
352{
353 struct controller *ctrl = to_ctrl(hotplug_slot);
354 struct pci_dev *pdev = ctrl_dev(ctrl);
355 u16 slot_ctrl;
356
357 pci_config_pm_runtime_get(dev: pdev);
358 pcie_capability_read_word(dev: pdev, PCI_EXP_SLTCTL, val: &slot_ctrl);
359 pci_config_pm_runtime_put(dev: pdev);
360 *status = (slot_ctrl & (PCI_EXP_SLTCTL_AIC | PCI_EXP_SLTCTL_PIC)) >> 6;
361 return 0;
362}
363
364int pciehp_get_attention_status(struct hotplug_slot *hotplug_slot, u8 *status)
365{
366 struct controller *ctrl = to_ctrl(hotplug_slot);
367 struct pci_dev *pdev = ctrl_dev(ctrl);
368 u16 slot_ctrl;
369
370 pci_config_pm_runtime_get(dev: pdev);
371 pcie_capability_read_word(dev: pdev, PCI_EXP_SLTCTL, val: &slot_ctrl);
372 pci_config_pm_runtime_put(dev: pdev);
373 ctrl_dbg(ctrl, "%s: SLOTCTRL %x, value read %x\n", __func__,
374 pci_pcie_cap(ctrl->pcie->port) + PCI_EXP_SLTCTL, slot_ctrl);
375
376 switch (slot_ctrl & PCI_EXP_SLTCTL_AIC) {
377 case PCI_EXP_SLTCTL_ATTN_IND_ON:
378 *status = 1; /* On */
379 break;
380 case PCI_EXP_SLTCTL_ATTN_IND_BLINK:
381 *status = 2; /* Blink */
382 break;
383 case PCI_EXP_SLTCTL_ATTN_IND_OFF:
384 *status = 0; /* Off */
385 break;
386 default:
387 *status = 0xFF;
388 break;
389 }
390
391 return 0;
392}
393
394void pciehp_get_power_status(struct controller *ctrl, u8 *status)
395{
396 struct pci_dev *pdev = ctrl_dev(ctrl);
397 u16 slot_ctrl;
398
399 pcie_capability_read_word(dev: pdev, PCI_EXP_SLTCTL, val: &slot_ctrl);
400 ctrl_dbg(ctrl, "%s: SLOTCTRL %x value read %x\n", __func__,
401 pci_pcie_cap(ctrl->pcie->port) + PCI_EXP_SLTCTL, slot_ctrl);
402
403 switch (slot_ctrl & PCI_EXP_SLTCTL_PCC) {
404 case PCI_EXP_SLTCTL_PWR_ON:
405 *status = 1; /* On */
406 break;
407 case PCI_EXP_SLTCTL_PWR_OFF:
408 *status = 0; /* Off */
409 break;
410 default:
411 *status = 0xFF;
412 break;
413 }
414}
415
416void pciehp_get_latch_status(struct controller *ctrl, u8 *status)
417{
418 struct pci_dev *pdev = ctrl_dev(ctrl);
419 u16 slot_status;
420
421 pcie_capability_read_word(dev: pdev, PCI_EXP_SLTSTA, val: &slot_status);
422 *status = !!(slot_status & PCI_EXP_SLTSTA_MRLSS);
423}
424
425/**
426 * pciehp_card_present() - Is the card present
427 * @ctrl: PCIe hotplug controller
428 *
429 * Function checks whether the card is currently present in the slot and
430 * in that case returns true. Note it is possible that the card is
431 * removed immediately after the check so the caller may need to take
432 * this into account.
433 *
434 * If the hotplug controller itself is not available anymore returns
435 * %-ENODEV.
436 */
437int pciehp_card_present(struct controller *ctrl)
438{
439 struct pci_dev *pdev = ctrl_dev(ctrl);
440 u16 slot_status;
441 int ret;
442
443 ret = pcie_capability_read_word(dev: pdev, PCI_EXP_SLTSTA, val: &slot_status);
444 if (ret == PCIBIOS_DEVICE_NOT_FOUND || PCI_POSSIBLE_ERROR(slot_status))
445 return -ENODEV;
446
447 return !!(slot_status & PCI_EXP_SLTSTA_PDS);
448}
449
450/**
451 * pciehp_card_present_or_link_active() - whether given slot is occupied
452 * @ctrl: PCIe hotplug controller
453 *
454 * Unlike pciehp_card_present(), which determines presence solely from the
455 * Presence Detect State bit, this helper also returns true if the Link Active
456 * bit is set. This is a concession to broken hotplug ports which hardwire
457 * Presence Detect State to zero, such as Wilocity's [1ae9:0200].
458 *
459 * Returns: %1 if the slot is occupied and %0 if it is not. If the hotplug
460 * port is not present anymore returns %-ENODEV.
461 */
462int pciehp_card_present_or_link_active(struct controller *ctrl)
463{
464 int ret;
465
466 ret = pciehp_card_present(ctrl);
467 if (ret)
468 return ret;
469
470 return pciehp_check_link_active(ctrl);
471}
472
473int pciehp_query_power_fault(struct controller *ctrl)
474{
475 struct pci_dev *pdev = ctrl_dev(ctrl);
476 u16 slot_status;
477
478 pcie_capability_read_word(dev: pdev, PCI_EXP_SLTSTA, val: &slot_status);
479 return !!(slot_status & PCI_EXP_SLTSTA_PFD);
480}
481
482int pciehp_set_raw_indicator_status(struct hotplug_slot *hotplug_slot,
483 u8 status)
484{
485 struct controller *ctrl = to_ctrl(hotplug_slot);
486 struct pci_dev *pdev = ctrl_dev(ctrl);
487
488 pci_config_pm_runtime_get(dev: pdev);
489
490 /* Attention and Power Indicator Control bits are supported */
491 pcie_write_cmd_nowait(ctrl, FIELD_PREP(PCI_EXP_SLTCTL_AIC | PCI_EXP_SLTCTL_PIC, status),
492 PCI_EXP_SLTCTL_AIC | PCI_EXP_SLTCTL_PIC);
493 pci_config_pm_runtime_put(dev: pdev);
494 return 0;
495}
496
497/**
498 * pciehp_set_indicators() - set attention indicator, power indicator, or both
499 * @ctrl: PCIe hotplug controller
500 * @pwr: one of:
501 * PCI_EXP_SLTCTL_PWR_IND_ON
502 * PCI_EXP_SLTCTL_PWR_IND_BLINK
503 * PCI_EXP_SLTCTL_PWR_IND_OFF
504 * @attn: one of:
505 * PCI_EXP_SLTCTL_ATTN_IND_ON
506 * PCI_EXP_SLTCTL_ATTN_IND_BLINK
507 * PCI_EXP_SLTCTL_ATTN_IND_OFF
508 *
509 * Either @pwr or @attn can also be INDICATOR_NOOP to leave that indicator
510 * unchanged.
511 */
512void pciehp_set_indicators(struct controller *ctrl, int pwr, int attn)
513{
514 u16 cmd = 0, mask = 0;
515
516 if (PWR_LED(ctrl) && pwr != INDICATOR_NOOP) {
517 cmd |= (pwr & PCI_EXP_SLTCTL_PIC);
518 mask |= PCI_EXP_SLTCTL_PIC;
519 }
520
521 if (ATTN_LED(ctrl) && attn != INDICATOR_NOOP) {
522 cmd |= (attn & PCI_EXP_SLTCTL_AIC);
523 mask |= PCI_EXP_SLTCTL_AIC;
524 }
525
526 if (cmd) {
527 pcie_write_cmd_nowait(ctrl, cmd, mask);
528 ctrl_dbg(ctrl, "%s: SLOTCTRL %x write cmd %x\n", __func__,
529 pci_pcie_cap(ctrl->pcie->port) + PCI_EXP_SLTCTL, cmd);
530 }
531}
532
533int pciehp_power_on_slot(struct controller *ctrl)
534{
535 struct pci_dev *pdev = ctrl_dev(ctrl);
536 u16 slot_status;
537 int retval;
538
539 /* Clear power-fault bit from previous power failures */
540 pcie_capability_read_word(dev: pdev, PCI_EXP_SLTSTA, val: &slot_status);
541 if (slot_status & PCI_EXP_SLTSTA_PFD)
542 pcie_capability_write_word(dev: pdev, PCI_EXP_SLTSTA,
543 PCI_EXP_SLTSTA_PFD);
544 ctrl->power_fault_detected = 0;
545
546 pcie_write_cmd(ctrl, PCI_EXP_SLTCTL_PWR_ON, PCI_EXP_SLTCTL_PCC);
547 ctrl_dbg(ctrl, "%s: SLOTCTRL %x write cmd %x\n", __func__,
548 pci_pcie_cap(ctrl->pcie->port) + PCI_EXP_SLTCTL,
549 PCI_EXP_SLTCTL_PWR_ON);
550
551 retval = pciehp_link_enable(ctrl);
552 if (retval)
553 ctrl_err(ctrl, "%s: Can not enable the link!\n", __func__);
554
555 return retval;
556}
557
558void pciehp_power_off_slot(struct controller *ctrl)
559{
560 pcie_write_cmd(ctrl, PCI_EXP_SLTCTL_PWR_OFF, PCI_EXP_SLTCTL_PCC);
561 ctrl_dbg(ctrl, "%s: SLOTCTRL %x write cmd %x\n", __func__,
562 pci_pcie_cap(ctrl->pcie->port) + PCI_EXP_SLTCTL,
563 PCI_EXP_SLTCTL_PWR_OFF);
564}
565
566bool pciehp_device_replaced(struct controller *ctrl)
567{
568 struct pci_dev *pdev __free(pci_dev_put) = NULL;
569 u32 reg;
570
571 if (pci_dev_is_disconnected(dev: ctrl->pcie->port))
572 return false;
573
574 pdev = pci_get_slot(bus: ctrl->pcie->port->subordinate, PCI_DEVFN(0, 0));
575 if (!pdev)
576 return true;
577
578 if (pci_read_config_dword(dev: pdev, PCI_VENDOR_ID, val: &reg) ||
579 reg != (pdev->vendor | (pdev->device << 16)) ||
580 pci_read_config_dword(dev: pdev, PCI_CLASS_REVISION, val: &reg) ||
581 reg != (pdev->revision | (pdev->class << 8)))
582 return true;
583
584 if (pdev->hdr_type == PCI_HEADER_TYPE_NORMAL &&
585 (pci_read_config_dword(dev: pdev, PCI_SUBSYSTEM_VENDOR_ID, val: &reg) ||
586 reg != (pdev->subsystem_vendor | (pdev->subsystem_device << 16))))
587 return true;
588
589 if (pci_get_dsn(dev: pdev) != ctrl->dsn)
590 return true;
591
592 return false;
593}
594
595static void pciehp_ignore_link_change(struct controller *ctrl,
596 struct pci_dev *pdev, int irq,
597 u16 ignored_events)
598{
599 /*
600 * Ignore link changes which occurred while waiting for DPC recovery.
601 * Could be several if DPC triggered multiple times consecutively.
602 * Also ignore link changes caused by Secondary Bus Reset, etc.
603 */
604 synchronize_hardirq(irq);
605 atomic_and(i: ~ignored_events, v: &ctrl->pending_events);
606 if (pciehp_poll_mode)
607 pcie_capability_write_word(dev: pdev, PCI_EXP_SLTSTA,
608 val: ignored_events);
609 ctrl_info(ctrl, "Slot(%s): Link Down/Up ignored\n", slot_name(ctrl));
610
611 /*
612 * If the link is unexpectedly down after successful recovery,
613 * the corresponding link change may have been ignored above.
614 * Synthesize it to ensure that it is acted on.
615 */
616 down_read_nested(sem: &ctrl->reset_lock, subclass: ctrl->depth);
617 if (!pciehp_check_link_active(ctrl) || pciehp_device_replaced(ctrl))
618 pciehp_request(ctrl, action: ignored_events);
619 up_read(sem: &ctrl->reset_lock);
620}
621
622static irqreturn_t pciehp_isr(int irq, void *dev_id)
623{
624 struct controller *ctrl = (struct controller *)dev_id;
625 struct pci_dev *pdev = ctrl_dev(ctrl);
626 struct device *parent = pdev->dev.parent;
627 u16 status, events = 0;
628
629 /*
630 * Interrupts only occur in D3hot or shallower and only if enabled
631 * in the Slot Control register (PCIe r4.0, sec 6.7.3.4).
632 */
633 if (pdev->current_state == PCI_D3cold ||
634 (!(ctrl->slot_ctrl & PCI_EXP_SLTCTL_HPIE) && !pciehp_poll_mode))
635 return IRQ_NONE;
636
637 /*
638 * Keep the port accessible by holding a runtime PM ref on its parent.
639 * Defer resume of the parent to the IRQ thread if it's suspended.
640 * Mask the interrupt until then.
641 */
642 if (parent) {
643 pm_runtime_get_noresume(dev: parent);
644 if (!pm_runtime_active(dev: parent)) {
645 pm_runtime_put(dev: parent);
646 disable_irq_nosync(irq);
647 atomic_or(RERUN_ISR, v: &ctrl->pending_events);
648 return IRQ_WAKE_THREAD;
649 }
650 }
651
652read_status:
653 pcie_capability_read_word(dev: pdev, PCI_EXP_SLTSTA, val: &status);
654 if (PCI_POSSIBLE_ERROR(status)) {
655 ctrl_info(ctrl, "%s: no response from device\n", __func__);
656 if (parent)
657 pm_runtime_put(dev: parent);
658 return IRQ_NONE;
659 }
660
661 /*
662 * Slot Status contains plain status bits as well as event
663 * notification bits; right now we only want the event bits.
664 */
665 status &= PCI_EXP_SLTSTA_ABP | PCI_EXP_SLTSTA_PFD |
666 PCI_EXP_SLTSTA_PDC | PCI_EXP_SLTSTA_CC |
667 PCI_EXP_SLTSTA_DLLSC;
668
669 /*
670 * If we've already reported a power fault, don't report it again
671 * until we've done something to handle it.
672 */
673 if (ctrl->power_fault_detected)
674 status &= ~PCI_EXP_SLTSTA_PFD;
675 else if (status & PCI_EXP_SLTSTA_PFD)
676 ctrl->power_fault_detected = true;
677
678 events |= status;
679 if (!events) {
680 if (parent)
681 pm_runtime_put(dev: parent);
682 return IRQ_NONE;
683 }
684
685 if (status) {
686 pcie_capability_write_word(dev: pdev, PCI_EXP_SLTSTA, val: status);
687
688 /*
689 * In MSI mode, all event bits must be zero before the port
690 * will send a new interrupt (PCIe Base Spec r5.0 sec 6.7.3.4).
691 * So re-read the Slot Status register in case a bit was set
692 * between read and write.
693 */
694 if (pci_dev_msi_enabled(pci_dev: pdev) && !pciehp_poll_mode)
695 goto read_status;
696 }
697
698 ctrl_dbg(ctrl, "pending interrupts %#06x from Slot Status\n", events);
699 if (parent)
700 pm_runtime_put(dev: parent);
701
702 /*
703 * Command Completed notifications are not deferred to the
704 * IRQ thread because it may be waiting for their arrival.
705 */
706 if (events & PCI_EXP_SLTSTA_CC) {
707 ctrl->cmd_busy = 0;
708 smp_mb();
709 wake_up(&ctrl->queue);
710
711 if (events == PCI_EXP_SLTSTA_CC)
712 return IRQ_HANDLED;
713
714 events &= ~PCI_EXP_SLTSTA_CC;
715 }
716
717 if (pdev->ignore_hotplug) {
718 ctrl_dbg(ctrl, "ignoring hotplug event %#06x\n", events);
719 return IRQ_HANDLED;
720 }
721
722 /* Save pending events for consumption by IRQ thread. */
723 atomic_or(i: events, v: &ctrl->pending_events);
724 return IRQ_WAKE_THREAD;
725}
726
727static irqreturn_t pciehp_ist(int irq, void *dev_id)
728{
729 struct controller *ctrl = (struct controller *)dev_id;
730 struct pci_dev *pdev = ctrl_dev(ctrl);
731 irqreturn_t ret;
732 u32 events;
733
734 ctrl->ist_running = true;
735 pci_config_pm_runtime_get(dev: pdev);
736
737 /* rerun pciehp_isr() if the port was inaccessible on interrupt */
738 if (atomic_fetch_and(i: ~RERUN_ISR, v: &ctrl->pending_events) & RERUN_ISR) {
739 ret = pciehp_isr(irq, dev_id);
740 enable_irq(irq);
741 if (ret != IRQ_WAKE_THREAD)
742 goto out;
743 }
744
745 synchronize_hardirq(irq);
746 events = atomic_xchg(v: &ctrl->pending_events, new: 0);
747 if (!events) {
748 ret = IRQ_NONE;
749 goto out;
750 }
751
752 /* Check Attention Button Pressed */
753 if (events & PCI_EXP_SLTSTA_ABP)
754 pciehp_handle_button_press(ctrl);
755
756 /* Check Power Fault Detected */
757 if (events & PCI_EXP_SLTSTA_PFD) {
758 ctrl_err(ctrl, "Slot(%s): Power fault\n", slot_name(ctrl));
759 pciehp_set_indicators(ctrl, PCI_EXP_SLTCTL_PWR_IND_OFF,
760 PCI_EXP_SLTCTL_ATTN_IND_ON);
761 }
762
763 /*
764 * Ignore Link Down/Up events caused by Downstream Port Containment
765 * if recovery succeeded, or caused by Secondary Bus Reset,
766 * suspend to D3cold, firmware update, FPGA reconfiguration, etc.
767 */
768 if ((events & (PCI_EXP_SLTSTA_PDC | PCI_EXP_SLTSTA_DLLSC)) &&
769 (pci_dpc_recovered(pdev) || pci_hp_spurious_link_change(pdev)) &&
770 ctrl->state == ON_STATE) {
771 u16 ignored_events = PCI_EXP_SLTSTA_DLLSC;
772
773 if (!ctrl->inband_presence_disabled)
774 ignored_events |= events & PCI_EXP_SLTSTA_PDC;
775
776 events &= ~ignored_events;
777 pciehp_ignore_link_change(ctrl, pdev, irq, ignored_events);
778 }
779
780 /*
781 * Disable requests have higher priority than Presence Detect Changed
782 * or Data Link Layer State Changed events.
783 */
784 down_read_nested(sem: &ctrl->reset_lock, subclass: ctrl->depth);
785 if (events & DISABLE_SLOT)
786 pciehp_handle_disable_request(ctrl);
787 else if (events & (PCI_EXP_SLTSTA_PDC | PCI_EXP_SLTSTA_DLLSC))
788 pciehp_handle_presence_or_link_change(ctrl, events);
789 up_read(sem: &ctrl->reset_lock);
790
791 ret = IRQ_HANDLED;
792out:
793 pci_config_pm_runtime_put(dev: pdev);
794 ctrl->ist_running = false;
795 wake_up(&ctrl->requester);
796 return ret;
797}
798
799static int pciehp_poll(void *data)
800{
801 struct controller *ctrl = data;
802
803 schedule_timeout_idle(timeout: 10 * HZ); /* start with 10 sec delay */
804
805 while (!kthread_should_stop()) {
806 /* poll for interrupt events or user requests */
807 while (pciehp_isr(IRQ_NOTCONNECTED, dev_id: ctrl) == IRQ_WAKE_THREAD ||
808 atomic_read(v: &ctrl->pending_events))
809 pciehp_ist(IRQ_NOTCONNECTED, dev_id: ctrl);
810
811 if (pciehp_poll_time <= 0 || pciehp_poll_time > 60)
812 pciehp_poll_time = 2; /* clamp to sane value */
813
814 schedule_timeout_idle(timeout: pciehp_poll_time * HZ);
815 }
816
817 return 0;
818}
819
820static void pcie_enable_notification(struct controller *ctrl)
821{
822 u16 cmd, mask;
823
824 /*
825 * TBD: Power fault detected software notification support.
826 *
827 * Power fault detected software notification is not enabled
828 * now, because it caused power fault detected interrupt storm
829 * on some machines. On those machines, power fault detected
830 * bit in the slot status register was set again immediately
831 * when it is cleared in the interrupt service routine, and
832 * next power fault detected interrupt was notified again.
833 */
834
835 /*
836 * Always enable link events: thus link-up and link-down shall
837 * always be treated as hotplug and unplug respectively. Enable
838 * presence detect only if Attention Button is not present.
839 */
840 cmd = PCI_EXP_SLTCTL_DLLSCE;
841 if (ATTN_BUTTN(ctrl))
842 cmd |= PCI_EXP_SLTCTL_ABPE;
843 else
844 cmd |= PCI_EXP_SLTCTL_PDCE;
845 if (!pciehp_poll_mode)
846 cmd |= PCI_EXP_SLTCTL_HPIE;
847 if (!pciehp_poll_mode && !NO_CMD_CMPL(ctrl))
848 cmd |= PCI_EXP_SLTCTL_CCIE;
849
850 mask = (PCI_EXP_SLTCTL_PDCE | PCI_EXP_SLTCTL_ABPE |
851 PCI_EXP_SLTCTL_PFDE |
852 PCI_EXP_SLTCTL_HPIE | PCI_EXP_SLTCTL_CCIE |
853 PCI_EXP_SLTCTL_DLLSCE);
854
855 pcie_write_cmd_nowait(ctrl, cmd, mask);
856 ctrl_dbg(ctrl, "%s: SLOTCTRL %x write cmd %x\n", __func__,
857 pci_pcie_cap(ctrl->pcie->port) + PCI_EXP_SLTCTL, cmd);
858}
859
860static void pcie_disable_notification(struct controller *ctrl)
861{
862 u16 mask;
863
864 mask = (PCI_EXP_SLTCTL_PDCE | PCI_EXP_SLTCTL_ABPE |
865 PCI_EXP_SLTCTL_MRLSCE | PCI_EXP_SLTCTL_PFDE |
866 PCI_EXP_SLTCTL_HPIE | PCI_EXP_SLTCTL_CCIE |
867 PCI_EXP_SLTCTL_DLLSCE);
868 pcie_write_cmd(ctrl, cmd: 0, mask);
869 ctrl_dbg(ctrl, "%s: SLOTCTRL %x write cmd %x\n", __func__,
870 pci_pcie_cap(ctrl->pcie->port) + PCI_EXP_SLTCTL, 0);
871}
872
873void pcie_clear_hotplug_events(struct controller *ctrl)
874{
875 pcie_capability_write_word(dev: ctrl_dev(ctrl), PCI_EXP_SLTSTA,
876 PCI_EXP_SLTSTA_PDC | PCI_EXP_SLTSTA_DLLSC);
877}
878
879void pcie_enable_interrupt(struct controller *ctrl)
880{
881 u16 mask;
882
883 mask = PCI_EXP_SLTCTL_DLLSCE;
884 if (!pciehp_poll_mode)
885 mask |= PCI_EXP_SLTCTL_HPIE;
886 pcie_write_cmd(ctrl, cmd: mask, mask);
887}
888
889void pcie_disable_interrupt(struct controller *ctrl)
890{
891 u16 mask;
892
893 /*
894 * Mask hot-plug interrupt to prevent it triggering immediately
895 * when the link goes inactive (we still get PME when any of the
896 * enabled events is detected). Same goes with Link Layer State
897 * changed event which generates PME immediately when the link goes
898 * inactive so mask it as well.
899 */
900 mask = PCI_EXP_SLTCTL_HPIE | PCI_EXP_SLTCTL_DLLSCE;
901 pcie_write_cmd(ctrl, cmd: 0, mask);
902}
903
904/**
905 * pciehp_slot_reset() - ignore link event caused by error-induced hot reset
906 * @dev: PCI Express port service device
907 *
908 * Called from pcie_portdrv_slot_reset() after AER or DPC initiated a reset
909 * further up in the hierarchy to recover from an error. The reset was
910 * propagated down to this hotplug port. Ignore the resulting link flap.
911 * If the link failed to retrain successfully, synthesize the ignored event.
912 * Surprise removal during reset is detected through Presence Detect Changed.
913 */
914int pciehp_slot_reset(struct pcie_device *dev)
915{
916 struct controller *ctrl = get_service_data(dev);
917
918 if (ctrl->state != ON_STATE)
919 return 0;
920
921 pcie_capability_write_word(dev: dev->port, PCI_EXP_SLTSTA,
922 PCI_EXP_SLTSTA_DLLSC);
923
924 if (!pciehp_check_link_active(ctrl))
925 pciehp_request(ctrl, PCI_EXP_SLTSTA_DLLSC);
926
927 return 0;
928}
929
930/*
931 * pciehp has a 1:1 bus:slot relationship so we ultimately want a secondary
932 * bus reset of the bridge, but at the same time we want to ensure that it is
933 * not seen as a hot-unplug, followed by the hot-plug of the device. Thus,
934 * disable link state notification and presence detection change notification
935 * momentarily, if we see that they could interfere. Also, clear any spurious
936 * events after.
937 */
938int pciehp_reset_slot(struct hotplug_slot *hotplug_slot, bool probe)
939{
940 struct controller *ctrl = to_ctrl(hotplug_slot);
941 struct pci_dev *pdev = ctrl_dev(ctrl);
942 int rc;
943
944 if (probe)
945 return 0;
946
947 down_write_nested(sem: &ctrl->reset_lock, subclass: ctrl->depth);
948
949 pci_hp_ignore_link_change(pdev);
950
951 rc = pci_bridge_secondary_bus_reset(dev: ctrl->pcie->port);
952
953 pci_hp_unignore_link_change(pdev);
954
955 up_write(sem: &ctrl->reset_lock);
956 return rc;
957}
958
959int pcie_init_notification(struct controller *ctrl)
960{
961 if (pciehp_request_irq(ctrl))
962 return -1;
963 pcie_enable_notification(ctrl);
964 ctrl->notification_enabled = 1;
965 return 0;
966}
967
968void pcie_shutdown_notification(struct controller *ctrl)
969{
970 if (ctrl->notification_enabled) {
971 pcie_disable_notification(ctrl);
972 pciehp_free_irq(ctrl);
973 ctrl->notification_enabled = 0;
974 }
975}
976
977static inline void dbg_ctrl(struct controller *ctrl)
978{
979 struct pci_dev *pdev = ctrl->pcie->port;
980 u16 reg16;
981
982 ctrl_dbg(ctrl, "Slot Capabilities : 0x%08x\n", ctrl->slot_cap);
983 pcie_capability_read_word(dev: pdev, PCI_EXP_SLTSTA, val: &reg16);
984 ctrl_dbg(ctrl, "Slot Status : 0x%04x\n", reg16);
985 pcie_capability_read_word(dev: pdev, PCI_EXP_SLTCTL, val: &reg16);
986 ctrl_dbg(ctrl, "Slot Control : 0x%04x\n", reg16);
987}
988
989#define FLAG(x, y) (((x) & (y)) ? '+' : '-')
990
991static inline int pcie_hotplug_depth(struct pci_dev *dev)
992{
993 struct pci_bus *bus = dev->bus;
994 int depth = 0;
995
996 while (bus->parent) {
997 bus = bus->parent;
998 if (bus->self && bus->self->is_hotplug_bridge)
999 depth++;
1000 }
1001
1002 return depth;
1003}
1004
1005struct controller *pcie_init(struct pcie_device *dev)
1006{
1007 struct controller *ctrl;
1008 u32 slot_cap, slot_cap2;
1009 u8 poweron;
1010 struct pci_dev *pdev = dev->port;
1011 struct pci_bus *subordinate = pdev->subordinate;
1012
1013 ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
1014 if (!ctrl)
1015 return NULL;
1016
1017 ctrl->pcie = dev;
1018 ctrl->depth = pcie_hotplug_depth(dev: dev->port);
1019 pcie_capability_read_dword(dev: pdev, PCI_EXP_SLTCAP, val: &slot_cap);
1020
1021 if (pdev->hotplug_user_indicators)
1022 slot_cap &= ~(PCI_EXP_SLTCAP_AIP | PCI_EXP_SLTCAP_PIP);
1023
1024 /*
1025 * We assume no Thunderbolt controllers support Command Complete events,
1026 * but some controllers falsely claim they do.
1027 */
1028 if (pdev->is_thunderbolt)
1029 slot_cap |= PCI_EXP_SLTCAP_NCCS;
1030
1031 ctrl->slot_cap = slot_cap;
1032 mutex_init(&ctrl->ctrl_lock);
1033 mutex_init(&ctrl->state_lock);
1034 init_rwsem(&ctrl->reset_lock);
1035 init_waitqueue_head(&ctrl->requester);
1036 init_waitqueue_head(&ctrl->queue);
1037 INIT_DELAYED_WORK(&ctrl->button_work, pciehp_queue_pushbutton_work);
1038 dbg_ctrl(ctrl);
1039
1040 down_read(sem: &pci_bus_sem);
1041 ctrl->state = list_empty(head: &subordinate->devices) ? OFF_STATE : ON_STATE;
1042 up_read(sem: &pci_bus_sem);
1043
1044 pcie_capability_read_dword(dev: pdev, PCI_EXP_SLTCAP2, val: &slot_cap2);
1045 if (slot_cap2 & PCI_EXP_SLTCAP2_IBPD) {
1046 pcie_write_cmd_nowait(ctrl, PCI_EXP_SLTCTL_IBPD_DISABLE,
1047 PCI_EXP_SLTCTL_IBPD_DISABLE);
1048 ctrl->inband_presence_disabled = 1;
1049 }
1050
1051 if (dmi_first_match(list: inband_presence_disabled_dmi_table))
1052 ctrl->inband_presence_disabled = 1;
1053
1054 /* Clear all remaining event bits in Slot Status register. */
1055 pcie_capability_write_word(dev: pdev, PCI_EXP_SLTSTA,
1056 PCI_EXP_SLTSTA_ABP | PCI_EXP_SLTSTA_PFD |
1057 PCI_EXP_SLTSTA_MRLSC | PCI_EXP_SLTSTA_CC |
1058 PCI_EXP_SLTSTA_DLLSC | PCI_EXP_SLTSTA_PDC);
1059
1060 ctrl_info(ctrl, "Slot #%d AttnBtn%c PwrCtrl%c MRL%c AttnInd%c PwrInd%c HotPlug%c Surprise%c Interlock%c NoCompl%c IbPresDis%c LLActRep%c%s\n",
1061 FIELD_GET(PCI_EXP_SLTCAP_PSN, slot_cap),
1062 FLAG(slot_cap, PCI_EXP_SLTCAP_ABP),
1063 FLAG(slot_cap, PCI_EXP_SLTCAP_PCP),
1064 FLAG(slot_cap, PCI_EXP_SLTCAP_MRLSP),
1065 FLAG(slot_cap, PCI_EXP_SLTCAP_AIP),
1066 FLAG(slot_cap, PCI_EXP_SLTCAP_PIP),
1067 FLAG(slot_cap, PCI_EXP_SLTCAP_HPC),
1068 FLAG(slot_cap, PCI_EXP_SLTCAP_HPS),
1069 FLAG(slot_cap, PCI_EXP_SLTCAP_EIP),
1070 FLAG(slot_cap, PCI_EXP_SLTCAP_NCCS),
1071 FLAG(slot_cap2, PCI_EXP_SLTCAP2_IBPD),
1072 FLAG(pdev->link_active_reporting, true),
1073 pdev->broken_cmd_compl ? " (with Cmd Compl erratum)" : "");
1074
1075 /*
1076 * If empty slot's power status is on, turn power off. The IRQ isn't
1077 * requested yet, so avoid triggering a notification with this command.
1078 */
1079 if (POWER_CTRL(ctrl)) {
1080 pciehp_get_power_status(ctrl, status: &poweron);
1081 if (!pciehp_card_present_or_link_active(ctrl) && poweron) {
1082 pcie_disable_notification(ctrl);
1083 pciehp_power_off_slot(ctrl);
1084 }
1085 }
1086
1087 pdev = pci_get_slot(bus: subordinate, PCI_DEVFN(0, 0));
1088 if (pdev)
1089 ctrl->dsn = pci_get_dsn(dev: pdev);
1090 pci_dev_put(dev: pdev);
1091
1092 return ctrl;
1093}
1094
1095void pciehp_release_ctrl(struct controller *ctrl)
1096{
1097 cancel_delayed_work_sync(dwork: &ctrl->button_work);
1098 kfree(objp: ctrl);
1099}
1100
1101static void quirk_cmd_compl(struct pci_dev *pdev)
1102{
1103 u32 slot_cap;
1104
1105 if (pci_is_pcie(dev: pdev)) {
1106 pcie_capability_read_dword(dev: pdev, PCI_EXP_SLTCAP, val: &slot_cap);
1107 if (slot_cap & PCI_EXP_SLTCAP_HPC &&
1108 !(slot_cap & PCI_EXP_SLTCAP_NCCS))
1109 pdev->broken_cmd_compl = 1;
1110 }
1111}
1112DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_INTEL, PCI_ANY_ID,
1113 PCI_CLASS_BRIDGE_PCI, 8, quirk_cmd_compl);
1114DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_QCOM, 0x010e,
1115 PCI_CLASS_BRIDGE_PCI, 8, quirk_cmd_compl);
1116DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_QCOM, 0x0110,
1117 PCI_CLASS_BRIDGE_PCI, 8, quirk_cmd_compl);
1118DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_QCOM, 0x0400,
1119 PCI_CLASS_BRIDGE_PCI, 8, quirk_cmd_compl);
1120DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_QCOM, 0x0401,
1121 PCI_CLASS_BRIDGE_PCI, 8, quirk_cmd_compl);
1122DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_HXT, 0x0401,
1123 PCI_CLASS_BRIDGE_PCI, 8, quirk_cmd_compl);
1124

Provided by KDAB

Privacy Policy
Improve your Profiling and Debugging skills
Find out more

source code of linux/drivers/pci/hotplug/pciehp_hpc.c