1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Implement the AER root port service driver. The driver registers an IRQ
4 * handler. When a root port triggers an AER interrupt, the IRQ handler
5 * collects Root Port status and schedules work.
6 *
7 * Copyright (C) 2006 Intel Corp.
8 * Tom Long Nguyen (tom.l.nguyen@intel.com)
9 * Zhang Yanmin (yanmin.zhang@intel.com)
10 *
11 * (C) Copyright 2009 Hewlett-Packard Development Company, L.P.
12 * Andrew Patterson <andrew.patterson@hp.com>
13 */
14
15#define pr_fmt(fmt) "AER: " fmt
16#define dev_fmt pr_fmt
17
18#include <linux/bitops.h>
19#include <linux/cper.h>
20#include <linux/dev_printk.h>
21#include <linux/pci.h>
22#include <linux/pci-acpi.h>
23#include <linux/sched.h>
24#include <linux/kernel.h>
25#include <linux/errno.h>
26#include <linux/pm.h>
27#include <linux/init.h>
28#include <linux/interrupt.h>
29#include <linux/delay.h>
30#include <linux/kfifo.h>
31#include <linux/ratelimit.h>
32#include <linux/slab.h>
33#include <linux/vmcore_info.h>
34#include <acpi/apei.h>
35#include <acpi/ghes.h>
36#include <ras/ras_event.h>
37
38#include "../pci.h"
39#include "portdrv.h"
40
41#define aer_printk(level, pdev, fmt, arg...) \
42 dev_printk(level, &(pdev)->dev, fmt, ##arg)
43
44#define AER_ERROR_SOURCES_MAX 128
45
46#define AER_MAX_TYPEOF_COR_ERRS 16 /* as per PCI_ERR_COR_STATUS */
47#define AER_MAX_TYPEOF_UNCOR_ERRS 32 /* as per PCI_ERR_UNCOR_STATUS*/
48
49struct aer_err_source {
50 u32 status; /* PCI_ERR_ROOT_STATUS */
51 u32 id; /* PCI_ERR_ROOT_ERR_SRC */
52};
53
54struct aer_rpc {
55 struct pci_dev *rpd; /* Root Port device */
56 DECLARE_KFIFO(aer_fifo, struct aer_err_source, AER_ERROR_SOURCES_MAX);
57};
58
59/* AER info for the device */
60struct aer_info {
61
62 /*
63 * Fields for all AER capable devices. They indicate the errors
64 * "as seen by this device". Note that this may mean that if an
65 * Endpoint is causing problems, the AER counters may increment
66 * at its link partner (e.g. Root Port) because the errors will be
67 * "seen" by the link partner and not the problematic Endpoint
68 * itself (which may report all counters as 0 as it never saw any
69 * problems).
70 */
71 /* Counters for different type of correctable errors */
72 u64 dev_cor_errs[AER_MAX_TYPEOF_COR_ERRS];
73 /* Counters for different type of fatal uncorrectable errors */
74 u64 dev_fatal_errs[AER_MAX_TYPEOF_UNCOR_ERRS];
75 /* Counters for different type of nonfatal uncorrectable errors */
76 u64 dev_nonfatal_errs[AER_MAX_TYPEOF_UNCOR_ERRS];
77 /* Total number of ERR_COR sent by this device */
78 u64 dev_total_cor_errs;
79 /* Total number of ERR_FATAL sent by this device */
80 u64 dev_total_fatal_errs;
81 /* Total number of ERR_NONFATAL sent by this device */
82 u64 dev_total_nonfatal_errs;
83
84 /*
85 * Fields for Root Ports & Root Complex Event Collectors only; these
86 * indicate the total number of ERR_COR, ERR_FATAL, and ERR_NONFATAL
87 * messages received by the Root Port / Event Collector, INCLUDING the
88 * ones that are generated internally (by the Root Port itself)
89 */
90 u64 rootport_total_cor_errs;
91 u64 rootport_total_fatal_errs;
92 u64 rootport_total_nonfatal_errs;
93
94 /* Ratelimits for errors */
95 struct ratelimit_state correctable_ratelimit;
96 struct ratelimit_state nonfatal_ratelimit;
97};
98
99#define AER_LOG_TLP_MASKS (PCI_ERR_UNC_POISON_TLP| \
100 PCI_ERR_UNC_POISON_BLK | \
101 PCI_ERR_UNC_ECRC| \
102 PCI_ERR_UNC_UNSUP| \
103 PCI_ERR_UNC_COMP_ABORT| \
104 PCI_ERR_UNC_UNX_COMP| \
105 PCI_ERR_UNC_ACSV | \
106 PCI_ERR_UNC_MCBTLP | \
107 PCI_ERR_UNC_ATOMEG | \
108 PCI_ERR_UNC_DMWR_BLK | \
109 PCI_ERR_UNC_XLAT_BLK | \
110 PCI_ERR_UNC_TLPPRE | \
111 PCI_ERR_UNC_MALF_TLP | \
112 PCI_ERR_UNC_IDE_CHECK | \
113 PCI_ERR_UNC_MISR_IDE | \
114 PCI_ERR_UNC_PCRC_CHECK)
115
116#define SYSTEM_ERROR_INTR_ON_MESG_MASK (PCI_EXP_RTCTL_SECEE| \
117 PCI_EXP_RTCTL_SENFEE| \
118 PCI_EXP_RTCTL_SEFEE)
119#define ROOT_PORT_INTR_ON_MESG_MASK (PCI_ERR_ROOT_CMD_COR_EN| \
120 PCI_ERR_ROOT_CMD_NONFATAL_EN| \
121 PCI_ERR_ROOT_CMD_FATAL_EN)
122#define ERR_COR_ID(d) (d & 0xffff)
123#define ERR_UNCOR_ID(d) (d >> 16)
124
125#define AER_ERR_STATUS_MASK (PCI_ERR_ROOT_UNCOR_RCV | \
126 PCI_ERR_ROOT_COR_RCV | \
127 PCI_ERR_ROOT_MULTI_COR_RCV | \
128 PCI_ERR_ROOT_MULTI_UNCOR_RCV)
129
130static bool pcie_aer_disable;
131static pci_ers_result_t aer_root_reset(struct pci_dev *dev);
132
133void pci_no_aer(void)
134{
135 pcie_aer_disable = true;
136}
137
138bool pci_aer_available(void)
139{
140 return !pcie_aer_disable && pci_msi_enabled();
141}
142
143#ifdef CONFIG_PCIE_ECRC
144
145#define ECRC_POLICY_DEFAULT 0 /* ECRC set by BIOS */
146#define ECRC_POLICY_OFF 1 /* ECRC off for performance */
147#define ECRC_POLICY_ON 2 /* ECRC on for data integrity */
148
149static int ecrc_policy = ECRC_POLICY_DEFAULT;
150
151static const char * const ecrc_policy_str[] = {
152 [ECRC_POLICY_DEFAULT] = "bios",
153 [ECRC_POLICY_OFF] = "off",
154 [ECRC_POLICY_ON] = "on"
155};
156
157/**
158 * enable_ecrc_checking - enable PCIe ECRC checking for a device
159 * @dev: the PCI device
160 *
161 * Return: 0 on success, or negative on failure.
162 */
163static int enable_ecrc_checking(struct pci_dev *dev)
164{
165 int aer = dev->aer_cap;
166 u32 reg32;
167
168 if (!aer)
169 return -ENODEV;
170
171 pci_read_config_dword(dev, where: aer + PCI_ERR_CAP, val: &reg32);
172 if (reg32 & PCI_ERR_CAP_ECRC_GENC)
173 reg32 |= PCI_ERR_CAP_ECRC_GENE;
174 if (reg32 & PCI_ERR_CAP_ECRC_CHKC)
175 reg32 |= PCI_ERR_CAP_ECRC_CHKE;
176 pci_write_config_dword(dev, where: aer + PCI_ERR_CAP, val: reg32);
177
178 return 0;
179}
180
181/**
182 * disable_ecrc_checking - disable PCIe ECRC checking for a device
183 * @dev: the PCI device
184 *
185 * Return: 0 on success, or negative on failure.
186 */
187static int disable_ecrc_checking(struct pci_dev *dev)
188{
189 int aer = dev->aer_cap;
190 u32 reg32;
191
192 if (!aer)
193 return -ENODEV;
194
195 pci_read_config_dword(dev, where: aer + PCI_ERR_CAP, val: &reg32);
196 reg32 &= ~(PCI_ERR_CAP_ECRC_GENE | PCI_ERR_CAP_ECRC_CHKE);
197 pci_write_config_dword(dev, where: aer + PCI_ERR_CAP, val: reg32);
198
199 return 0;
200}
201
202/**
203 * pcie_set_ecrc_checking - set/unset PCIe ECRC checking for a device based
204 * on global policy
205 * @dev: the PCI device
206 */
207void pcie_set_ecrc_checking(struct pci_dev *dev)
208{
209 if (!pcie_aer_is_native(dev))
210 return;
211
212 switch (ecrc_policy) {
213 case ECRC_POLICY_DEFAULT:
214 return;
215 case ECRC_POLICY_OFF:
216 disable_ecrc_checking(dev);
217 break;
218 case ECRC_POLICY_ON:
219 enable_ecrc_checking(dev);
220 break;
221 default:
222 return;
223 }
224}
225
226/**
227 * pcie_ecrc_get_policy - parse kernel command-line ecrc option
228 * @str: ECRC policy from kernel command line to use
229 */
230void pcie_ecrc_get_policy(char *str)
231{
232 int i;
233
234 i = match_string(array: ecrc_policy_str, ARRAY_SIZE(ecrc_policy_str), string: str);
235 if (i < 0)
236 return;
237
238 ecrc_policy = i;
239}
240#endif /* CONFIG_PCIE_ECRC */
241
242#define PCI_EXP_AER_FLAGS (PCI_EXP_DEVCTL_CERE | PCI_EXP_DEVCTL_NFERE | \
243 PCI_EXP_DEVCTL_FERE | PCI_EXP_DEVCTL_URRE)
244
245int pcie_aer_is_native(struct pci_dev *dev)
246{
247 struct pci_host_bridge *host = pci_find_host_bridge(bus: dev->bus);
248
249 if (!dev->aer_cap)
250 return 0;
251
252 return pcie_ports_native || host->native_aer;
253}
254EXPORT_SYMBOL_NS_GPL(pcie_aer_is_native, "CXL");
255
256static int pci_enable_pcie_error_reporting(struct pci_dev *dev)
257{
258 int rc;
259
260 if (!pcie_aer_is_native(dev))
261 return -EIO;
262
263 rc = pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_AER_FLAGS);
264 return pcibios_err_to_errno(err: rc);
265}
266
267int pci_aer_clear_nonfatal_status(struct pci_dev *dev)
268{
269 int aer = dev->aer_cap;
270 u32 status, sev;
271
272 if (!pcie_aer_is_native(dev))
273 return -EIO;
274
275 /* Clear status bits for ERR_NONFATAL errors only */
276 pci_read_config_dword(dev, where: aer + PCI_ERR_UNCOR_STATUS, val: &status);
277 pci_read_config_dword(dev, where: aer + PCI_ERR_UNCOR_SEVER, val: &sev);
278 status &= ~sev;
279 if (status)
280 pci_write_config_dword(dev, where: aer + PCI_ERR_UNCOR_STATUS, val: status);
281
282 return 0;
283}
284EXPORT_SYMBOL_GPL(pci_aer_clear_nonfatal_status);
285
286void pci_aer_clear_fatal_status(struct pci_dev *dev)
287{
288 int aer = dev->aer_cap;
289 u32 status, sev;
290
291 if (!pcie_aer_is_native(dev))
292 return;
293
294 /* Clear status bits for ERR_FATAL errors only */
295 pci_read_config_dword(dev, where: aer + PCI_ERR_UNCOR_STATUS, val: &status);
296 pci_read_config_dword(dev, where: aer + PCI_ERR_UNCOR_SEVER, val: &sev);
297 status &= sev;
298 if (status)
299 pci_write_config_dword(dev, where: aer + PCI_ERR_UNCOR_STATUS, val: status);
300}
301
302/**
303 * pci_aer_raw_clear_status - Clear AER error registers.
304 * @dev: the PCI device
305 *
306 * Clear AER error status registers unconditionally, regardless of
307 * whether they're owned by firmware or the OS.
308 *
309 * Return: 0 on success, or negative on failure.
310 */
311int pci_aer_raw_clear_status(struct pci_dev *dev)
312{
313 int aer = dev->aer_cap;
314 u32 status;
315 int port_type;
316
317 if (!aer)
318 return -EIO;
319
320 port_type = pci_pcie_type(dev);
321 if (port_type == PCI_EXP_TYPE_ROOT_PORT ||
322 port_type == PCI_EXP_TYPE_RC_EC) {
323 pci_read_config_dword(dev, where: aer + PCI_ERR_ROOT_STATUS, val: &status);
324 pci_write_config_dword(dev, where: aer + PCI_ERR_ROOT_STATUS, val: status);
325 }
326
327 pci_read_config_dword(dev, where: aer + PCI_ERR_COR_STATUS, val: &status);
328 pci_write_config_dword(dev, where: aer + PCI_ERR_COR_STATUS, val: status);
329
330 pci_read_config_dword(dev, where: aer + PCI_ERR_UNCOR_STATUS, val: &status);
331 pci_write_config_dword(dev, where: aer + PCI_ERR_UNCOR_STATUS, val: status);
332
333 return 0;
334}
335
336int pci_aer_clear_status(struct pci_dev *dev)
337{
338 if (!pcie_aer_is_native(dev))
339 return -EIO;
340
341 return pci_aer_raw_clear_status(dev);
342}
343
344void pci_save_aer_state(struct pci_dev *dev)
345{
346 int aer = dev->aer_cap;
347 struct pci_cap_saved_state *save_state;
348 u32 *cap;
349
350 if (!aer)
351 return;
352
353 save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_ERR);
354 if (!save_state)
355 return;
356
357 cap = &save_state->cap.data[0];
358 pci_read_config_dword(dev, where: aer + PCI_ERR_UNCOR_MASK, val: cap++);
359 pci_read_config_dword(dev, where: aer + PCI_ERR_UNCOR_SEVER, val: cap++);
360 pci_read_config_dword(dev, where: aer + PCI_ERR_COR_MASK, val: cap++);
361 pci_read_config_dword(dev, where: aer + PCI_ERR_CAP, val: cap++);
362 if (pcie_cap_has_rtctl(dev))
363 pci_read_config_dword(dev, where: aer + PCI_ERR_ROOT_COMMAND, val: cap++);
364}
365
366void pci_restore_aer_state(struct pci_dev *dev)
367{
368 int aer = dev->aer_cap;
369 struct pci_cap_saved_state *save_state;
370 u32 *cap;
371
372 if (!aer)
373 return;
374
375 save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_ERR);
376 if (!save_state)
377 return;
378
379 cap = &save_state->cap.data[0];
380 pci_write_config_dword(dev, where: aer + PCI_ERR_UNCOR_MASK, val: *cap++);
381 pci_write_config_dword(dev, where: aer + PCI_ERR_UNCOR_SEVER, val: *cap++);
382 pci_write_config_dword(dev, where: aer + PCI_ERR_COR_MASK, val: *cap++);
383 pci_write_config_dword(dev, where: aer + PCI_ERR_CAP, val: *cap++);
384 if (pcie_cap_has_rtctl(dev))
385 pci_write_config_dword(dev, where: aer + PCI_ERR_ROOT_COMMAND, val: *cap++);
386}
387
388void pci_aer_init(struct pci_dev *dev)
389{
390 int n;
391
392 dev->aer_cap = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);
393 if (!dev->aer_cap)
394 return;
395
396 dev->aer_info = kzalloc(sizeof(*dev->aer_info), GFP_KERNEL);
397 if (!dev->aer_info) {
398 dev->aer_cap = 0;
399 return;
400 }
401
402 ratelimit_state_init(rs: &dev->aer_info->correctable_ratelimit,
403 DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST);
404 ratelimit_state_init(rs: &dev->aer_info->nonfatal_ratelimit,
405 DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST);
406
407 /*
408 * We save/restore PCI_ERR_UNCOR_MASK, PCI_ERR_UNCOR_SEVER,
409 * PCI_ERR_COR_MASK, and PCI_ERR_CAP. Root and Root Complex Event
410 * Collectors also implement PCI_ERR_ROOT_COMMAND (PCIe r6.0, sec
411 * 7.8.4.9).
412 */
413 n = pcie_cap_has_rtctl(dev) ? 5 : 4;
414 pci_add_ext_cap_save_buffer(dev, PCI_EXT_CAP_ID_ERR, size: sizeof(u32) * n);
415
416 pci_aer_clear_status(dev);
417
418 if (pci_aer_available())
419 pci_enable_pcie_error_reporting(dev);
420
421 pcie_set_ecrc_checking(dev);
422}
423
424void pci_aer_exit(struct pci_dev *dev)
425{
426 kfree(objp: dev->aer_info);
427 dev->aer_info = NULL;
428}
429
430#define AER_AGENT_RECEIVER 0
431#define AER_AGENT_REQUESTER 1
432#define AER_AGENT_COMPLETER 2
433#define AER_AGENT_TRANSMITTER 3
434
435#define AER_AGENT_REQUESTER_MASK(t) ((t == AER_CORRECTABLE) ? \
436 0 : (PCI_ERR_UNC_COMP_TIME|PCI_ERR_UNC_UNSUP))
437#define AER_AGENT_COMPLETER_MASK(t) ((t == AER_CORRECTABLE) ? \
438 0 : PCI_ERR_UNC_COMP_ABORT)
439#define AER_AGENT_TRANSMITTER_MASK(t) ((t == AER_CORRECTABLE) ? \
440 (PCI_ERR_COR_REP_ROLL|PCI_ERR_COR_REP_TIMER) : 0)
441
442#define AER_GET_AGENT(t, e) \
443 ((e & AER_AGENT_COMPLETER_MASK(t)) ? AER_AGENT_COMPLETER : \
444 (e & AER_AGENT_REQUESTER_MASK(t)) ? AER_AGENT_REQUESTER : \
445 (e & AER_AGENT_TRANSMITTER_MASK(t)) ? AER_AGENT_TRANSMITTER : \
446 AER_AGENT_RECEIVER)
447
448#define AER_PHYSICAL_LAYER_ERROR 0
449#define AER_DATA_LINK_LAYER_ERROR 1
450#define AER_TRANSACTION_LAYER_ERROR 2
451
452#define AER_PHYSICAL_LAYER_ERROR_MASK(t) ((t == AER_CORRECTABLE) ? \
453 PCI_ERR_COR_RCVR : 0)
454#define AER_DATA_LINK_LAYER_ERROR_MASK(t) ((t == AER_CORRECTABLE) ? \
455 (PCI_ERR_COR_BAD_TLP| \
456 PCI_ERR_COR_BAD_DLLP| \
457 PCI_ERR_COR_REP_ROLL| \
458 PCI_ERR_COR_REP_TIMER) : PCI_ERR_UNC_DLP)
459
460#define AER_GET_LAYER_ERROR(t, e) \
461 ((e & AER_PHYSICAL_LAYER_ERROR_MASK(t)) ? AER_PHYSICAL_LAYER_ERROR : \
462 (e & AER_DATA_LINK_LAYER_ERROR_MASK(t)) ? AER_DATA_LINK_LAYER_ERROR : \
463 AER_TRANSACTION_LAYER_ERROR)
464
465/*
466 * AER error strings
467 */
468static const char * const aer_error_severity_string[] = {
469 "Uncorrectable (Non-Fatal)",
470 "Uncorrectable (Fatal)",
471 "Correctable"
472};
473
474static const char *aer_error_layer[] = {
475 "Physical Layer",
476 "Data Link Layer",
477 "Transaction Layer"
478};
479
480static const char *aer_correctable_error_string[] = {
481 "RxErr", /* Bit Position 0 */
482 NULL,
483 NULL,
484 NULL,
485 NULL,
486 NULL,
487 "BadTLP", /* Bit Position 6 */
488 "BadDLLP", /* Bit Position 7 */
489 "Rollover", /* Bit Position 8 */
490 NULL,
491 NULL,
492 NULL,
493 "Timeout", /* Bit Position 12 */
494 "NonFatalErr", /* Bit Position 13 */
495 "CorrIntErr", /* Bit Position 14 */
496 "HeaderOF", /* Bit Position 15 */
497 NULL, /* Bit Position 16 */
498 NULL, /* Bit Position 17 */
499 NULL, /* Bit Position 18 */
500 NULL, /* Bit Position 19 */
501 NULL, /* Bit Position 20 */
502 NULL, /* Bit Position 21 */
503 NULL, /* Bit Position 22 */
504 NULL, /* Bit Position 23 */
505 NULL, /* Bit Position 24 */
506 NULL, /* Bit Position 25 */
507 NULL, /* Bit Position 26 */
508 NULL, /* Bit Position 27 */
509 NULL, /* Bit Position 28 */
510 NULL, /* Bit Position 29 */
511 NULL, /* Bit Position 30 */
512 NULL, /* Bit Position 31 */
513};
514
515static const char *aer_uncorrectable_error_string[] = {
516 "Undefined", /* Bit Position 0 */
517 NULL,
518 NULL,
519 NULL,
520 "DLP", /* Bit Position 4 */
521 "SDES", /* Bit Position 5 */
522 NULL,
523 NULL,
524 NULL,
525 NULL,
526 NULL,
527 NULL,
528 "TLP", /* Bit Position 12 */
529 "FCP", /* Bit Position 13 */
530 "CmpltTO", /* Bit Position 14 */
531 "CmpltAbrt", /* Bit Position 15 */
532 "UnxCmplt", /* Bit Position 16 */
533 "RxOF", /* Bit Position 17 */
534 "MalfTLP", /* Bit Position 18 */
535 "ECRC", /* Bit Position 19 */
536 "UnsupReq", /* Bit Position 20 */
537 "ACSViol", /* Bit Position 21 */
538 "UncorrIntErr", /* Bit Position 22 */
539 "BlockedTLP", /* Bit Position 23 */
540 "AtomicOpBlocked", /* Bit Position 24 */
541 "TLPBlockedErr", /* Bit Position 25 */
542 "PoisonTLPBlocked", /* Bit Position 26 */
543 "DMWrReqBlocked", /* Bit Position 27 */
544 "IDECheck", /* Bit Position 28 */
545 "MisIDETLP", /* Bit Position 29 */
546 "PCRC_CHECK", /* Bit Position 30 */
547 "TLPXlatBlocked", /* Bit Position 31 */
548};
549
550static const char *aer_agent_string[] = {
551 "Receiver ID",
552 "Requester ID",
553 "Completer ID",
554 "Transmitter ID"
555};
556
557#define aer_stats_dev_attr(name, stats_array, strings_array, \
558 total_string, total_field) \
559 static ssize_t \
560 name##_show(struct device *dev, struct device_attribute *attr, \
561 char *buf) \
562{ \
563 unsigned int i; \
564 struct pci_dev *pdev = to_pci_dev(dev); \
565 u64 *stats = pdev->aer_info->stats_array; \
566 size_t len = 0; \
567 \
568 for (i = 0; i < ARRAY_SIZE(pdev->aer_info->stats_array); i++) { \
569 if (strings_array[i]) \
570 len += sysfs_emit_at(buf, len, "%s %llu\n", \
571 strings_array[i], \
572 stats[i]); \
573 else if (stats[i]) \
574 len += sysfs_emit_at(buf, len, \
575 #stats_array "_bit[%d] %llu\n",\
576 i, stats[i]); \
577 } \
578 len += sysfs_emit_at(buf, len, "TOTAL_%s %llu\n", total_string, \
579 pdev->aer_info->total_field); \
580 return len; \
581} \
582static DEVICE_ATTR_RO(name)
583
584aer_stats_dev_attr(aer_dev_correctable, dev_cor_errs,
585 aer_correctable_error_string, "ERR_COR",
586 dev_total_cor_errs);
587aer_stats_dev_attr(aer_dev_fatal, dev_fatal_errs,
588 aer_uncorrectable_error_string, "ERR_FATAL",
589 dev_total_fatal_errs);
590aer_stats_dev_attr(aer_dev_nonfatal, dev_nonfatal_errs,
591 aer_uncorrectable_error_string, "ERR_NONFATAL",
592 dev_total_nonfatal_errs);
593
594#define aer_stats_rootport_attr(name, field) \
595 static ssize_t \
596 name##_show(struct device *dev, struct device_attribute *attr, \
597 char *buf) \
598{ \
599 struct pci_dev *pdev = to_pci_dev(dev); \
600 return sysfs_emit(buf, "%llu\n", pdev->aer_info->field); \
601} \
602static DEVICE_ATTR_RO(name)
603
604aer_stats_rootport_attr(aer_rootport_total_err_cor,
605 rootport_total_cor_errs);
606aer_stats_rootport_attr(aer_rootport_total_err_fatal,
607 rootport_total_fatal_errs);
608aer_stats_rootport_attr(aer_rootport_total_err_nonfatal,
609 rootport_total_nonfatal_errs);
610
611static struct attribute *aer_stats_attrs[] __ro_after_init = {
612 &dev_attr_aer_dev_correctable.attr,
613 &dev_attr_aer_dev_fatal.attr,
614 &dev_attr_aer_dev_nonfatal.attr,
615 &dev_attr_aer_rootport_total_err_cor.attr,
616 &dev_attr_aer_rootport_total_err_fatal.attr,
617 &dev_attr_aer_rootport_total_err_nonfatal.attr,
618 NULL
619};
620
621static umode_t aer_stats_attrs_are_visible(struct kobject *kobj,
622 struct attribute *a, int n)
623{
624 struct device *dev = kobj_to_dev(kobj);
625 struct pci_dev *pdev = to_pci_dev(dev);
626
627 if (!pdev->aer_info)
628 return 0;
629
630 if ((a == &dev_attr_aer_rootport_total_err_cor.attr ||
631 a == &dev_attr_aer_rootport_total_err_fatal.attr ||
632 a == &dev_attr_aer_rootport_total_err_nonfatal.attr) &&
633 ((pci_pcie_type(dev: pdev) != PCI_EXP_TYPE_ROOT_PORT) &&
634 (pci_pcie_type(dev: pdev) != PCI_EXP_TYPE_RC_EC)))
635 return 0;
636
637 return a->mode;
638}
639
640const struct attribute_group aer_stats_attr_group = {
641 .attrs = aer_stats_attrs,
642 .is_visible = aer_stats_attrs_are_visible,
643};
644
645/*
646 * Ratelimit interval
647 * <=0: disabled with ratelimit.interval = 0
648 * >0: enabled with ratelimit.interval in ms
649 */
650#define aer_ratelimit_interval_attr(name, ratelimit) \
651 static ssize_t \
652 name##_show(struct device *dev, struct device_attribute *attr, \
653 char *buf) \
654 { \
655 struct pci_dev *pdev = to_pci_dev(dev); \
656 \
657 return sysfs_emit(buf, "%d\n", \
658 pdev->aer_info->ratelimit.interval); \
659 } \
660 \
661 static ssize_t \
662 name##_store(struct device *dev, struct device_attribute *attr, \
663 const char *buf, size_t count) \
664 { \
665 struct pci_dev *pdev = to_pci_dev(dev); \
666 int interval; \
667 \
668 if (!capable(CAP_SYS_ADMIN)) \
669 return -EPERM; \
670 \
671 if (kstrtoint(buf, 0, &interval) < 0) \
672 return -EINVAL; \
673 \
674 if (interval <= 0) \
675 interval = 0; \
676 else \
677 interval = msecs_to_jiffies(interval); \
678 \
679 pdev->aer_info->ratelimit.interval = interval; \
680 \
681 return count; \
682 } \
683 static DEVICE_ATTR_RW(name);
684
685#define aer_ratelimit_burst_attr(name, ratelimit) \
686 static ssize_t \
687 name##_show(struct device *dev, struct device_attribute *attr, \
688 char *buf) \
689 { \
690 struct pci_dev *pdev = to_pci_dev(dev); \
691 \
692 return sysfs_emit(buf, "%d\n", \
693 pdev->aer_info->ratelimit.burst); \
694 } \
695 \
696 static ssize_t \
697 name##_store(struct device *dev, struct device_attribute *attr, \
698 const char *buf, size_t count) \
699 { \
700 struct pci_dev *pdev = to_pci_dev(dev); \
701 int burst; \
702 \
703 if (!capable(CAP_SYS_ADMIN)) \
704 return -EPERM; \
705 \
706 if (kstrtoint(buf, 0, &burst) < 0) \
707 return -EINVAL; \
708 \
709 pdev->aer_info->ratelimit.burst = burst; \
710 \
711 return count; \
712 } \
713 static DEVICE_ATTR_RW(name);
714
715#define aer_ratelimit_attrs(name) \
716 aer_ratelimit_interval_attr(name##_ratelimit_interval_ms, \
717 name##_ratelimit) \
718 aer_ratelimit_burst_attr(name##_ratelimit_burst, \
719 name##_ratelimit)
720
721aer_ratelimit_attrs(correctable)
722aer_ratelimit_attrs(nonfatal)
723
724static struct attribute *aer_attrs[] = {
725 &dev_attr_correctable_ratelimit_interval_ms.attr,
726 &dev_attr_correctable_ratelimit_burst.attr,
727 &dev_attr_nonfatal_ratelimit_interval_ms.attr,
728 &dev_attr_nonfatal_ratelimit_burst.attr,
729 NULL
730};
731
732static umode_t aer_attrs_are_visible(struct kobject *kobj,
733 struct attribute *a, int n)
734{
735 struct device *dev = kobj_to_dev(kobj);
736 struct pci_dev *pdev = to_pci_dev(dev);
737
738 if (!pdev->aer_info)
739 return 0;
740
741 return a->mode;
742}
743
744const struct attribute_group aer_attr_group = {
745 .name = "aer",
746 .attrs = aer_attrs,
747 .is_visible = aer_attrs_are_visible,
748};
749
750static void pci_dev_aer_stats_incr(struct pci_dev *pdev,
751 struct aer_err_info *info)
752{
753 unsigned long status = info->status & ~info->mask;
754 int i, max = -1;
755 u64 *counter = NULL;
756 struct aer_info *aer_info = pdev->aer_info;
757
758 if (!aer_info)
759 return;
760
761 switch (info->severity) {
762 case AER_CORRECTABLE:
763 aer_info->dev_total_cor_errs++;
764 counter = &aer_info->dev_cor_errs[0];
765 max = AER_MAX_TYPEOF_COR_ERRS;
766 break;
767 case AER_NONFATAL:
768 aer_info->dev_total_nonfatal_errs++;
769 hwerr_log_error_type(src: HWERR_RECOV_PCI);
770 counter = &aer_info->dev_nonfatal_errs[0];
771 max = AER_MAX_TYPEOF_UNCOR_ERRS;
772 break;
773 case AER_FATAL:
774 aer_info->dev_total_fatal_errs++;
775 counter = &aer_info->dev_fatal_errs[0];
776 max = AER_MAX_TYPEOF_UNCOR_ERRS;
777 break;
778 }
779
780 for_each_set_bit(i, &status, max)
781 counter[i]++;
782}
783
784static void pci_rootport_aer_stats_incr(struct pci_dev *pdev,
785 struct aer_err_source *e_src)
786{
787 struct aer_info *aer_info = pdev->aer_info;
788
789 if (!aer_info)
790 return;
791
792 if (e_src->status & PCI_ERR_ROOT_COR_RCV)
793 aer_info->rootport_total_cor_errs++;
794
795 if (e_src->status & PCI_ERR_ROOT_UNCOR_RCV) {
796 if (e_src->status & PCI_ERR_ROOT_FATAL_RCV)
797 aer_info->rootport_total_fatal_errs++;
798 else
799 aer_info->rootport_total_nonfatal_errs++;
800 }
801}
802
803static int aer_ratelimit(struct pci_dev *dev, unsigned int severity)
804{
805 if (!dev->aer_info)
806 return 1;
807
808 switch (severity) {
809 case AER_NONFATAL:
810 return __ratelimit(&dev->aer_info->nonfatal_ratelimit);
811 case AER_CORRECTABLE:
812 return __ratelimit(&dev->aer_info->correctable_ratelimit);
813 default:
814 return 1; /* Don't ratelimit fatal errors */
815 }
816}
817
818static bool tlp_header_logged(u32 status, u32 capctl)
819{
820 /* Errors for which a header is always logged (PCIe r7.0 sec 6.2.7) */
821 if (status & AER_LOG_TLP_MASKS)
822 return true;
823
824 /* Completion Timeout header is only logged on capable devices */
825 if (status & PCI_ERR_UNC_COMP_TIME &&
826 capctl & PCI_ERR_CAP_COMP_TIME_LOG)
827 return true;
828
829 return false;
830}
831
832static void __aer_print_error(struct pci_dev *dev, struct aer_err_info *info)
833{
834 const char **strings;
835 unsigned long status = info->status & ~info->mask;
836 const char *level = info->level;
837 const char *errmsg;
838 int i;
839
840 if (info->severity == AER_CORRECTABLE)
841 strings = aer_correctable_error_string;
842 else
843 strings = aer_uncorrectable_error_string;
844
845 for_each_set_bit(i, &status, 32) {
846 errmsg = strings[i];
847 if (!errmsg)
848 errmsg = "Unknown Error Bit";
849
850 aer_printk(level, dev, " [%2d] %-22s%s\n", i, errmsg,
851 info->first_error == i ? " (First)" : "");
852 }
853}
854
855static void aer_print_source(struct pci_dev *dev, struct aer_err_info *info,
856 bool found)
857{
858 u16 source = info->id;
859
860 pci_info(dev, "%s%s error message received from %04x:%02x:%02x.%d%s\n",
861 info->multi_error_valid ? "Multiple " : "",
862 aer_error_severity_string[info->severity],
863 pci_domain_nr(dev->bus), PCI_BUS_NUM(source),
864 PCI_SLOT(source), PCI_FUNC(source),
865 found ? "" : " (no details found");
866}
867
868void aer_print_error(struct aer_err_info *info, int i)
869{
870 struct pci_dev *dev;
871 int layer, agent, id;
872 const char *level = info->level;
873
874 if (WARN_ON_ONCE(i >= AER_MAX_MULTI_ERR_DEVICES))
875 return;
876
877 dev = info->dev[i];
878 id = pci_dev_id(dev);
879
880 pci_dev_aer_stats_incr(pdev: dev, info);
881 trace_aer_event(dev_name: pci_name(pdev: dev), status: (info->status & ~info->mask),
882 severity: info->severity, tlp_header_valid: info->tlp_header_valid, tlp: &info->tlp);
883
884 if (!info->ratelimit_print[i])
885 return;
886
887 if (!info->status) {
888 pci_err(dev, "PCIe Bus Error: severity=%s, type=Inaccessible, (Unregistered Agent ID)\n",
889 aer_error_severity_string[info->severity]);
890 goto out;
891 }
892
893 layer = AER_GET_LAYER_ERROR(info->severity, info->status);
894 agent = AER_GET_AGENT(info->severity, info->status);
895
896 aer_printk(level, dev, "PCIe Bus Error: severity=%s, type=%s, (%s)\n",
897 aer_error_severity_string[info->severity],
898 aer_error_layer[layer], aer_agent_string[agent]);
899
900 aer_printk(level, dev, " device [%04x:%04x] error status/mask=%08x/%08x\n",
901 dev->vendor, dev->device, info->status, info->mask);
902
903 __aer_print_error(dev, info);
904
905 if (info->tlp_header_valid)
906 pcie_print_tlp_log(dev, log: &info->tlp, level, dev_fmt(" "));
907
908out:
909 if (info->id && info->error_dev_num > 1 && info->id == id)
910 pci_err(dev, " Error of this Agent is reported first\n");
911}
912
913#ifdef CONFIG_ACPI_APEI_PCIEAER
914int cper_severity_to_aer(int cper_severity)
915{
916 switch (cper_severity) {
917 case CPER_SEV_RECOVERABLE:
918 return AER_NONFATAL;
919 case CPER_SEV_FATAL:
920 return AER_FATAL;
921 default:
922 return AER_CORRECTABLE;
923 }
924}
925EXPORT_SYMBOL_GPL(cper_severity_to_aer);
926#endif
927
928void pci_print_aer(struct pci_dev *dev, int aer_severity,
929 struct aer_capability_regs *aer)
930{
931 int layer, agent, tlp_header_valid = 0;
932 u32 status, mask;
933 struct aer_err_info info = {
934 .severity = aer_severity,
935 .first_error = PCI_ERR_CAP_FEP(aer->cap_control),
936 };
937
938 if (aer_severity == AER_CORRECTABLE) {
939 status = aer->cor_status;
940 mask = aer->cor_mask;
941 info.level = KERN_WARNING;
942 } else {
943 status = aer->uncor_status;
944 mask = aer->uncor_mask;
945 info.level = KERN_ERR;
946 tlp_header_valid = tlp_header_logged(status, capctl: aer->cap_control);
947 }
948
949 info.status = status;
950 info.mask = mask;
951
952 pci_dev_aer_stats_incr(pdev: dev, info: &info);
953 trace_aer_event(dev_name: pci_name(pdev: dev), status: (status & ~mask),
954 severity: aer_severity, tlp_header_valid, tlp: &aer->header_log);
955
956 if (!aer_ratelimit(dev, severity: info.severity))
957 return;
958
959 layer = AER_GET_LAYER_ERROR(aer_severity, status);
960 agent = AER_GET_AGENT(aer_severity, status);
961
962 aer_printk(info.level, dev, "aer_status: 0x%08x, aer_mask: 0x%08x\n",
963 status, mask);
964 __aer_print_error(dev, info: &info);
965 aer_printk(info.level, dev, "aer_layer=%s, aer_agent=%s\n",
966 aer_error_layer[layer], aer_agent_string[agent]);
967
968 if (aer_severity != AER_CORRECTABLE)
969 aer_printk(info.level, dev, "aer_uncor_severity: 0x%08x\n",
970 aer->uncor_severity);
971
972 if (tlp_header_valid)
973 pcie_print_tlp_log(dev, log: &aer->header_log, level: info.level,
974 dev_fmt(" "));
975}
976EXPORT_SYMBOL_NS_GPL(pci_print_aer, "CXL");
977
978/**
979 * add_error_device - list device to be handled
980 * @e_info: pointer to error info
981 * @dev: pointer to pci_dev to be added
982 */
983static int add_error_device(struct aer_err_info *e_info, struct pci_dev *dev)
984{
985 int i = e_info->error_dev_num;
986
987 if (i >= AER_MAX_MULTI_ERR_DEVICES)
988 return -ENOSPC;
989
990 e_info->dev[i] = pci_dev_get(dev);
991 e_info->error_dev_num++;
992
993 /*
994 * Ratelimit AER log messages. "dev" is either the source
995 * identified by the root's Error Source ID or it has an unmasked
996 * error logged in its own AER Capability. Messages are emitted
997 * when "ratelimit_print[i]" is non-zero. If we will print detail
998 * for a downstream device, make sure we print the Error Source ID
999 * from the root as well.
1000 */
1001 if (aer_ratelimit(dev, severity: e_info->severity)) {
1002 e_info->ratelimit_print[i] = 1;
1003 e_info->root_ratelimit_print = 1;
1004 }
1005 return 0;
1006}
1007
1008/**
1009 * is_error_source - check whether the device is source of reported error
1010 * @dev: pointer to pci_dev to be checked
1011 * @e_info: pointer to reported error info
1012 */
1013static bool is_error_source(struct pci_dev *dev, struct aer_err_info *e_info)
1014{
1015 int aer = dev->aer_cap;
1016 u32 status, mask;
1017 u16 reg16;
1018
1019 /*
1020 * When bus ID is equal to 0, it might be a bad ID
1021 * reported by Root Port.
1022 */
1023 if ((PCI_BUS_NUM(e_info->id) != 0) &&
1024 !(dev->bus->bus_flags & PCI_BUS_FLAGS_NO_AERSID)) {
1025 /* Device ID match? */
1026 if (e_info->id == pci_dev_id(dev))
1027 return true;
1028
1029 /* Continue ID comparing if there is no multiple error */
1030 if (!e_info->multi_error_valid)
1031 return false;
1032 }
1033
1034 /*
1035 * When either
1036 * 1) bus ID is equal to 0. Some ports might lose the bus
1037 * ID of error source id;
1038 * 2) bus flag PCI_BUS_FLAGS_NO_AERSID is set
1039 * 3) There are multiple errors and prior ID comparing fails;
1040 * We check AER status registers to find possible reporter.
1041 */
1042 if (atomic_read(v: &dev->enable_cnt) == 0)
1043 return false;
1044
1045 /* Check if AER is enabled */
1046 pcie_capability_read_word(dev, PCI_EXP_DEVCTL, val: &reg16);
1047 if (!(reg16 & PCI_EXP_AER_FLAGS))
1048 return false;
1049
1050 if (!aer)
1051 return false;
1052
1053 /* Check if error is recorded */
1054 if (e_info->severity == AER_CORRECTABLE) {
1055 pci_read_config_dword(dev, where: aer + PCI_ERR_COR_STATUS, val: &status);
1056 pci_read_config_dword(dev, where: aer + PCI_ERR_COR_MASK, val: &mask);
1057 } else {
1058 pci_read_config_dword(dev, where: aer + PCI_ERR_UNCOR_STATUS, val: &status);
1059 pci_read_config_dword(dev, where: aer + PCI_ERR_UNCOR_MASK, val: &mask);
1060 }
1061 if (status & ~mask)
1062 return true;
1063
1064 return false;
1065}
1066
1067static int find_device_iter(struct pci_dev *dev, void *data)
1068{
1069 struct aer_err_info *e_info = (struct aer_err_info *)data;
1070
1071 if (is_error_source(dev, e_info)) {
1072 /* List this device */
1073 if (add_error_device(e_info, dev)) {
1074 /* We cannot handle more... Stop iteration */
1075 pci_err(dev, "Exceeded max supported (%d) devices with errors logged\n",
1076 AER_MAX_MULTI_ERR_DEVICES);
1077 return 1;
1078 }
1079
1080 /* If there is only a single error, stop iteration */
1081 if (!e_info->multi_error_valid)
1082 return 1;
1083 }
1084 return 0;
1085}
1086
1087/**
1088 * find_source_device - search through device hierarchy for source device
1089 * @parent: pointer to Root Port pci_dev data structure
1090 * @e_info: including detailed error information such as ID
1091 *
1092 * Return: true if found.
1093 *
1094 * Invoked by DPC when error is detected at the Root Port.
1095 * Caller of this function must set id, severity, and multi_error_valid of
1096 * struct aer_err_info pointed by @e_info properly. This function must fill
1097 * e_info->error_dev_num and e_info->dev[], based on the given information.
1098 */
1099static bool find_source_device(struct pci_dev *parent,
1100 struct aer_err_info *e_info)
1101{
1102 struct pci_dev *dev = parent;
1103 int result;
1104
1105 /* Must reset in this function */
1106 e_info->error_dev_num = 0;
1107
1108 /* Is Root Port an agent that sends error message? */
1109 result = find_device_iter(dev, data: e_info);
1110 if (result)
1111 return true;
1112
1113 if (pci_pcie_type(dev: parent) == PCI_EXP_TYPE_RC_EC)
1114 pcie_walk_rcec(rcec: parent, cb: find_device_iter, userdata: e_info);
1115 else
1116 pci_walk_bus(top: parent->subordinate, cb: find_device_iter, userdata: e_info);
1117
1118 if (!e_info->error_dev_num)
1119 return false;
1120 return true;
1121}
1122
1123#ifdef CONFIG_PCIEAER_CXL
1124
1125/**
1126 * pci_aer_unmask_internal_errors - unmask internal errors
1127 * @dev: pointer to the pci_dev data structure
1128 *
1129 * Unmask internal errors in the Uncorrectable and Correctable Error
1130 * Mask registers.
1131 *
1132 * Note: AER must be enabled and supported by the device which must be
1133 * checked in advance, e.g. with pcie_aer_is_native().
1134 */
1135static void pci_aer_unmask_internal_errors(struct pci_dev *dev)
1136{
1137 int aer = dev->aer_cap;
1138 u32 mask;
1139
1140 pci_read_config_dword(dev, where: aer + PCI_ERR_UNCOR_MASK, val: &mask);
1141 mask &= ~PCI_ERR_UNC_INTN;
1142 pci_write_config_dword(dev, where: aer + PCI_ERR_UNCOR_MASK, val: mask);
1143
1144 pci_read_config_dword(dev, where: aer + PCI_ERR_COR_MASK, val: &mask);
1145 mask &= ~PCI_ERR_COR_INTERNAL;
1146 pci_write_config_dword(dev, where: aer + PCI_ERR_COR_MASK, val: mask);
1147}
1148
1149static bool is_cxl_mem_dev(struct pci_dev *dev)
1150{
1151 /*
1152 * The capability, status, and control fields in Device 0,
1153 * Function 0 DVSEC control the CXL functionality of the
1154 * entire device (CXL 3.0, 8.1.3).
1155 */
1156 if (dev->devfn != PCI_DEVFN(0, 0))
1157 return false;
1158
1159 /*
1160 * CXL Memory Devices must have the 502h class code set (CXL
1161 * 3.0, 8.1.12.1).
1162 */
1163 if ((dev->class >> 8) != PCI_CLASS_MEMORY_CXL)
1164 return false;
1165
1166 return true;
1167}
1168
1169static bool cxl_error_is_native(struct pci_dev *dev)
1170{
1171 struct pci_host_bridge *host = pci_find_host_bridge(bus: dev->bus);
1172
1173 return (pcie_ports_native || host->native_aer);
1174}
1175
1176static bool is_internal_error(struct aer_err_info *info)
1177{
1178 if (info->severity == AER_CORRECTABLE)
1179 return info->status & PCI_ERR_COR_INTERNAL;
1180
1181 return info->status & PCI_ERR_UNC_INTN;
1182}
1183
1184static int cxl_rch_handle_error_iter(struct pci_dev *dev, void *data)
1185{
1186 struct aer_err_info *info = (struct aer_err_info *)data;
1187 const struct pci_error_handlers *err_handler;
1188
1189 if (!is_cxl_mem_dev(dev) || !cxl_error_is_native(dev))
1190 return 0;
1191
1192 /* Protect dev->driver */
1193 device_lock(dev: &dev->dev);
1194
1195 err_handler = dev->driver ? dev->driver->err_handler : NULL;
1196 if (!err_handler)
1197 goto out;
1198
1199 if (info->severity == AER_CORRECTABLE) {
1200 if (err_handler->cor_error_detected)
1201 err_handler->cor_error_detected(dev);
1202 } else if (err_handler->error_detected) {
1203 if (info->severity == AER_NONFATAL)
1204 err_handler->error_detected(dev, pci_channel_io_normal);
1205 else if (info->severity == AER_FATAL)
1206 err_handler->error_detected(dev, pci_channel_io_frozen);
1207 }
1208out:
1209 device_unlock(dev: &dev->dev);
1210 return 0;
1211}
1212
1213static void cxl_rch_handle_error(struct pci_dev *dev, struct aer_err_info *info)
1214{
1215 /*
1216 * Internal errors of an RCEC indicate an AER error in an
1217 * RCH's downstream port. Check and handle them in the CXL.mem
1218 * device driver.
1219 */
1220 if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_EC &&
1221 is_internal_error(info))
1222 pcie_walk_rcec(rcec: dev, cb: cxl_rch_handle_error_iter, userdata: info);
1223}
1224
1225static int handles_cxl_error_iter(struct pci_dev *dev, void *data)
1226{
1227 bool *handles_cxl = data;
1228
1229 if (!*handles_cxl)
1230 *handles_cxl = is_cxl_mem_dev(dev) && cxl_error_is_native(dev);
1231
1232 /* Non-zero terminates iteration */
1233 return *handles_cxl;
1234}
1235
1236static bool handles_cxl_errors(struct pci_dev *rcec)
1237{
1238 bool handles_cxl = false;
1239
1240 if (pci_pcie_type(dev: rcec) == PCI_EXP_TYPE_RC_EC &&
1241 pcie_aer_is_native(rcec))
1242 pcie_walk_rcec(rcec, cb: handles_cxl_error_iter, userdata: &handles_cxl);
1243
1244 return handles_cxl;
1245}
1246
1247static void cxl_rch_enable_rcec(struct pci_dev *rcec)
1248{
1249 if (!handles_cxl_errors(rcec))
1250 return;
1251
1252 pci_aer_unmask_internal_errors(dev: rcec);
1253 pci_info(rcec, "CXL: Internal errors unmasked");
1254}
1255
1256#else
1257static inline void cxl_rch_enable_rcec(struct pci_dev *dev) { }
1258static inline void cxl_rch_handle_error(struct pci_dev *dev,
1259 struct aer_err_info *info) { }
1260#endif
1261
1262/**
1263 * pci_aer_handle_error - handle logging error into an event log
1264 * @dev: pointer to pci_dev data structure of error source device
1265 * @info: comprehensive error information
1266 *
1267 * Invoked when an error being detected by Root Port.
1268 */
1269static void pci_aer_handle_error(struct pci_dev *dev, struct aer_err_info *info)
1270{
1271 int aer = dev->aer_cap;
1272
1273 if (info->severity == AER_CORRECTABLE) {
1274 /*
1275 * Correctable error does not need software intervention.
1276 * No need to go through error recovery process.
1277 */
1278 if (aer)
1279 pci_write_config_dword(dev, where: aer + PCI_ERR_COR_STATUS,
1280 val: info->status);
1281 if (pcie_aer_is_native(dev)) {
1282 struct pci_driver *pdrv = dev->driver;
1283
1284 if (pdrv && pdrv->err_handler &&
1285 pdrv->err_handler->cor_error_detected)
1286 pdrv->err_handler->cor_error_detected(dev);
1287 pcie_clear_device_status(dev);
1288 }
1289 } else if (info->severity == AER_NONFATAL)
1290 pcie_do_recovery(dev, state: pci_channel_io_normal, reset_subordinates: aer_root_reset);
1291 else if (info->severity == AER_FATAL)
1292 pcie_do_recovery(dev, state: pci_channel_io_frozen, reset_subordinates: aer_root_reset);
1293}
1294
1295static void handle_error_source(struct pci_dev *dev, struct aer_err_info *info)
1296{
1297 cxl_rch_handle_error(dev, info);
1298 pci_aer_handle_error(dev, info);
1299 pci_dev_put(dev);
1300}
1301
1302#ifdef CONFIG_ACPI_APEI_PCIEAER
1303
1304#define AER_RECOVER_RING_SIZE 16
1305
1306struct aer_recover_entry {
1307 u8 bus;
1308 u8 devfn;
1309 u16 domain;
1310 int severity;
1311 struct aer_capability_regs *regs;
1312};
1313
1314static DEFINE_KFIFO(aer_recover_ring, struct aer_recover_entry,
1315 AER_RECOVER_RING_SIZE);
1316
1317static void aer_recover_work_func(struct work_struct *work)
1318{
1319 struct aer_recover_entry entry;
1320 struct pci_dev *pdev;
1321
1322 while (kfifo_get(&aer_recover_ring, &entry)) {
1323 pdev = pci_get_domain_bus_and_slot(domain: entry.domain, bus: entry.bus,
1324 devfn: entry.devfn);
1325 if (!pdev) {
1326 pr_err_ratelimited("%04x:%02x:%02x.%x: no pci_dev found\n",
1327 entry.domain, entry.bus,
1328 PCI_SLOT(entry.devfn),
1329 PCI_FUNC(entry.devfn));
1330 continue;
1331 }
1332 pci_print_aer(pdev, entry.severity, entry.regs);
1333
1334 /*
1335 * Memory for aer_capability_regs(entry.regs) is being
1336 * allocated from the ghes_estatus_pool to protect it from
1337 * overwriting when multiple sections are present in the
1338 * error status. Thus free the same after processing the
1339 * data.
1340 */
1341 ghes_estatus_pool_region_free(addr: (unsigned long)entry.regs,
1342 size: sizeof(struct aer_capability_regs));
1343
1344 if (entry.severity == AER_NONFATAL)
1345 pcie_do_recovery(dev: pdev, state: pci_channel_io_normal,
1346 reset_subordinates: aer_root_reset);
1347 else if (entry.severity == AER_FATAL)
1348 pcie_do_recovery(dev: pdev, state: pci_channel_io_frozen,
1349 reset_subordinates: aer_root_reset);
1350 pci_dev_put(dev: pdev);
1351 }
1352}
1353
1354/*
1355 * Mutual exclusion for writers of aer_recover_ring, reader side don't
1356 * need lock, because there is only one reader and lock is not needed
1357 * between reader and writer.
1358 */
1359static DEFINE_SPINLOCK(aer_recover_ring_lock);
1360static DECLARE_WORK(aer_recover_work, aer_recover_work_func);
1361
1362void aer_recover_queue(int domain, unsigned int bus, unsigned int devfn,
1363 int severity, struct aer_capability_regs *aer_regs)
1364{
1365 struct aer_recover_entry entry = {
1366 .bus = bus,
1367 .devfn = devfn,
1368 .domain = domain,
1369 .severity = severity,
1370 .regs = aer_regs,
1371 };
1372
1373 if (kfifo_in_spinlocked(&aer_recover_ring, &entry, 1,
1374 &aer_recover_ring_lock))
1375 schedule_work(work: &aer_recover_work);
1376 else
1377 pr_err("buffer overflow in recovery for %04x:%02x:%02x.%x\n",
1378 domain, bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1379}
1380EXPORT_SYMBOL_GPL(aer_recover_queue);
1381#endif
1382
1383/**
1384 * aer_get_device_error_info - read error status from dev and store it to info
1385 * @info: pointer to structure to store the error record
1386 * @i: index into info->dev[]
1387 *
1388 * Return: 1 on success, 0 on error.
1389 *
1390 * Note that @info is reused among all error devices. Clear fields properly.
1391 */
1392int aer_get_device_error_info(struct aer_err_info *info, int i)
1393{
1394 struct pci_dev *dev;
1395 int type, aer;
1396 u32 aercc;
1397
1398 if (i >= AER_MAX_MULTI_ERR_DEVICES)
1399 return 0;
1400
1401 dev = info->dev[i];
1402 aer = dev->aer_cap;
1403 type = pci_pcie_type(dev);
1404
1405 /* Must reset in this function */
1406 info->status = 0;
1407 info->tlp_header_valid = 0;
1408
1409 /* The device might not support AER */
1410 if (!aer)
1411 return 0;
1412
1413 if (info->severity == AER_CORRECTABLE) {
1414 pci_read_config_dword(dev, where: aer + PCI_ERR_COR_STATUS,
1415 val: &info->status);
1416 pci_read_config_dword(dev, where: aer + PCI_ERR_COR_MASK,
1417 val: &info->mask);
1418 if (!(info->status & ~info->mask))
1419 return 0;
1420 } else if (type == PCI_EXP_TYPE_ROOT_PORT ||
1421 type == PCI_EXP_TYPE_RC_EC ||
1422 type == PCI_EXP_TYPE_DOWNSTREAM ||
1423 info->severity == AER_NONFATAL) {
1424
1425 /* Link is still healthy for IO reads */
1426 pci_read_config_dword(dev, where: aer + PCI_ERR_UNCOR_STATUS,
1427 val: &info->status);
1428 pci_read_config_dword(dev, where: aer + PCI_ERR_UNCOR_MASK,
1429 val: &info->mask);
1430 if (!(info->status & ~info->mask))
1431 return 0;
1432
1433 /* Get First Error Pointer */
1434 pci_read_config_dword(dev, where: aer + PCI_ERR_CAP, val: &aercc);
1435 info->first_error = PCI_ERR_CAP_FEP(aercc);
1436
1437 if (tlp_header_logged(status: info->status, capctl: aercc)) {
1438 info->tlp_header_valid = 1;
1439 pcie_read_tlp_log(dev, where: aer + PCI_ERR_HEADER_LOG,
1440 where2: aer + PCI_ERR_PREFIX_LOG,
1441 tlp_len: aer_tlp_log_len(dev, aercc),
1442 flit: aercc & PCI_ERR_CAP_TLP_LOG_FLIT,
1443 log: &info->tlp);
1444 }
1445 }
1446
1447 return 1;
1448}
1449
1450static inline void aer_process_err_devices(struct aer_err_info *e_info)
1451{
1452 int i;
1453
1454 /* Report all before handling them, to not lose records by reset etc. */
1455 for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) {
1456 if (aer_get_device_error_info(info: e_info, i))
1457 aer_print_error(info: e_info, i);
1458 }
1459 for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) {
1460 if (aer_get_device_error_info(info: e_info, i))
1461 handle_error_source(dev: e_info->dev[i], info: e_info);
1462 }
1463}
1464
1465/**
1466 * aer_isr_one_error_type - consume a Correctable or Uncorrectable Error
1467 * detected by Root Port or RCEC
1468 * @root: pointer to Root Port or RCEC that signaled AER interrupt
1469 * @info: pointer to AER error info
1470 */
1471static void aer_isr_one_error_type(struct pci_dev *root,
1472 struct aer_err_info *info)
1473{
1474 bool found;
1475
1476 found = find_source_device(parent: root, e_info: info);
1477
1478 /*
1479 * If we're going to log error messages, we've already set
1480 * "info->root_ratelimit_print" and "info->ratelimit_print[i]" to
1481 * non-zero (which enables printing) because this is either an
1482 * ERR_FATAL or we found a device with an error logged in its AER
1483 * Capability.
1484 *
1485 * If we didn't find the Error Source device, at least log the
1486 * Requester ID from the ERR_* Message received by the Root Port or
1487 * RCEC, ratelimited by the RP or RCEC.
1488 */
1489 if (info->root_ratelimit_print ||
1490 (!found && aer_ratelimit(dev: root, severity: info->severity)))
1491 aer_print_source(dev: root, info, found);
1492
1493 if (found)
1494 aer_process_err_devices(e_info: info);
1495}
1496
1497/**
1498 * aer_isr_one_error - consume error(s) signaled by an AER interrupt from
1499 * Root Port or RCEC
1500 * @root: pointer to Root Port or RCEC that signaled AER interrupt
1501 * @e_src: pointer to an error source
1502 */
1503static void aer_isr_one_error(struct pci_dev *root,
1504 struct aer_err_source *e_src)
1505{
1506 u32 status = e_src->status;
1507
1508 pci_rootport_aer_stats_incr(pdev: root, e_src);
1509
1510 /*
1511 * There is a possibility that both correctable error and
1512 * uncorrectable error being logged. Report correctable error first.
1513 */
1514 if (status & PCI_ERR_ROOT_COR_RCV) {
1515 int multi = status & PCI_ERR_ROOT_MULTI_COR_RCV;
1516 struct aer_err_info e_info = {
1517 .id = ERR_COR_ID(e_src->id),
1518 .severity = AER_CORRECTABLE,
1519 .level = KERN_WARNING,
1520 .multi_error_valid = multi ? 1 : 0,
1521 };
1522
1523 aer_isr_one_error_type(root, info: &e_info);
1524 }
1525
1526 if (status & PCI_ERR_ROOT_UNCOR_RCV) {
1527 int fatal = status & PCI_ERR_ROOT_FATAL_RCV;
1528 int multi = status & PCI_ERR_ROOT_MULTI_UNCOR_RCV;
1529 struct aer_err_info e_info = {
1530 .id = ERR_UNCOR_ID(e_src->id),
1531 .severity = fatal ? AER_FATAL : AER_NONFATAL,
1532 .level = KERN_ERR,
1533 .multi_error_valid = multi ? 1 : 0,
1534 };
1535
1536 aer_isr_one_error_type(root, info: &e_info);
1537 }
1538}
1539
1540/**
1541 * aer_isr - consume errors detected by Root Port
1542 * @irq: IRQ assigned to Root Port
1543 * @context: pointer to Root Port data structure
1544 *
1545 * Invoked, as DPC, when Root Port records new detected error
1546 */
1547static irqreturn_t aer_isr(int irq, void *context)
1548{
1549 struct pcie_device *dev = (struct pcie_device *)context;
1550 struct aer_rpc *rpc = get_service_data(dev);
1551 struct aer_err_source e_src;
1552
1553 if (kfifo_is_empty(&rpc->aer_fifo))
1554 return IRQ_NONE;
1555
1556 while (kfifo_get(&rpc->aer_fifo, &e_src))
1557 aer_isr_one_error(root: rpc->rpd, e_src: &e_src);
1558 return IRQ_HANDLED;
1559}
1560
1561/**
1562 * aer_irq - Root Port's ISR
1563 * @irq: IRQ assigned to Root Port
1564 * @context: pointer to Root Port data structure
1565 *
1566 * Invoked when Root Port detects AER messages.
1567 */
1568static irqreturn_t aer_irq(int irq, void *context)
1569{
1570 struct pcie_device *pdev = (struct pcie_device *)context;
1571 struct aer_rpc *rpc = get_service_data(dev: pdev);
1572 struct pci_dev *rp = rpc->rpd;
1573 int aer = rp->aer_cap;
1574 struct aer_err_source e_src = {};
1575
1576 pci_read_config_dword(dev: rp, where: aer + PCI_ERR_ROOT_STATUS, val: &e_src.status);
1577 if (!(e_src.status & AER_ERR_STATUS_MASK))
1578 return IRQ_NONE;
1579
1580 pci_read_config_dword(dev: rp, where: aer + PCI_ERR_ROOT_ERR_SRC, val: &e_src.id);
1581 pci_write_config_dword(dev: rp, where: aer + PCI_ERR_ROOT_STATUS, val: e_src.status);
1582
1583 if (!kfifo_put(&rpc->aer_fifo, e_src))
1584 return IRQ_HANDLED;
1585
1586 return IRQ_WAKE_THREAD;
1587}
1588
1589static void aer_enable_irq(struct pci_dev *pdev)
1590{
1591 int aer = pdev->aer_cap;
1592 u32 reg32;
1593
1594 /* Enable Root Port's interrupt in response to error messages */
1595 pci_read_config_dword(dev: pdev, where: aer + PCI_ERR_ROOT_COMMAND, val: &reg32);
1596 reg32 |= ROOT_PORT_INTR_ON_MESG_MASK;
1597 pci_write_config_dword(dev: pdev, where: aer + PCI_ERR_ROOT_COMMAND, val: reg32);
1598}
1599
1600static void aer_disable_irq(struct pci_dev *pdev)
1601{
1602 int aer = pdev->aer_cap;
1603 u32 reg32;
1604
1605 /* Disable Root Port's interrupt in response to error messages */
1606 pci_read_config_dword(dev: pdev, where: aer + PCI_ERR_ROOT_COMMAND, val: &reg32);
1607 reg32 &= ~ROOT_PORT_INTR_ON_MESG_MASK;
1608 pci_write_config_dword(dev: pdev, where: aer + PCI_ERR_ROOT_COMMAND, val: reg32);
1609}
1610
1611/**
1612 * aer_enable_rootport - enable Root Port's interrupts when receiving messages
1613 * @rpc: pointer to a Root Port data structure
1614 *
1615 * Invoked when PCIe bus loads AER service driver.
1616 */
1617static void aer_enable_rootport(struct aer_rpc *rpc)
1618{
1619 struct pci_dev *pdev = rpc->rpd;
1620 int aer = pdev->aer_cap;
1621 u16 reg16;
1622 u32 reg32;
1623
1624 /* Clear PCIe Capability's Device Status */
1625 pcie_capability_read_word(dev: pdev, PCI_EXP_DEVSTA, val: &reg16);
1626 pcie_capability_write_word(dev: pdev, PCI_EXP_DEVSTA, val: reg16);
1627
1628 /* Disable system error generation in response to error messages */
1629 pcie_capability_clear_word(dev: pdev, PCI_EXP_RTCTL,
1630 SYSTEM_ERROR_INTR_ON_MESG_MASK);
1631
1632 /* Clear error status */
1633 pci_read_config_dword(dev: pdev, where: aer + PCI_ERR_ROOT_STATUS, val: &reg32);
1634 pci_write_config_dword(dev: pdev, where: aer + PCI_ERR_ROOT_STATUS, val: reg32);
1635 pci_read_config_dword(dev: pdev, where: aer + PCI_ERR_COR_STATUS, val: &reg32);
1636 pci_write_config_dword(dev: pdev, where: aer + PCI_ERR_COR_STATUS, val: reg32);
1637 pci_read_config_dword(dev: pdev, where: aer + PCI_ERR_UNCOR_STATUS, val: &reg32);
1638 pci_write_config_dword(dev: pdev, where: aer + PCI_ERR_UNCOR_STATUS, val: reg32);
1639
1640 aer_enable_irq(pdev);
1641}
1642
1643/**
1644 * aer_disable_rootport - disable Root Port's interrupts when receiving messages
1645 * @rpc: pointer to a Root Port data structure
1646 *
1647 * Invoked when PCIe bus unloads AER service driver.
1648 */
1649static void aer_disable_rootport(struct aer_rpc *rpc)
1650{
1651 struct pci_dev *pdev = rpc->rpd;
1652 int aer = pdev->aer_cap;
1653 u32 reg32;
1654
1655 aer_disable_irq(pdev);
1656
1657 /* Clear Root's error status reg */
1658 pci_read_config_dword(dev: pdev, where: aer + PCI_ERR_ROOT_STATUS, val: &reg32);
1659 pci_write_config_dword(dev: pdev, where: aer + PCI_ERR_ROOT_STATUS, val: reg32);
1660}
1661
1662/**
1663 * aer_remove - clean up resources
1664 * @dev: pointer to the pcie_dev data structure
1665 *
1666 * Invoked when PCI Express bus unloads or AER probe fails.
1667 */
1668static void aer_remove(struct pcie_device *dev)
1669{
1670 struct aer_rpc *rpc = get_service_data(dev);
1671
1672 aer_disable_rootport(rpc);
1673}
1674
1675/**
1676 * aer_probe - initialize resources
1677 * @dev: pointer to the pcie_dev data structure
1678 *
1679 * Invoked when PCI Express bus loads AER service driver.
1680 */
1681static int aer_probe(struct pcie_device *dev)
1682{
1683 int status;
1684 struct aer_rpc *rpc;
1685 struct device *device = &dev->device;
1686 struct pci_dev *port = dev->port;
1687
1688 BUILD_BUG_ON(ARRAY_SIZE(aer_correctable_error_string) <
1689 AER_MAX_TYPEOF_COR_ERRS);
1690 BUILD_BUG_ON(ARRAY_SIZE(aer_uncorrectable_error_string) <
1691 AER_MAX_TYPEOF_UNCOR_ERRS);
1692
1693 /* Limit to Root Ports or Root Complex Event Collectors */
1694 if ((pci_pcie_type(dev: port) != PCI_EXP_TYPE_RC_EC) &&
1695 (pci_pcie_type(dev: port) != PCI_EXP_TYPE_ROOT_PORT))
1696 return -ENODEV;
1697
1698 rpc = devm_kzalloc(dev: device, size: sizeof(struct aer_rpc), GFP_KERNEL);
1699 if (!rpc)
1700 return -ENOMEM;
1701
1702 rpc->rpd = port;
1703 INIT_KFIFO(rpc->aer_fifo);
1704 set_service_data(dev, data: rpc);
1705
1706 status = devm_request_threaded_irq(dev: device, irq: dev->irq, handler: aer_irq, thread_fn: aer_isr,
1707 IRQF_SHARED, devname: "aerdrv", dev_id: dev);
1708 if (status) {
1709 pci_err(port, "request AER IRQ %d failed\n", dev->irq);
1710 return status;
1711 }
1712
1713 cxl_rch_enable_rcec(rcec: port);
1714 aer_enable_rootport(rpc);
1715 pci_info(port, "enabled with IRQ %d\n", dev->irq);
1716 return 0;
1717}
1718
1719static int aer_suspend(struct pcie_device *dev)
1720{
1721 struct aer_rpc *rpc = get_service_data(dev);
1722
1723 aer_disable_rootport(rpc);
1724 return 0;
1725}
1726
1727static int aer_resume(struct pcie_device *dev)
1728{
1729 struct aer_rpc *rpc = get_service_data(dev);
1730
1731 aer_enable_rootport(rpc);
1732 return 0;
1733}
1734
1735/**
1736 * aer_root_reset - reset Root Port hierarchy, RCEC, or RCiEP
1737 * @dev: pointer to Root Port, RCEC, or RCiEP
1738 *
1739 * Invoked by Port Bus driver when performing reset.
1740 */
1741static pci_ers_result_t aer_root_reset(struct pci_dev *dev)
1742{
1743 int type = pci_pcie_type(dev);
1744 struct pci_dev *root;
1745 int aer;
1746 struct pci_host_bridge *host = pci_find_host_bridge(bus: dev->bus);
1747 u32 reg32;
1748 int rc;
1749
1750 /*
1751 * Only Root Ports and RCECs have AER Root Command and Root Status
1752 * registers. If "dev" is an RCiEP, the relevant registers are in
1753 * the RCEC.
1754 */
1755 if (type == PCI_EXP_TYPE_RC_END)
1756 root = dev->rcec;
1757 else
1758 root = pcie_find_root_port(dev);
1759
1760 /*
1761 * If the platform retained control of AER, an RCiEP may not have
1762 * an RCEC visible to us, so dev->rcec ("root") may be NULL. In
1763 * that case, firmware is responsible for these registers.
1764 */
1765 aer = root ? root->aer_cap : 0;
1766
1767 if ((host->native_aer || pcie_ports_native) && aer)
1768 aer_disable_irq(pdev: root);
1769
1770 if (type == PCI_EXP_TYPE_RC_EC || type == PCI_EXP_TYPE_RC_END) {
1771 rc = pcie_reset_flr(dev, PCI_RESET_DO_RESET);
1772 if (!rc)
1773 pci_info(dev, "has been reset\n");
1774 else
1775 pci_info(dev, "not reset (no FLR support: %d)\n", rc);
1776 } else {
1777 rc = pci_bus_error_reset(dev);
1778 pci_info(dev, "%s Port link has been reset (%d)\n",
1779 pci_is_root_bus(dev->bus) ? "Root" : "Downstream", rc);
1780 }
1781
1782 if ((host->native_aer || pcie_ports_native) && aer) {
1783 /* Clear Root Error Status */
1784 pci_read_config_dword(dev: root, where: aer + PCI_ERR_ROOT_STATUS, val: &reg32);
1785 pci_write_config_dword(dev: root, where: aer + PCI_ERR_ROOT_STATUS, val: reg32);
1786
1787 aer_enable_irq(pdev: root);
1788 }
1789
1790 return rc ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
1791}
1792
1793static struct pcie_port_service_driver aerdriver = {
1794 .name = "aer",
1795 .port_type = PCIE_ANY_PORT,
1796 .service = PCIE_PORT_SERVICE_AER,
1797
1798 .probe = aer_probe,
1799 .suspend = aer_suspend,
1800 .resume = aer_resume,
1801 .remove = aer_remove,
1802};
1803
1804/**
1805 * pcie_aer_init - register AER service driver
1806 *
1807 * Invoked when AER service driver is loaded.
1808 */
1809int __init pcie_aer_init(void)
1810{
1811 if (!pci_aer_available())
1812 return -ENXIO;
1813 return pcie_port_service_register(new: &aerdriver);
1814}
1815

source code of linux/drivers/pci/pcie/aer.c