1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Implement the AER root port service driver. The driver registers an IRQ |
4 | * handler. When a root port triggers an AER interrupt, the IRQ handler |
5 | * collects root port status and schedules work. |
6 | * |
7 | * Copyright (C) 2006 Intel Corp. |
8 | * Tom Long Nguyen (tom.l.nguyen@intel.com) |
9 | * Zhang Yanmin (yanmin.zhang@intel.com) |
10 | * |
11 | * (C) Copyright 2009 Hewlett-Packard Development Company, L.P. |
12 | * Andrew Patterson <andrew.patterson@hp.com> |
13 | */ |
14 | |
15 | #define pr_fmt(fmt) "AER: " fmt |
16 | #define dev_fmt pr_fmt |
17 | |
18 | #include <linux/bitops.h> |
19 | #include <linux/cper.h> |
20 | #include <linux/pci.h> |
21 | #include <linux/pci-acpi.h> |
22 | #include <linux/sched.h> |
23 | #include <linux/kernel.h> |
24 | #include <linux/errno.h> |
25 | #include <linux/pm.h> |
26 | #include <linux/init.h> |
27 | #include <linux/interrupt.h> |
28 | #include <linux/delay.h> |
29 | #include <linux/kfifo.h> |
30 | #include <linux/slab.h> |
31 | #include <acpi/apei.h> |
32 | #include <acpi/ghes.h> |
33 | #include <ras/ras_event.h> |
34 | |
35 | #include "../pci.h" |
36 | #include "portdrv.h" |
37 | |
38 | #define AER_ERROR_SOURCES_MAX 128 |
39 | |
40 | #define AER_MAX_TYPEOF_COR_ERRS 16 /* as per PCI_ERR_COR_STATUS */ |
41 | #define AER_MAX_TYPEOF_UNCOR_ERRS 27 /* as per PCI_ERR_UNCOR_STATUS*/ |
42 | |
43 | struct aer_err_source { |
44 | u32 status; /* PCI_ERR_ROOT_STATUS */ |
45 | u32 id; /* PCI_ERR_ROOT_ERR_SRC */ |
46 | }; |
47 | |
48 | struct aer_rpc { |
49 | struct pci_dev *rpd; /* Root Port device */ |
50 | DECLARE_KFIFO(aer_fifo, struct aer_err_source, AER_ERROR_SOURCES_MAX); |
51 | }; |
52 | |
53 | /* AER stats for the device */ |
54 | struct aer_stats { |
55 | |
56 | /* |
57 | * Fields for all AER capable devices. They indicate the errors |
58 | * "as seen by this device". Note that this may mean that if an |
59 | * end point is causing problems, the AER counters may increment |
60 | * at its link partner (e.g. root port) because the errors will be |
61 | * "seen" by the link partner and not the problematic end point |
62 | * itself (which may report all counters as 0 as it never saw any |
63 | * problems). |
64 | */ |
65 | /* Counters for different type of correctable errors */ |
66 | u64 dev_cor_errs[AER_MAX_TYPEOF_COR_ERRS]; |
67 | /* Counters for different type of fatal uncorrectable errors */ |
68 | u64 dev_fatal_errs[AER_MAX_TYPEOF_UNCOR_ERRS]; |
69 | /* Counters for different type of nonfatal uncorrectable errors */ |
70 | u64 dev_nonfatal_errs[AER_MAX_TYPEOF_UNCOR_ERRS]; |
71 | /* Total number of ERR_COR sent by this device */ |
72 | u64 dev_total_cor_errs; |
73 | /* Total number of ERR_FATAL sent by this device */ |
74 | u64 dev_total_fatal_errs; |
75 | /* Total number of ERR_NONFATAL sent by this device */ |
76 | u64 dev_total_nonfatal_errs; |
77 | |
78 | /* |
79 | * Fields for Root ports & root complex event collectors only, these |
80 | * indicate the total number of ERR_COR, ERR_FATAL, and ERR_NONFATAL |
81 | * messages received by the root port / event collector, INCLUDING the |
82 | * ones that are generated internally (by the rootport itself) |
83 | */ |
84 | u64 rootport_total_cor_errs; |
85 | u64 rootport_total_fatal_errs; |
86 | u64 rootport_total_nonfatal_errs; |
87 | }; |
88 | |
89 | #define AER_LOG_TLP_MASKS (PCI_ERR_UNC_POISON_TLP| \ |
90 | PCI_ERR_UNC_ECRC| \ |
91 | PCI_ERR_UNC_UNSUP| \ |
92 | PCI_ERR_UNC_COMP_ABORT| \ |
93 | PCI_ERR_UNC_UNX_COMP| \ |
94 | PCI_ERR_UNC_MALF_TLP) |
95 | |
96 | #define SYSTEM_ERROR_INTR_ON_MESG_MASK (PCI_EXP_RTCTL_SECEE| \ |
97 | PCI_EXP_RTCTL_SENFEE| \ |
98 | PCI_EXP_RTCTL_SEFEE) |
99 | #define ROOT_PORT_INTR_ON_MESG_MASK (PCI_ERR_ROOT_CMD_COR_EN| \ |
100 | PCI_ERR_ROOT_CMD_NONFATAL_EN| \ |
101 | PCI_ERR_ROOT_CMD_FATAL_EN) |
102 | #define ERR_COR_ID(d) (d & 0xffff) |
103 | #define ERR_UNCOR_ID(d) (d >> 16) |
104 | |
105 | #define AER_ERR_STATUS_MASK (PCI_ERR_ROOT_UNCOR_RCV | \ |
106 | PCI_ERR_ROOT_COR_RCV | \ |
107 | PCI_ERR_ROOT_MULTI_COR_RCV | \ |
108 | PCI_ERR_ROOT_MULTI_UNCOR_RCV) |
109 | |
110 | static int pcie_aer_disable; |
111 | static pci_ers_result_t aer_root_reset(struct pci_dev *dev); |
112 | |
113 | void pci_no_aer(void) |
114 | { |
115 | pcie_aer_disable = 1; |
116 | } |
117 | |
118 | bool pci_aer_available(void) |
119 | { |
120 | return !pcie_aer_disable && pci_msi_enabled(); |
121 | } |
122 | |
123 | #ifdef CONFIG_PCIE_ECRC |
124 | |
125 | #define ECRC_POLICY_DEFAULT 0 /* ECRC set by BIOS */ |
126 | #define ECRC_POLICY_OFF 1 /* ECRC off for performance */ |
127 | #define ECRC_POLICY_ON 2 /* ECRC on for data integrity */ |
128 | |
129 | static int ecrc_policy = ECRC_POLICY_DEFAULT; |
130 | |
131 | static const char * const ecrc_policy_str[] = { |
132 | [ECRC_POLICY_DEFAULT] = "bios" , |
133 | [ECRC_POLICY_OFF] = "off" , |
134 | [ECRC_POLICY_ON] = "on" |
135 | }; |
136 | |
137 | /** |
138 | * enable_ecrc_checking - enable PCIe ECRC checking for a device |
139 | * @dev: the PCI device |
140 | * |
141 | * Returns 0 on success, or negative on failure. |
142 | */ |
143 | static int enable_ecrc_checking(struct pci_dev *dev) |
144 | { |
145 | int aer = dev->aer_cap; |
146 | u32 reg32; |
147 | |
148 | if (!aer) |
149 | return -ENODEV; |
150 | |
151 | pci_read_config_dword(dev, where: aer + PCI_ERR_CAP, val: ®32); |
152 | if (reg32 & PCI_ERR_CAP_ECRC_GENC) |
153 | reg32 |= PCI_ERR_CAP_ECRC_GENE; |
154 | if (reg32 & PCI_ERR_CAP_ECRC_CHKC) |
155 | reg32 |= PCI_ERR_CAP_ECRC_CHKE; |
156 | pci_write_config_dword(dev, where: aer + PCI_ERR_CAP, val: reg32); |
157 | |
158 | return 0; |
159 | } |
160 | |
161 | /** |
162 | * disable_ecrc_checking - disables PCIe ECRC checking for a device |
163 | * @dev: the PCI device |
164 | * |
165 | * Returns 0 on success, or negative on failure. |
166 | */ |
167 | static int disable_ecrc_checking(struct pci_dev *dev) |
168 | { |
169 | int aer = dev->aer_cap; |
170 | u32 reg32; |
171 | |
172 | if (!aer) |
173 | return -ENODEV; |
174 | |
175 | pci_read_config_dword(dev, where: aer + PCI_ERR_CAP, val: ®32); |
176 | reg32 &= ~(PCI_ERR_CAP_ECRC_GENE | PCI_ERR_CAP_ECRC_CHKE); |
177 | pci_write_config_dword(dev, where: aer + PCI_ERR_CAP, val: reg32); |
178 | |
179 | return 0; |
180 | } |
181 | |
182 | /** |
183 | * pcie_set_ecrc_checking - set/unset PCIe ECRC checking for a device based on global policy |
184 | * @dev: the PCI device |
185 | */ |
186 | void pcie_set_ecrc_checking(struct pci_dev *dev) |
187 | { |
188 | if (!pcie_aer_is_native(dev)) |
189 | return; |
190 | |
191 | switch (ecrc_policy) { |
192 | case ECRC_POLICY_DEFAULT: |
193 | return; |
194 | case ECRC_POLICY_OFF: |
195 | disable_ecrc_checking(dev); |
196 | break; |
197 | case ECRC_POLICY_ON: |
198 | enable_ecrc_checking(dev); |
199 | break; |
200 | default: |
201 | return; |
202 | } |
203 | } |
204 | |
205 | /** |
206 | * pcie_ecrc_get_policy - parse kernel command-line ecrc option |
207 | * @str: ECRC policy from kernel command line to use |
208 | */ |
209 | void pcie_ecrc_get_policy(char *str) |
210 | { |
211 | int i; |
212 | |
213 | i = match_string(array: ecrc_policy_str, ARRAY_SIZE(ecrc_policy_str), string: str); |
214 | if (i < 0) |
215 | return; |
216 | |
217 | ecrc_policy = i; |
218 | } |
219 | #endif /* CONFIG_PCIE_ECRC */ |
220 | |
221 | #define PCI_EXP_AER_FLAGS (PCI_EXP_DEVCTL_CERE | PCI_EXP_DEVCTL_NFERE | \ |
222 | PCI_EXP_DEVCTL_FERE | PCI_EXP_DEVCTL_URRE) |
223 | |
224 | int pcie_aer_is_native(struct pci_dev *dev) |
225 | { |
226 | struct pci_host_bridge *host = pci_find_host_bridge(bus: dev->bus); |
227 | |
228 | if (!dev->aer_cap) |
229 | return 0; |
230 | |
231 | return pcie_ports_native || host->native_aer; |
232 | } |
233 | EXPORT_SYMBOL_NS_GPL(pcie_aer_is_native, CXL); |
234 | |
235 | static int pci_enable_pcie_error_reporting(struct pci_dev *dev) |
236 | { |
237 | int rc; |
238 | |
239 | if (!pcie_aer_is_native(dev)) |
240 | return -EIO; |
241 | |
242 | rc = pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_AER_FLAGS); |
243 | return pcibios_err_to_errno(err: rc); |
244 | } |
245 | |
246 | int pci_aer_clear_nonfatal_status(struct pci_dev *dev) |
247 | { |
248 | int aer = dev->aer_cap; |
249 | u32 status, sev; |
250 | |
251 | if (!pcie_aer_is_native(dev)) |
252 | return -EIO; |
253 | |
254 | /* Clear status bits for ERR_NONFATAL errors only */ |
255 | pci_read_config_dword(dev, where: aer + PCI_ERR_UNCOR_STATUS, val: &status); |
256 | pci_read_config_dword(dev, where: aer + PCI_ERR_UNCOR_SEVER, val: &sev); |
257 | status &= ~sev; |
258 | if (status) |
259 | pci_write_config_dword(dev, where: aer + PCI_ERR_UNCOR_STATUS, val: status); |
260 | |
261 | return 0; |
262 | } |
263 | EXPORT_SYMBOL_GPL(pci_aer_clear_nonfatal_status); |
264 | |
265 | void pci_aer_clear_fatal_status(struct pci_dev *dev) |
266 | { |
267 | int aer = dev->aer_cap; |
268 | u32 status, sev; |
269 | |
270 | if (!pcie_aer_is_native(dev)) |
271 | return; |
272 | |
273 | /* Clear status bits for ERR_FATAL errors only */ |
274 | pci_read_config_dword(dev, where: aer + PCI_ERR_UNCOR_STATUS, val: &status); |
275 | pci_read_config_dword(dev, where: aer + PCI_ERR_UNCOR_SEVER, val: &sev); |
276 | status &= sev; |
277 | if (status) |
278 | pci_write_config_dword(dev, where: aer + PCI_ERR_UNCOR_STATUS, val: status); |
279 | } |
280 | |
281 | /** |
282 | * pci_aer_raw_clear_status - Clear AER error registers. |
283 | * @dev: the PCI device |
284 | * |
285 | * Clearing AER error status registers unconditionally, regardless of |
286 | * whether they're owned by firmware or the OS. |
287 | * |
288 | * Returns 0 on success, or negative on failure. |
289 | */ |
290 | int pci_aer_raw_clear_status(struct pci_dev *dev) |
291 | { |
292 | int aer = dev->aer_cap; |
293 | u32 status; |
294 | int port_type; |
295 | |
296 | if (!aer) |
297 | return -EIO; |
298 | |
299 | port_type = pci_pcie_type(dev); |
300 | if (port_type == PCI_EXP_TYPE_ROOT_PORT || |
301 | port_type == PCI_EXP_TYPE_RC_EC) { |
302 | pci_read_config_dword(dev, where: aer + PCI_ERR_ROOT_STATUS, val: &status); |
303 | pci_write_config_dword(dev, where: aer + PCI_ERR_ROOT_STATUS, val: status); |
304 | } |
305 | |
306 | pci_read_config_dword(dev, where: aer + PCI_ERR_COR_STATUS, val: &status); |
307 | pci_write_config_dword(dev, where: aer + PCI_ERR_COR_STATUS, val: status); |
308 | |
309 | pci_read_config_dword(dev, where: aer + PCI_ERR_UNCOR_STATUS, val: &status); |
310 | pci_write_config_dword(dev, where: aer + PCI_ERR_UNCOR_STATUS, val: status); |
311 | |
312 | return 0; |
313 | } |
314 | |
315 | int pci_aer_clear_status(struct pci_dev *dev) |
316 | { |
317 | if (!pcie_aer_is_native(dev)) |
318 | return -EIO; |
319 | |
320 | return pci_aer_raw_clear_status(dev); |
321 | } |
322 | |
323 | void pci_save_aer_state(struct pci_dev *dev) |
324 | { |
325 | int aer = dev->aer_cap; |
326 | struct pci_cap_saved_state *save_state; |
327 | u32 *cap; |
328 | |
329 | if (!aer) |
330 | return; |
331 | |
332 | save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_ERR); |
333 | if (!save_state) |
334 | return; |
335 | |
336 | cap = &save_state->cap.data[0]; |
337 | pci_read_config_dword(dev, where: aer + PCI_ERR_UNCOR_MASK, val: cap++); |
338 | pci_read_config_dword(dev, where: aer + PCI_ERR_UNCOR_SEVER, val: cap++); |
339 | pci_read_config_dword(dev, where: aer + PCI_ERR_COR_MASK, val: cap++); |
340 | pci_read_config_dword(dev, where: aer + PCI_ERR_CAP, val: cap++); |
341 | if (pcie_cap_has_rtctl(dev)) |
342 | pci_read_config_dword(dev, where: aer + PCI_ERR_ROOT_COMMAND, val: cap++); |
343 | } |
344 | |
345 | void pci_restore_aer_state(struct pci_dev *dev) |
346 | { |
347 | int aer = dev->aer_cap; |
348 | struct pci_cap_saved_state *save_state; |
349 | u32 *cap; |
350 | |
351 | if (!aer) |
352 | return; |
353 | |
354 | save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_ERR); |
355 | if (!save_state) |
356 | return; |
357 | |
358 | cap = &save_state->cap.data[0]; |
359 | pci_write_config_dword(dev, where: aer + PCI_ERR_UNCOR_MASK, val: *cap++); |
360 | pci_write_config_dword(dev, where: aer + PCI_ERR_UNCOR_SEVER, val: *cap++); |
361 | pci_write_config_dword(dev, where: aer + PCI_ERR_COR_MASK, val: *cap++); |
362 | pci_write_config_dword(dev, where: aer + PCI_ERR_CAP, val: *cap++); |
363 | if (pcie_cap_has_rtctl(dev)) |
364 | pci_write_config_dword(dev, where: aer + PCI_ERR_ROOT_COMMAND, val: *cap++); |
365 | } |
366 | |
367 | void pci_aer_init(struct pci_dev *dev) |
368 | { |
369 | int n; |
370 | |
371 | dev->aer_cap = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR); |
372 | if (!dev->aer_cap) |
373 | return; |
374 | |
375 | dev->aer_stats = kzalloc(size: sizeof(struct aer_stats), GFP_KERNEL); |
376 | |
377 | /* |
378 | * We save/restore PCI_ERR_UNCOR_MASK, PCI_ERR_UNCOR_SEVER, |
379 | * PCI_ERR_COR_MASK, and PCI_ERR_CAP. Root and Root Complex Event |
380 | * Collectors also implement PCI_ERR_ROOT_COMMAND (PCIe r5.0, sec |
381 | * 7.8.4). |
382 | */ |
383 | n = pcie_cap_has_rtctl(dev) ? 5 : 4; |
384 | pci_add_ext_cap_save_buffer(dev, PCI_EXT_CAP_ID_ERR, size: sizeof(u32) * n); |
385 | |
386 | pci_aer_clear_status(dev); |
387 | |
388 | if (pci_aer_available()) |
389 | pci_enable_pcie_error_reporting(dev); |
390 | |
391 | pcie_set_ecrc_checking(dev); |
392 | } |
393 | |
394 | void pci_aer_exit(struct pci_dev *dev) |
395 | { |
396 | kfree(objp: dev->aer_stats); |
397 | dev->aer_stats = NULL; |
398 | } |
399 | |
400 | #define AER_AGENT_RECEIVER 0 |
401 | #define AER_AGENT_REQUESTER 1 |
402 | #define AER_AGENT_COMPLETER 2 |
403 | #define AER_AGENT_TRANSMITTER 3 |
404 | |
405 | #define AER_AGENT_REQUESTER_MASK(t) ((t == AER_CORRECTABLE) ? \ |
406 | 0 : (PCI_ERR_UNC_COMP_TIME|PCI_ERR_UNC_UNSUP)) |
407 | #define AER_AGENT_COMPLETER_MASK(t) ((t == AER_CORRECTABLE) ? \ |
408 | 0 : PCI_ERR_UNC_COMP_ABORT) |
409 | #define AER_AGENT_TRANSMITTER_MASK(t) ((t == AER_CORRECTABLE) ? \ |
410 | (PCI_ERR_COR_REP_ROLL|PCI_ERR_COR_REP_TIMER) : 0) |
411 | |
412 | #define AER_GET_AGENT(t, e) \ |
413 | ((e & AER_AGENT_COMPLETER_MASK(t)) ? AER_AGENT_COMPLETER : \ |
414 | (e & AER_AGENT_REQUESTER_MASK(t)) ? AER_AGENT_REQUESTER : \ |
415 | (e & AER_AGENT_TRANSMITTER_MASK(t)) ? AER_AGENT_TRANSMITTER : \ |
416 | AER_AGENT_RECEIVER) |
417 | |
418 | #define AER_PHYSICAL_LAYER_ERROR 0 |
419 | #define AER_DATA_LINK_LAYER_ERROR 1 |
420 | #define AER_TRANSACTION_LAYER_ERROR 2 |
421 | |
422 | #define AER_PHYSICAL_LAYER_ERROR_MASK(t) ((t == AER_CORRECTABLE) ? \ |
423 | PCI_ERR_COR_RCVR : 0) |
424 | #define AER_DATA_LINK_LAYER_ERROR_MASK(t) ((t == AER_CORRECTABLE) ? \ |
425 | (PCI_ERR_COR_BAD_TLP| \ |
426 | PCI_ERR_COR_BAD_DLLP| \ |
427 | PCI_ERR_COR_REP_ROLL| \ |
428 | PCI_ERR_COR_REP_TIMER) : PCI_ERR_UNC_DLP) |
429 | |
430 | #define AER_GET_LAYER_ERROR(t, e) \ |
431 | ((e & AER_PHYSICAL_LAYER_ERROR_MASK(t)) ? AER_PHYSICAL_LAYER_ERROR : \ |
432 | (e & AER_DATA_LINK_LAYER_ERROR_MASK(t)) ? AER_DATA_LINK_LAYER_ERROR : \ |
433 | AER_TRANSACTION_LAYER_ERROR) |
434 | |
435 | /* |
436 | * AER error strings |
437 | */ |
438 | static const char * const aer_error_severity_string[] = { |
439 | "Uncorrectable (Non-Fatal)" , |
440 | "Uncorrectable (Fatal)" , |
441 | "Correctable" |
442 | }; |
443 | |
444 | static const char *aer_error_layer[] = { |
445 | "Physical Layer" , |
446 | "Data Link Layer" , |
447 | "Transaction Layer" |
448 | }; |
449 | |
450 | static const char *aer_correctable_error_string[] = { |
451 | "RxErr" , /* Bit Position 0 */ |
452 | NULL, |
453 | NULL, |
454 | NULL, |
455 | NULL, |
456 | NULL, |
457 | "BadTLP" , /* Bit Position 6 */ |
458 | "BadDLLP" , /* Bit Position 7 */ |
459 | "Rollover" , /* Bit Position 8 */ |
460 | NULL, |
461 | NULL, |
462 | NULL, |
463 | "Timeout" , /* Bit Position 12 */ |
464 | "NonFatalErr" , /* Bit Position 13 */ |
465 | "CorrIntErr" , /* Bit Position 14 */ |
466 | "HeaderOF" , /* Bit Position 15 */ |
467 | NULL, /* Bit Position 16 */ |
468 | NULL, /* Bit Position 17 */ |
469 | NULL, /* Bit Position 18 */ |
470 | NULL, /* Bit Position 19 */ |
471 | NULL, /* Bit Position 20 */ |
472 | NULL, /* Bit Position 21 */ |
473 | NULL, /* Bit Position 22 */ |
474 | NULL, /* Bit Position 23 */ |
475 | NULL, /* Bit Position 24 */ |
476 | NULL, /* Bit Position 25 */ |
477 | NULL, /* Bit Position 26 */ |
478 | NULL, /* Bit Position 27 */ |
479 | NULL, /* Bit Position 28 */ |
480 | NULL, /* Bit Position 29 */ |
481 | NULL, /* Bit Position 30 */ |
482 | NULL, /* Bit Position 31 */ |
483 | }; |
484 | |
485 | static const char *aer_uncorrectable_error_string[] = { |
486 | "Undefined" , /* Bit Position 0 */ |
487 | NULL, |
488 | NULL, |
489 | NULL, |
490 | "DLP" , /* Bit Position 4 */ |
491 | "SDES" , /* Bit Position 5 */ |
492 | NULL, |
493 | NULL, |
494 | NULL, |
495 | NULL, |
496 | NULL, |
497 | NULL, |
498 | "TLP" , /* Bit Position 12 */ |
499 | "FCP" , /* Bit Position 13 */ |
500 | "CmpltTO" , /* Bit Position 14 */ |
501 | "CmpltAbrt" , /* Bit Position 15 */ |
502 | "UnxCmplt" , /* Bit Position 16 */ |
503 | "RxOF" , /* Bit Position 17 */ |
504 | "MalfTLP" , /* Bit Position 18 */ |
505 | "ECRC" , /* Bit Position 19 */ |
506 | "UnsupReq" , /* Bit Position 20 */ |
507 | "ACSViol" , /* Bit Position 21 */ |
508 | "UncorrIntErr" , /* Bit Position 22 */ |
509 | "BlockedTLP" , /* Bit Position 23 */ |
510 | "AtomicOpBlocked" , /* Bit Position 24 */ |
511 | "TLPBlockedErr" , /* Bit Position 25 */ |
512 | "PoisonTLPBlocked" , /* Bit Position 26 */ |
513 | NULL, /* Bit Position 27 */ |
514 | NULL, /* Bit Position 28 */ |
515 | NULL, /* Bit Position 29 */ |
516 | NULL, /* Bit Position 30 */ |
517 | NULL, /* Bit Position 31 */ |
518 | }; |
519 | |
520 | static const char *aer_agent_string[] = { |
521 | "Receiver ID" , |
522 | "Requester ID" , |
523 | "Completer ID" , |
524 | "Transmitter ID" |
525 | }; |
526 | |
527 | #define aer_stats_dev_attr(name, stats_array, strings_array, \ |
528 | total_string, total_field) \ |
529 | static ssize_t \ |
530 | name##_show(struct device *dev, struct device_attribute *attr, \ |
531 | char *buf) \ |
532 | { \ |
533 | unsigned int i; \ |
534 | struct pci_dev *pdev = to_pci_dev(dev); \ |
535 | u64 *stats = pdev->aer_stats->stats_array; \ |
536 | size_t len = 0; \ |
537 | \ |
538 | for (i = 0; i < ARRAY_SIZE(pdev->aer_stats->stats_array); i++) {\ |
539 | if (strings_array[i]) \ |
540 | len += sysfs_emit_at(buf, len, "%s %llu\n", \ |
541 | strings_array[i], \ |
542 | stats[i]); \ |
543 | else if (stats[i]) \ |
544 | len += sysfs_emit_at(buf, len, \ |
545 | #stats_array "_bit[%d] %llu\n",\ |
546 | i, stats[i]); \ |
547 | } \ |
548 | len += sysfs_emit_at(buf, len, "TOTAL_%s %llu\n", total_string, \ |
549 | pdev->aer_stats->total_field); \ |
550 | return len; \ |
551 | } \ |
552 | static DEVICE_ATTR_RO(name) |
553 | |
554 | aer_stats_dev_attr(aer_dev_correctable, dev_cor_errs, |
555 | aer_correctable_error_string, "ERR_COR" , |
556 | dev_total_cor_errs); |
557 | aer_stats_dev_attr(aer_dev_fatal, dev_fatal_errs, |
558 | aer_uncorrectable_error_string, "ERR_FATAL" , |
559 | dev_total_fatal_errs); |
560 | aer_stats_dev_attr(aer_dev_nonfatal, dev_nonfatal_errs, |
561 | aer_uncorrectable_error_string, "ERR_NONFATAL" , |
562 | dev_total_nonfatal_errs); |
563 | |
564 | #define aer_stats_rootport_attr(name, field) \ |
565 | static ssize_t \ |
566 | name##_show(struct device *dev, struct device_attribute *attr, \ |
567 | char *buf) \ |
568 | { \ |
569 | struct pci_dev *pdev = to_pci_dev(dev); \ |
570 | return sysfs_emit(buf, "%llu\n", pdev->aer_stats->field); \ |
571 | } \ |
572 | static DEVICE_ATTR_RO(name) |
573 | |
574 | aer_stats_rootport_attr(aer_rootport_total_err_cor, |
575 | rootport_total_cor_errs); |
576 | aer_stats_rootport_attr(aer_rootport_total_err_fatal, |
577 | rootport_total_fatal_errs); |
578 | aer_stats_rootport_attr(aer_rootport_total_err_nonfatal, |
579 | rootport_total_nonfatal_errs); |
580 | |
581 | static struct attribute *aer_stats_attrs[] __ro_after_init = { |
582 | &dev_attr_aer_dev_correctable.attr, |
583 | &dev_attr_aer_dev_fatal.attr, |
584 | &dev_attr_aer_dev_nonfatal.attr, |
585 | &dev_attr_aer_rootport_total_err_cor.attr, |
586 | &dev_attr_aer_rootport_total_err_fatal.attr, |
587 | &dev_attr_aer_rootport_total_err_nonfatal.attr, |
588 | NULL |
589 | }; |
590 | |
591 | static umode_t aer_stats_attrs_are_visible(struct kobject *kobj, |
592 | struct attribute *a, int n) |
593 | { |
594 | struct device *dev = kobj_to_dev(kobj); |
595 | struct pci_dev *pdev = to_pci_dev(dev); |
596 | |
597 | if (!pdev->aer_stats) |
598 | return 0; |
599 | |
600 | if ((a == &dev_attr_aer_rootport_total_err_cor.attr || |
601 | a == &dev_attr_aer_rootport_total_err_fatal.attr || |
602 | a == &dev_attr_aer_rootport_total_err_nonfatal.attr) && |
603 | ((pci_pcie_type(dev: pdev) != PCI_EXP_TYPE_ROOT_PORT) && |
604 | (pci_pcie_type(dev: pdev) != PCI_EXP_TYPE_RC_EC))) |
605 | return 0; |
606 | |
607 | return a->mode; |
608 | } |
609 | |
610 | const struct attribute_group aer_stats_attr_group = { |
611 | .attrs = aer_stats_attrs, |
612 | .is_visible = aer_stats_attrs_are_visible, |
613 | }; |
614 | |
615 | static void pci_dev_aer_stats_incr(struct pci_dev *pdev, |
616 | struct aer_err_info *info) |
617 | { |
618 | unsigned long status = info->status & ~info->mask; |
619 | int i, max = -1; |
620 | u64 *counter = NULL; |
621 | struct aer_stats *aer_stats = pdev->aer_stats; |
622 | |
623 | if (!aer_stats) |
624 | return; |
625 | |
626 | switch (info->severity) { |
627 | case AER_CORRECTABLE: |
628 | aer_stats->dev_total_cor_errs++; |
629 | counter = &aer_stats->dev_cor_errs[0]; |
630 | max = AER_MAX_TYPEOF_COR_ERRS; |
631 | break; |
632 | case AER_NONFATAL: |
633 | aer_stats->dev_total_nonfatal_errs++; |
634 | counter = &aer_stats->dev_nonfatal_errs[0]; |
635 | max = AER_MAX_TYPEOF_UNCOR_ERRS; |
636 | break; |
637 | case AER_FATAL: |
638 | aer_stats->dev_total_fatal_errs++; |
639 | counter = &aer_stats->dev_fatal_errs[0]; |
640 | max = AER_MAX_TYPEOF_UNCOR_ERRS; |
641 | break; |
642 | } |
643 | |
644 | for_each_set_bit(i, &status, max) |
645 | counter[i]++; |
646 | } |
647 | |
648 | static void pci_rootport_aer_stats_incr(struct pci_dev *pdev, |
649 | struct aer_err_source *e_src) |
650 | { |
651 | struct aer_stats *aer_stats = pdev->aer_stats; |
652 | |
653 | if (!aer_stats) |
654 | return; |
655 | |
656 | if (e_src->status & PCI_ERR_ROOT_COR_RCV) |
657 | aer_stats->rootport_total_cor_errs++; |
658 | |
659 | if (e_src->status & PCI_ERR_ROOT_UNCOR_RCV) { |
660 | if (e_src->status & PCI_ERR_ROOT_FATAL_RCV) |
661 | aer_stats->rootport_total_fatal_errs++; |
662 | else |
663 | aer_stats->rootport_total_nonfatal_errs++; |
664 | } |
665 | } |
666 | |
667 | static void (struct pci_dev *dev, struct pcie_tlp_log *t) |
668 | { |
669 | pci_err(dev, " TLP Header: %08x %08x %08x %08x\n" , |
670 | t->dw[0], t->dw[1], t->dw[2], t->dw[3]); |
671 | } |
672 | |
673 | static void __aer_print_error(struct pci_dev *dev, |
674 | struct aer_err_info *info) |
675 | { |
676 | const char **strings; |
677 | unsigned long status = info->status & ~info->mask; |
678 | const char *level, *errmsg; |
679 | int i; |
680 | |
681 | if (info->severity == AER_CORRECTABLE) { |
682 | strings = aer_correctable_error_string; |
683 | level = KERN_WARNING; |
684 | } else { |
685 | strings = aer_uncorrectable_error_string; |
686 | level = KERN_ERR; |
687 | } |
688 | |
689 | for_each_set_bit(i, &status, 32) { |
690 | errmsg = strings[i]; |
691 | if (!errmsg) |
692 | errmsg = "Unknown Error Bit" ; |
693 | |
694 | pci_printk(level, dev, " [%2d] %-22s%s\n" , i, errmsg, |
695 | info->first_error == i ? " (First)" : "" ); |
696 | } |
697 | pci_dev_aer_stats_incr(pdev: dev, info); |
698 | } |
699 | |
700 | void aer_print_error(struct pci_dev *dev, struct aer_err_info *info) |
701 | { |
702 | int layer, agent; |
703 | int id = pci_dev_id(dev); |
704 | const char *level; |
705 | |
706 | if (!info->status) { |
707 | pci_err(dev, "PCIe Bus Error: severity=%s, type=Inaccessible, (Unregistered Agent ID)\n" , |
708 | aer_error_severity_string[info->severity]); |
709 | goto out; |
710 | } |
711 | |
712 | layer = AER_GET_LAYER_ERROR(info->severity, info->status); |
713 | agent = AER_GET_AGENT(info->severity, info->status); |
714 | |
715 | level = (info->severity == AER_CORRECTABLE) ? KERN_WARNING : KERN_ERR; |
716 | |
717 | pci_printk(level, dev, "PCIe Bus Error: severity=%s, type=%s, (%s)\n" , |
718 | aer_error_severity_string[info->severity], |
719 | aer_error_layer[layer], aer_agent_string[agent]); |
720 | |
721 | pci_printk(level, dev, " device [%04x:%04x] error status/mask=%08x/%08x\n" , |
722 | dev->vendor, dev->device, info->status, info->mask); |
723 | |
724 | __aer_print_error(dev, info); |
725 | |
726 | if (info->tlp_header_valid) |
727 | __print_tlp_header(dev, t: &info->tlp); |
728 | |
729 | out: |
730 | if (info->id && info->error_dev_num > 1 && info->id == id) |
731 | pci_err(dev, " Error of this Agent is reported first\n" ); |
732 | |
733 | trace_aer_event(dev_name: dev_name(dev: &dev->dev), status: (info->status & ~info->mask), |
734 | severity: info->severity, tlp_header_valid: info->tlp_header_valid, tlp: &info->tlp); |
735 | } |
736 | |
737 | static void aer_print_port_info(struct pci_dev *dev, struct aer_err_info *info) |
738 | { |
739 | u8 bus = info->id >> 8; |
740 | u8 devfn = info->id & 0xff; |
741 | |
742 | pci_info(dev, "%s%s error message received from %04x:%02x:%02x.%d\n" , |
743 | info->multi_error_valid ? "Multiple " : "" , |
744 | aer_error_severity_string[info->severity], |
745 | pci_domain_nr(dev->bus), bus, PCI_SLOT(devfn), |
746 | PCI_FUNC(devfn)); |
747 | } |
748 | |
749 | #ifdef CONFIG_ACPI_APEI_PCIEAER |
750 | int cper_severity_to_aer(int cper_severity) |
751 | { |
752 | switch (cper_severity) { |
753 | case CPER_SEV_RECOVERABLE: |
754 | return AER_NONFATAL; |
755 | case CPER_SEV_FATAL: |
756 | return AER_FATAL; |
757 | default: |
758 | return AER_CORRECTABLE; |
759 | } |
760 | } |
761 | EXPORT_SYMBOL_GPL(cper_severity_to_aer); |
762 | #endif |
763 | |
764 | void pci_print_aer(struct pci_dev *dev, int aer_severity, |
765 | struct aer_capability_regs *aer) |
766 | { |
767 | int layer, agent, = 0; |
768 | u32 status, mask; |
769 | struct aer_err_info info; |
770 | |
771 | if (aer_severity == AER_CORRECTABLE) { |
772 | status = aer->cor_status; |
773 | mask = aer->cor_mask; |
774 | } else { |
775 | status = aer->uncor_status; |
776 | mask = aer->uncor_mask; |
777 | tlp_header_valid = status & AER_LOG_TLP_MASKS; |
778 | } |
779 | |
780 | layer = AER_GET_LAYER_ERROR(aer_severity, status); |
781 | agent = AER_GET_AGENT(aer_severity, status); |
782 | |
783 | memset(&info, 0, sizeof(info)); |
784 | info.severity = aer_severity; |
785 | info.status = status; |
786 | info.mask = mask; |
787 | info.first_error = PCI_ERR_CAP_FEP(aer->cap_control); |
788 | |
789 | pci_err(dev, "aer_status: 0x%08x, aer_mask: 0x%08x\n" , status, mask); |
790 | __aer_print_error(dev, info: &info); |
791 | pci_err(dev, "aer_layer=%s, aer_agent=%s\n" , |
792 | aer_error_layer[layer], aer_agent_string[agent]); |
793 | |
794 | if (aer_severity != AER_CORRECTABLE) |
795 | pci_err(dev, "aer_uncor_severity: 0x%08x\n" , |
796 | aer->uncor_severity); |
797 | |
798 | if (tlp_header_valid) |
799 | __print_tlp_header(dev, t: &aer->header_log); |
800 | |
801 | trace_aer_event(dev_name: dev_name(dev: &dev->dev), status: (status & ~mask), |
802 | severity: aer_severity, tlp_header_valid, tlp: &aer->header_log); |
803 | } |
804 | EXPORT_SYMBOL_NS_GPL(pci_print_aer, CXL); |
805 | |
806 | /** |
807 | * add_error_device - list device to be handled |
808 | * @e_info: pointer to error info |
809 | * @dev: pointer to pci_dev to be added |
810 | */ |
811 | static int add_error_device(struct aer_err_info *e_info, struct pci_dev *dev) |
812 | { |
813 | if (e_info->error_dev_num < AER_MAX_MULTI_ERR_DEVICES) { |
814 | e_info->dev[e_info->error_dev_num] = pci_dev_get(dev); |
815 | e_info->error_dev_num++; |
816 | return 0; |
817 | } |
818 | return -ENOSPC; |
819 | } |
820 | |
821 | /** |
822 | * is_error_source - check whether the device is source of reported error |
823 | * @dev: pointer to pci_dev to be checked |
824 | * @e_info: pointer to reported error info |
825 | */ |
826 | static bool is_error_source(struct pci_dev *dev, struct aer_err_info *e_info) |
827 | { |
828 | int aer = dev->aer_cap; |
829 | u32 status, mask; |
830 | u16 reg16; |
831 | |
832 | /* |
833 | * When bus id is equal to 0, it might be a bad id |
834 | * reported by root port. |
835 | */ |
836 | if ((PCI_BUS_NUM(e_info->id) != 0) && |
837 | !(dev->bus->bus_flags & PCI_BUS_FLAGS_NO_AERSID)) { |
838 | /* Device ID match? */ |
839 | if (e_info->id == pci_dev_id(dev)) |
840 | return true; |
841 | |
842 | /* Continue id comparing if there is no multiple error */ |
843 | if (!e_info->multi_error_valid) |
844 | return false; |
845 | } |
846 | |
847 | /* |
848 | * When either |
849 | * 1) bus id is equal to 0. Some ports might lose the bus |
850 | * id of error source id; |
851 | * 2) bus flag PCI_BUS_FLAGS_NO_AERSID is set |
852 | * 3) There are multiple errors and prior ID comparing fails; |
853 | * We check AER status registers to find possible reporter. |
854 | */ |
855 | if (atomic_read(v: &dev->enable_cnt) == 0) |
856 | return false; |
857 | |
858 | /* Check if AER is enabled */ |
859 | pcie_capability_read_word(dev, PCI_EXP_DEVCTL, val: ®16); |
860 | if (!(reg16 & PCI_EXP_AER_FLAGS)) |
861 | return false; |
862 | |
863 | if (!aer) |
864 | return false; |
865 | |
866 | /* Check if error is recorded */ |
867 | if (e_info->severity == AER_CORRECTABLE) { |
868 | pci_read_config_dword(dev, where: aer + PCI_ERR_COR_STATUS, val: &status); |
869 | pci_read_config_dword(dev, where: aer + PCI_ERR_COR_MASK, val: &mask); |
870 | } else { |
871 | pci_read_config_dword(dev, where: aer + PCI_ERR_UNCOR_STATUS, val: &status); |
872 | pci_read_config_dword(dev, where: aer + PCI_ERR_UNCOR_MASK, val: &mask); |
873 | } |
874 | if (status & ~mask) |
875 | return true; |
876 | |
877 | return false; |
878 | } |
879 | |
880 | static int find_device_iter(struct pci_dev *dev, void *data) |
881 | { |
882 | struct aer_err_info *e_info = (struct aer_err_info *)data; |
883 | |
884 | if (is_error_source(dev, e_info)) { |
885 | /* List this device */ |
886 | if (add_error_device(e_info, dev)) { |
887 | /* We cannot handle more... Stop iteration */ |
888 | /* TODO: Should print error message here? */ |
889 | return 1; |
890 | } |
891 | |
892 | /* If there is only a single error, stop iteration */ |
893 | if (!e_info->multi_error_valid) |
894 | return 1; |
895 | } |
896 | return 0; |
897 | } |
898 | |
899 | /** |
900 | * find_source_device - search through device hierarchy for source device |
901 | * @parent: pointer to Root Port pci_dev data structure |
902 | * @e_info: including detailed error information such like id |
903 | * |
904 | * Return true if found. |
905 | * |
906 | * Invoked by DPC when error is detected at the Root Port. |
907 | * Caller of this function must set id, severity, and multi_error_valid of |
908 | * struct aer_err_info pointed by @e_info properly. This function must fill |
909 | * e_info->error_dev_num and e_info->dev[], based on the given information. |
910 | */ |
911 | static bool find_source_device(struct pci_dev *parent, |
912 | struct aer_err_info *e_info) |
913 | { |
914 | struct pci_dev *dev = parent; |
915 | int result; |
916 | |
917 | /* Must reset in this function */ |
918 | e_info->error_dev_num = 0; |
919 | |
920 | /* Is Root Port an agent that sends error message? */ |
921 | result = find_device_iter(dev, data: e_info); |
922 | if (result) |
923 | return true; |
924 | |
925 | if (pci_pcie_type(dev: parent) == PCI_EXP_TYPE_RC_EC) |
926 | pcie_walk_rcec(rcec: parent, cb: find_device_iter, userdata: e_info); |
927 | else |
928 | pci_walk_bus(top: parent->subordinate, cb: find_device_iter, userdata: e_info); |
929 | |
930 | if (!e_info->error_dev_num) { |
931 | u8 bus = e_info->id >> 8; |
932 | u8 devfn = e_info->id & 0xff; |
933 | |
934 | pci_info(parent, "found no error details for %04x:%02x:%02x.%d\n" , |
935 | pci_domain_nr(parent->bus), bus, PCI_SLOT(devfn), |
936 | PCI_FUNC(devfn)); |
937 | return false; |
938 | } |
939 | return true; |
940 | } |
941 | |
942 | #ifdef CONFIG_PCIEAER_CXL |
943 | |
944 | /** |
945 | * pci_aer_unmask_internal_errors - unmask internal errors |
946 | * @dev: pointer to the pcie_dev data structure |
947 | * |
948 | * Unmasks internal errors in the Uncorrectable and Correctable Error |
949 | * Mask registers. |
950 | * |
951 | * Note: AER must be enabled and supported by the device which must be |
952 | * checked in advance, e.g. with pcie_aer_is_native(). |
953 | */ |
954 | static void pci_aer_unmask_internal_errors(struct pci_dev *dev) |
955 | { |
956 | int aer = dev->aer_cap; |
957 | u32 mask; |
958 | |
959 | pci_read_config_dword(dev, where: aer + PCI_ERR_UNCOR_MASK, val: &mask); |
960 | mask &= ~PCI_ERR_UNC_INTN; |
961 | pci_write_config_dword(dev, where: aer + PCI_ERR_UNCOR_MASK, val: mask); |
962 | |
963 | pci_read_config_dword(dev, where: aer + PCI_ERR_COR_MASK, val: &mask); |
964 | mask &= ~PCI_ERR_COR_INTERNAL; |
965 | pci_write_config_dword(dev, where: aer + PCI_ERR_COR_MASK, val: mask); |
966 | } |
967 | |
968 | static bool is_cxl_mem_dev(struct pci_dev *dev) |
969 | { |
970 | /* |
971 | * The capability, status, and control fields in Device 0, |
972 | * Function 0 DVSEC control the CXL functionality of the |
973 | * entire device (CXL 3.0, 8.1.3). |
974 | */ |
975 | if (dev->devfn != PCI_DEVFN(0, 0)) |
976 | return false; |
977 | |
978 | /* |
979 | * CXL Memory Devices must have the 502h class code set (CXL |
980 | * 3.0, 8.1.12.1). |
981 | */ |
982 | if ((dev->class >> 8) != PCI_CLASS_MEMORY_CXL) |
983 | return false; |
984 | |
985 | return true; |
986 | } |
987 | |
988 | static bool cxl_error_is_native(struct pci_dev *dev) |
989 | { |
990 | struct pci_host_bridge *host = pci_find_host_bridge(bus: dev->bus); |
991 | |
992 | return (pcie_ports_native || host->native_aer); |
993 | } |
994 | |
995 | static bool is_internal_error(struct aer_err_info *info) |
996 | { |
997 | if (info->severity == AER_CORRECTABLE) |
998 | return info->status & PCI_ERR_COR_INTERNAL; |
999 | |
1000 | return info->status & PCI_ERR_UNC_INTN; |
1001 | } |
1002 | |
1003 | static int cxl_rch_handle_error_iter(struct pci_dev *dev, void *data) |
1004 | { |
1005 | struct aer_err_info *info = (struct aer_err_info *)data; |
1006 | const struct pci_error_handlers *err_handler; |
1007 | |
1008 | if (!is_cxl_mem_dev(dev) || !cxl_error_is_native(dev)) |
1009 | return 0; |
1010 | |
1011 | /* protect dev->driver */ |
1012 | device_lock(dev: &dev->dev); |
1013 | |
1014 | err_handler = dev->driver ? dev->driver->err_handler : NULL; |
1015 | if (!err_handler) |
1016 | goto out; |
1017 | |
1018 | if (info->severity == AER_CORRECTABLE) { |
1019 | if (err_handler->cor_error_detected) |
1020 | err_handler->cor_error_detected(dev); |
1021 | } else if (err_handler->error_detected) { |
1022 | if (info->severity == AER_NONFATAL) |
1023 | err_handler->error_detected(dev, pci_channel_io_normal); |
1024 | else if (info->severity == AER_FATAL) |
1025 | err_handler->error_detected(dev, pci_channel_io_frozen); |
1026 | } |
1027 | out: |
1028 | device_unlock(dev: &dev->dev); |
1029 | return 0; |
1030 | } |
1031 | |
1032 | static void cxl_rch_handle_error(struct pci_dev *dev, struct aer_err_info *info) |
1033 | { |
1034 | /* |
1035 | * Internal errors of an RCEC indicate an AER error in an |
1036 | * RCH's downstream port. Check and handle them in the CXL.mem |
1037 | * device driver. |
1038 | */ |
1039 | if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_EC && |
1040 | is_internal_error(info)) |
1041 | pcie_walk_rcec(rcec: dev, cb: cxl_rch_handle_error_iter, userdata: info); |
1042 | } |
1043 | |
1044 | static int handles_cxl_error_iter(struct pci_dev *dev, void *data) |
1045 | { |
1046 | bool *handles_cxl = data; |
1047 | |
1048 | if (!*handles_cxl) |
1049 | *handles_cxl = is_cxl_mem_dev(dev) && cxl_error_is_native(dev); |
1050 | |
1051 | /* Non-zero terminates iteration */ |
1052 | return *handles_cxl; |
1053 | } |
1054 | |
1055 | static bool handles_cxl_errors(struct pci_dev *rcec) |
1056 | { |
1057 | bool handles_cxl = false; |
1058 | |
1059 | if (pci_pcie_type(dev: rcec) == PCI_EXP_TYPE_RC_EC && |
1060 | pcie_aer_is_native(rcec)) |
1061 | pcie_walk_rcec(rcec, cb: handles_cxl_error_iter, userdata: &handles_cxl); |
1062 | |
1063 | return handles_cxl; |
1064 | } |
1065 | |
1066 | static void cxl_rch_enable_rcec(struct pci_dev *rcec) |
1067 | { |
1068 | if (!handles_cxl_errors(rcec)) |
1069 | return; |
1070 | |
1071 | pci_aer_unmask_internal_errors(dev: rcec); |
1072 | pci_info(rcec, "CXL: Internal errors unmasked" ); |
1073 | } |
1074 | |
1075 | #else |
1076 | static inline void cxl_rch_enable_rcec(struct pci_dev *dev) { } |
1077 | static inline void cxl_rch_handle_error(struct pci_dev *dev, |
1078 | struct aer_err_info *info) { } |
1079 | #endif |
1080 | |
1081 | /** |
1082 | * pci_aer_handle_error - handle logging error into an event log |
1083 | * @dev: pointer to pci_dev data structure of error source device |
1084 | * @info: comprehensive error information |
1085 | * |
1086 | * Invoked when an error being detected by Root Port. |
1087 | */ |
1088 | static void pci_aer_handle_error(struct pci_dev *dev, struct aer_err_info *info) |
1089 | { |
1090 | int aer = dev->aer_cap; |
1091 | |
1092 | if (info->severity == AER_CORRECTABLE) { |
1093 | /* |
1094 | * Correctable error does not need software intervention. |
1095 | * No need to go through error recovery process. |
1096 | */ |
1097 | if (aer) |
1098 | pci_write_config_dword(dev, where: aer + PCI_ERR_COR_STATUS, |
1099 | val: info->status); |
1100 | if (pcie_aer_is_native(dev)) { |
1101 | struct pci_driver *pdrv = dev->driver; |
1102 | |
1103 | if (pdrv && pdrv->err_handler && |
1104 | pdrv->err_handler->cor_error_detected) |
1105 | pdrv->err_handler->cor_error_detected(dev); |
1106 | pcie_clear_device_status(dev); |
1107 | } |
1108 | } else if (info->severity == AER_NONFATAL) |
1109 | pcie_do_recovery(dev, state: pci_channel_io_normal, reset_subordinates: aer_root_reset); |
1110 | else if (info->severity == AER_FATAL) |
1111 | pcie_do_recovery(dev, state: pci_channel_io_frozen, reset_subordinates: aer_root_reset); |
1112 | } |
1113 | |
1114 | static void handle_error_source(struct pci_dev *dev, struct aer_err_info *info) |
1115 | { |
1116 | cxl_rch_handle_error(dev, info); |
1117 | pci_aer_handle_error(dev, info); |
1118 | pci_dev_put(dev); |
1119 | } |
1120 | |
1121 | #ifdef CONFIG_ACPI_APEI_PCIEAER |
1122 | |
1123 | #define AER_RECOVER_RING_SIZE 16 |
1124 | |
1125 | struct aer_recover_entry { |
1126 | u8 bus; |
1127 | u8 devfn; |
1128 | u16 domain; |
1129 | int severity; |
1130 | struct aer_capability_regs *regs; |
1131 | }; |
1132 | |
1133 | static DEFINE_KFIFO(aer_recover_ring, struct aer_recover_entry, |
1134 | AER_RECOVER_RING_SIZE); |
1135 | |
1136 | static void aer_recover_work_func(struct work_struct *work) |
1137 | { |
1138 | struct aer_recover_entry entry; |
1139 | struct pci_dev *pdev; |
1140 | |
1141 | while (kfifo_get(&aer_recover_ring, &entry)) { |
1142 | pdev = pci_get_domain_bus_and_slot(domain: entry.domain, bus: entry.bus, |
1143 | devfn: entry.devfn); |
1144 | if (!pdev) { |
1145 | pr_err("no pci_dev for %04x:%02x:%02x.%x\n" , |
1146 | entry.domain, entry.bus, |
1147 | PCI_SLOT(entry.devfn), PCI_FUNC(entry.devfn)); |
1148 | continue; |
1149 | } |
1150 | pci_print_aer(pdev, entry.severity, entry.regs); |
1151 | /* |
1152 | * Memory for aer_capability_regs(entry.regs) is being allocated from the |
1153 | * ghes_estatus_pool to protect it from overwriting when multiple sections |
1154 | * are present in the error status. Thus free the same after processing |
1155 | * the data. |
1156 | */ |
1157 | ghes_estatus_pool_region_free(addr: (unsigned long)entry.regs, |
1158 | size: sizeof(struct aer_capability_regs)); |
1159 | |
1160 | if (entry.severity == AER_NONFATAL) |
1161 | pcie_do_recovery(dev: pdev, state: pci_channel_io_normal, |
1162 | reset_subordinates: aer_root_reset); |
1163 | else if (entry.severity == AER_FATAL) |
1164 | pcie_do_recovery(dev: pdev, state: pci_channel_io_frozen, |
1165 | reset_subordinates: aer_root_reset); |
1166 | pci_dev_put(dev: pdev); |
1167 | } |
1168 | } |
1169 | |
1170 | /* |
1171 | * Mutual exclusion for writers of aer_recover_ring, reader side don't |
1172 | * need lock, because there is only one reader and lock is not needed |
1173 | * between reader and writer. |
1174 | */ |
1175 | static DEFINE_SPINLOCK(aer_recover_ring_lock); |
1176 | static DECLARE_WORK(aer_recover_work, aer_recover_work_func); |
1177 | |
1178 | void aer_recover_queue(int domain, unsigned int bus, unsigned int devfn, |
1179 | int severity, struct aer_capability_regs *aer_regs) |
1180 | { |
1181 | struct aer_recover_entry entry = { |
1182 | .bus = bus, |
1183 | .devfn = devfn, |
1184 | .domain = domain, |
1185 | .severity = severity, |
1186 | .regs = aer_regs, |
1187 | }; |
1188 | |
1189 | if (kfifo_in_spinlocked(&aer_recover_ring, &entry, 1, |
1190 | &aer_recover_ring_lock)) |
1191 | schedule_work(work: &aer_recover_work); |
1192 | else |
1193 | pr_err("buffer overflow in recovery for %04x:%02x:%02x.%x\n" , |
1194 | domain, bus, PCI_SLOT(devfn), PCI_FUNC(devfn)); |
1195 | } |
1196 | EXPORT_SYMBOL_GPL(aer_recover_queue); |
1197 | #endif |
1198 | |
1199 | /** |
1200 | * aer_get_device_error_info - read error status from dev and store it to info |
1201 | * @dev: pointer to the device expected to have a error record |
1202 | * @info: pointer to structure to store the error record |
1203 | * |
1204 | * Return 1 on success, 0 on error. |
1205 | * |
1206 | * Note that @info is reused among all error devices. Clear fields properly. |
1207 | */ |
1208 | int aer_get_device_error_info(struct pci_dev *dev, struct aer_err_info *info) |
1209 | { |
1210 | int type = pci_pcie_type(dev); |
1211 | int aer = dev->aer_cap; |
1212 | u32 aercc; |
1213 | |
1214 | /* Must reset in this function */ |
1215 | info->status = 0; |
1216 | info->tlp_header_valid = 0; |
1217 | |
1218 | /* The device might not support AER */ |
1219 | if (!aer) |
1220 | return 0; |
1221 | |
1222 | if (info->severity == AER_CORRECTABLE) { |
1223 | pci_read_config_dword(dev, where: aer + PCI_ERR_COR_STATUS, |
1224 | val: &info->status); |
1225 | pci_read_config_dword(dev, where: aer + PCI_ERR_COR_MASK, |
1226 | val: &info->mask); |
1227 | if (!(info->status & ~info->mask)) |
1228 | return 0; |
1229 | } else if (type == PCI_EXP_TYPE_ROOT_PORT || |
1230 | type == PCI_EXP_TYPE_RC_EC || |
1231 | type == PCI_EXP_TYPE_DOWNSTREAM || |
1232 | info->severity == AER_NONFATAL) { |
1233 | |
1234 | /* Link is still healthy for IO reads */ |
1235 | pci_read_config_dword(dev, where: aer + PCI_ERR_UNCOR_STATUS, |
1236 | val: &info->status); |
1237 | pci_read_config_dword(dev, where: aer + PCI_ERR_UNCOR_MASK, |
1238 | val: &info->mask); |
1239 | if (!(info->status & ~info->mask)) |
1240 | return 0; |
1241 | |
1242 | /* Get First Error Pointer */ |
1243 | pci_read_config_dword(dev, where: aer + PCI_ERR_CAP, val: &aercc); |
1244 | info->first_error = PCI_ERR_CAP_FEP(aercc); |
1245 | |
1246 | if (info->status & AER_LOG_TLP_MASKS) { |
1247 | info->tlp_header_valid = 1; |
1248 | pcie_read_tlp_log(dev, where: aer + PCI_ERR_HEADER_LOG, log: &info->tlp); |
1249 | } |
1250 | } |
1251 | |
1252 | return 1; |
1253 | } |
1254 | |
1255 | static inline void aer_process_err_devices(struct aer_err_info *e_info) |
1256 | { |
1257 | int i; |
1258 | |
1259 | /* Report all before handle them, not to lost records by reset etc. */ |
1260 | for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) { |
1261 | if (aer_get_device_error_info(dev: e_info->dev[i], info: e_info)) |
1262 | aer_print_error(dev: e_info->dev[i], info: e_info); |
1263 | } |
1264 | for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) { |
1265 | if (aer_get_device_error_info(dev: e_info->dev[i], info: e_info)) |
1266 | handle_error_source(dev: e_info->dev[i], info: e_info); |
1267 | } |
1268 | } |
1269 | |
1270 | /** |
1271 | * aer_isr_one_error - consume an error detected by root port |
1272 | * @rpc: pointer to the root port which holds an error |
1273 | * @e_src: pointer to an error source |
1274 | */ |
1275 | static void aer_isr_one_error(struct aer_rpc *rpc, |
1276 | struct aer_err_source *e_src) |
1277 | { |
1278 | struct pci_dev *pdev = rpc->rpd; |
1279 | struct aer_err_info e_info; |
1280 | |
1281 | pci_rootport_aer_stats_incr(pdev, e_src); |
1282 | |
1283 | /* |
1284 | * There is a possibility that both correctable error and |
1285 | * uncorrectable error being logged. Report correctable error first. |
1286 | */ |
1287 | if (e_src->status & PCI_ERR_ROOT_COR_RCV) { |
1288 | e_info.id = ERR_COR_ID(e_src->id); |
1289 | e_info.severity = AER_CORRECTABLE; |
1290 | |
1291 | if (e_src->status & PCI_ERR_ROOT_MULTI_COR_RCV) |
1292 | e_info.multi_error_valid = 1; |
1293 | else |
1294 | e_info.multi_error_valid = 0; |
1295 | aer_print_port_info(dev: pdev, info: &e_info); |
1296 | |
1297 | if (find_source_device(parent: pdev, e_info: &e_info)) |
1298 | aer_process_err_devices(e_info: &e_info); |
1299 | } |
1300 | |
1301 | if (e_src->status & PCI_ERR_ROOT_UNCOR_RCV) { |
1302 | e_info.id = ERR_UNCOR_ID(e_src->id); |
1303 | |
1304 | if (e_src->status & PCI_ERR_ROOT_FATAL_RCV) |
1305 | e_info.severity = AER_FATAL; |
1306 | else |
1307 | e_info.severity = AER_NONFATAL; |
1308 | |
1309 | if (e_src->status & PCI_ERR_ROOT_MULTI_UNCOR_RCV) |
1310 | e_info.multi_error_valid = 1; |
1311 | else |
1312 | e_info.multi_error_valid = 0; |
1313 | |
1314 | aer_print_port_info(dev: pdev, info: &e_info); |
1315 | |
1316 | if (find_source_device(parent: pdev, e_info: &e_info)) |
1317 | aer_process_err_devices(e_info: &e_info); |
1318 | } |
1319 | } |
1320 | |
1321 | /** |
1322 | * aer_isr - consume errors detected by root port |
1323 | * @irq: IRQ assigned to Root Port |
1324 | * @context: pointer to Root Port data structure |
1325 | * |
1326 | * Invoked, as DPC, when root port records new detected error |
1327 | */ |
1328 | static irqreturn_t aer_isr(int irq, void *context) |
1329 | { |
1330 | struct pcie_device *dev = (struct pcie_device *)context; |
1331 | struct aer_rpc *rpc = get_service_data(dev); |
1332 | struct aer_err_source e_src; |
1333 | |
1334 | if (kfifo_is_empty(&rpc->aer_fifo)) |
1335 | return IRQ_NONE; |
1336 | |
1337 | while (kfifo_get(&rpc->aer_fifo, &e_src)) |
1338 | aer_isr_one_error(rpc, e_src: &e_src); |
1339 | return IRQ_HANDLED; |
1340 | } |
1341 | |
1342 | /** |
1343 | * aer_irq - Root Port's ISR |
1344 | * @irq: IRQ assigned to Root Port |
1345 | * @context: pointer to Root Port data structure |
1346 | * |
1347 | * Invoked when Root Port detects AER messages. |
1348 | */ |
1349 | static irqreturn_t aer_irq(int irq, void *context) |
1350 | { |
1351 | struct pcie_device *pdev = (struct pcie_device *)context; |
1352 | struct aer_rpc *rpc = get_service_data(dev: pdev); |
1353 | struct pci_dev *rp = rpc->rpd; |
1354 | int aer = rp->aer_cap; |
1355 | struct aer_err_source e_src = {}; |
1356 | |
1357 | pci_read_config_dword(dev: rp, where: aer + PCI_ERR_ROOT_STATUS, val: &e_src.status); |
1358 | if (!(e_src.status & AER_ERR_STATUS_MASK)) |
1359 | return IRQ_NONE; |
1360 | |
1361 | pci_read_config_dword(dev: rp, where: aer + PCI_ERR_ROOT_ERR_SRC, val: &e_src.id); |
1362 | pci_write_config_dword(dev: rp, where: aer + PCI_ERR_ROOT_STATUS, val: e_src.status); |
1363 | |
1364 | if (!kfifo_put(&rpc->aer_fifo, e_src)) |
1365 | return IRQ_HANDLED; |
1366 | |
1367 | return IRQ_WAKE_THREAD; |
1368 | } |
1369 | |
1370 | static void aer_enable_irq(struct pci_dev *pdev) |
1371 | { |
1372 | int aer = pdev->aer_cap; |
1373 | u32 reg32; |
1374 | |
1375 | /* Enable Root Port's interrupt in response to error messages */ |
1376 | pci_read_config_dword(dev: pdev, where: aer + PCI_ERR_ROOT_COMMAND, val: ®32); |
1377 | reg32 |= ROOT_PORT_INTR_ON_MESG_MASK; |
1378 | pci_write_config_dword(dev: pdev, where: aer + PCI_ERR_ROOT_COMMAND, val: reg32); |
1379 | } |
1380 | |
1381 | static void aer_disable_irq(struct pci_dev *pdev) |
1382 | { |
1383 | int aer = pdev->aer_cap; |
1384 | u32 reg32; |
1385 | |
1386 | /* Disable Root's interrupt in response to error messages */ |
1387 | pci_read_config_dword(dev: pdev, where: aer + PCI_ERR_ROOT_COMMAND, val: ®32); |
1388 | reg32 &= ~ROOT_PORT_INTR_ON_MESG_MASK; |
1389 | pci_write_config_dword(dev: pdev, where: aer + PCI_ERR_ROOT_COMMAND, val: reg32); |
1390 | } |
1391 | |
1392 | /** |
1393 | * aer_enable_rootport - enable Root Port's interrupts when receiving messages |
1394 | * @rpc: pointer to a Root Port data structure |
1395 | * |
1396 | * Invoked when PCIe bus loads AER service driver. |
1397 | */ |
1398 | static void aer_enable_rootport(struct aer_rpc *rpc) |
1399 | { |
1400 | struct pci_dev *pdev = rpc->rpd; |
1401 | int aer = pdev->aer_cap; |
1402 | u16 reg16; |
1403 | u32 reg32; |
1404 | |
1405 | /* Clear PCIe Capability's Device Status */ |
1406 | pcie_capability_read_word(dev: pdev, PCI_EXP_DEVSTA, val: ®16); |
1407 | pcie_capability_write_word(dev: pdev, PCI_EXP_DEVSTA, val: reg16); |
1408 | |
1409 | /* Disable system error generation in response to error messages */ |
1410 | pcie_capability_clear_word(dev: pdev, PCI_EXP_RTCTL, |
1411 | SYSTEM_ERROR_INTR_ON_MESG_MASK); |
1412 | |
1413 | /* Clear error status */ |
1414 | pci_read_config_dword(dev: pdev, where: aer + PCI_ERR_ROOT_STATUS, val: ®32); |
1415 | pci_write_config_dword(dev: pdev, where: aer + PCI_ERR_ROOT_STATUS, val: reg32); |
1416 | pci_read_config_dword(dev: pdev, where: aer + PCI_ERR_COR_STATUS, val: ®32); |
1417 | pci_write_config_dword(dev: pdev, where: aer + PCI_ERR_COR_STATUS, val: reg32); |
1418 | pci_read_config_dword(dev: pdev, where: aer + PCI_ERR_UNCOR_STATUS, val: ®32); |
1419 | pci_write_config_dword(dev: pdev, where: aer + PCI_ERR_UNCOR_STATUS, val: reg32); |
1420 | |
1421 | aer_enable_irq(pdev); |
1422 | } |
1423 | |
1424 | /** |
1425 | * aer_disable_rootport - disable Root Port's interrupts when receiving messages |
1426 | * @rpc: pointer to a Root Port data structure |
1427 | * |
1428 | * Invoked when PCIe bus unloads AER service driver. |
1429 | */ |
1430 | static void aer_disable_rootport(struct aer_rpc *rpc) |
1431 | { |
1432 | struct pci_dev *pdev = rpc->rpd; |
1433 | int aer = pdev->aer_cap; |
1434 | u32 reg32; |
1435 | |
1436 | aer_disable_irq(pdev); |
1437 | |
1438 | /* Clear Root's error status reg */ |
1439 | pci_read_config_dword(dev: pdev, where: aer + PCI_ERR_ROOT_STATUS, val: ®32); |
1440 | pci_write_config_dword(dev: pdev, where: aer + PCI_ERR_ROOT_STATUS, val: reg32); |
1441 | } |
1442 | |
1443 | /** |
1444 | * aer_remove - clean up resources |
1445 | * @dev: pointer to the pcie_dev data structure |
1446 | * |
1447 | * Invoked when PCI Express bus unloads or AER probe fails. |
1448 | */ |
1449 | static void aer_remove(struct pcie_device *dev) |
1450 | { |
1451 | struct aer_rpc *rpc = get_service_data(dev); |
1452 | |
1453 | aer_disable_rootport(rpc); |
1454 | } |
1455 | |
1456 | /** |
1457 | * aer_probe - initialize resources |
1458 | * @dev: pointer to the pcie_dev data structure |
1459 | * |
1460 | * Invoked when PCI Express bus loads AER service driver. |
1461 | */ |
1462 | static int aer_probe(struct pcie_device *dev) |
1463 | { |
1464 | int status; |
1465 | struct aer_rpc *rpc; |
1466 | struct device *device = &dev->device; |
1467 | struct pci_dev *port = dev->port; |
1468 | |
1469 | BUILD_BUG_ON(ARRAY_SIZE(aer_correctable_error_string) < |
1470 | AER_MAX_TYPEOF_COR_ERRS); |
1471 | BUILD_BUG_ON(ARRAY_SIZE(aer_uncorrectable_error_string) < |
1472 | AER_MAX_TYPEOF_UNCOR_ERRS); |
1473 | |
1474 | /* Limit to Root Ports or Root Complex Event Collectors */ |
1475 | if ((pci_pcie_type(dev: port) != PCI_EXP_TYPE_RC_EC) && |
1476 | (pci_pcie_type(dev: port) != PCI_EXP_TYPE_ROOT_PORT)) |
1477 | return -ENODEV; |
1478 | |
1479 | rpc = devm_kzalloc(dev: device, size: sizeof(struct aer_rpc), GFP_KERNEL); |
1480 | if (!rpc) |
1481 | return -ENOMEM; |
1482 | |
1483 | rpc->rpd = port; |
1484 | INIT_KFIFO(rpc->aer_fifo); |
1485 | set_service_data(dev, data: rpc); |
1486 | |
1487 | status = devm_request_threaded_irq(dev: device, irq: dev->irq, handler: aer_irq, thread_fn: aer_isr, |
1488 | IRQF_SHARED, devname: "aerdrv" , dev_id: dev); |
1489 | if (status) { |
1490 | pci_err(port, "request AER IRQ %d failed\n" , dev->irq); |
1491 | return status; |
1492 | } |
1493 | |
1494 | cxl_rch_enable_rcec(rcec: port); |
1495 | aer_enable_rootport(rpc); |
1496 | pci_info(port, "enabled with IRQ %d\n" , dev->irq); |
1497 | return 0; |
1498 | } |
1499 | |
1500 | /** |
1501 | * aer_root_reset - reset Root Port hierarchy, RCEC, or RCiEP |
1502 | * @dev: pointer to Root Port, RCEC, or RCiEP |
1503 | * |
1504 | * Invoked by Port Bus driver when performing reset. |
1505 | */ |
1506 | static pci_ers_result_t aer_root_reset(struct pci_dev *dev) |
1507 | { |
1508 | int type = pci_pcie_type(dev); |
1509 | struct pci_dev *root; |
1510 | int aer; |
1511 | struct pci_host_bridge *host = pci_find_host_bridge(bus: dev->bus); |
1512 | u32 reg32; |
1513 | int rc; |
1514 | |
1515 | /* |
1516 | * Only Root Ports and RCECs have AER Root Command and Root Status |
1517 | * registers. If "dev" is an RCiEP, the relevant registers are in |
1518 | * the RCEC. |
1519 | */ |
1520 | if (type == PCI_EXP_TYPE_RC_END) |
1521 | root = dev->rcec; |
1522 | else |
1523 | root = pcie_find_root_port(dev); |
1524 | |
1525 | /* |
1526 | * If the platform retained control of AER, an RCiEP may not have |
1527 | * an RCEC visible to us, so dev->rcec ("root") may be NULL. In |
1528 | * that case, firmware is responsible for these registers. |
1529 | */ |
1530 | aer = root ? root->aer_cap : 0; |
1531 | |
1532 | if ((host->native_aer || pcie_ports_native) && aer) |
1533 | aer_disable_irq(pdev: root); |
1534 | |
1535 | if (type == PCI_EXP_TYPE_RC_EC || type == PCI_EXP_TYPE_RC_END) { |
1536 | rc = pcie_reset_flr(dev, PCI_RESET_DO_RESET); |
1537 | if (!rc) |
1538 | pci_info(dev, "has been reset\n" ); |
1539 | else |
1540 | pci_info(dev, "not reset (no FLR support: %d)\n" , rc); |
1541 | } else { |
1542 | rc = pci_bus_error_reset(dev); |
1543 | pci_info(dev, "%s Port link has been reset (%d)\n" , |
1544 | pci_is_root_bus(dev->bus) ? "Root" : "Downstream" , rc); |
1545 | } |
1546 | |
1547 | if ((host->native_aer || pcie_ports_native) && aer) { |
1548 | /* Clear Root Error Status */ |
1549 | pci_read_config_dword(dev: root, where: aer + PCI_ERR_ROOT_STATUS, val: ®32); |
1550 | pci_write_config_dword(dev: root, where: aer + PCI_ERR_ROOT_STATUS, val: reg32); |
1551 | |
1552 | aer_enable_irq(pdev: root); |
1553 | } |
1554 | |
1555 | return rc ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED; |
1556 | } |
1557 | |
1558 | static struct pcie_port_service_driver aerdriver = { |
1559 | .name = "aer" , |
1560 | .port_type = PCIE_ANY_PORT, |
1561 | .service = PCIE_PORT_SERVICE_AER, |
1562 | |
1563 | .probe = aer_probe, |
1564 | .remove = aer_remove, |
1565 | }; |
1566 | |
1567 | /** |
1568 | * pcie_aer_init - register AER root service driver |
1569 | * |
1570 | * Invoked when AER root service driver is loaded. |
1571 | */ |
1572 | int __init pcie_aer_init(void) |
1573 | { |
1574 | if (!pci_aer_available()) |
1575 | return -ENXIO; |
1576 | return pcie_port_service_register(new: &aerdriver); |
1577 | } |
1578 | |