1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright(c) 2021 Intel Corporation. All rights reserved. */
3#include <linux/units.h>
4#include <linux/io-64-nonatomic-lo-hi.h>
5#include <linux/device.h>
6#include <linux/delay.h>
7#include <linux/pci.h>
8#include <linux/pci-doe.h>
9#include <linux/aer.h>
10#include <cxlpci.h>
11#include <cxlmem.h>
12#include <cxl.h>
13#include "core.h"
14#include "trace.h"
15
16/**
17 * DOC: cxl core pci
18 *
19 * Compute Express Link protocols are layered on top of PCIe. CXL core provides
20 * a set of helpers for CXL interactions which occur via PCIe.
21 */
22
23static unsigned short media_ready_timeout = 60;
24module_param(media_ready_timeout, ushort, 0644);
25MODULE_PARM_DESC(media_ready_timeout, "seconds to wait for media ready");
26
27struct cxl_walk_context {
28 struct pci_bus *bus;
29 struct cxl_port *port;
30 int type;
31 int error;
32 int count;
33};
34
35static int match_add_dports(struct pci_dev *pdev, void *data)
36{
37 struct cxl_walk_context *ctx = data;
38 struct cxl_port *port = ctx->port;
39 int type = pci_pcie_type(dev: pdev);
40 struct cxl_register_map map;
41 struct cxl_dport *dport;
42 u32 lnkcap, port_num;
43 int rc;
44
45 if (pdev->bus != ctx->bus)
46 return 0;
47 if (!pci_is_pcie(dev: pdev))
48 return 0;
49 if (type != ctx->type)
50 return 0;
51 if (pci_read_config_dword(dev: pdev, where: pci_pcie_cap(dev: pdev) + PCI_EXP_LNKCAP,
52 val: &lnkcap))
53 return 0;
54
55 rc = cxl_find_regblock(pdev, type: CXL_REGLOC_RBI_COMPONENT, map: &map);
56 if (rc)
57 dev_dbg(&port->dev, "failed to find component registers\n");
58
59 port_num = FIELD_GET(PCI_EXP_LNKCAP_PN, lnkcap);
60 dport = devm_cxl_add_dport(port, dport: &pdev->dev, port_id: port_num, component_reg_phys: map.resource);
61 if (IS_ERR(ptr: dport)) {
62 ctx->error = PTR_ERR(ptr: dport);
63 return PTR_ERR(ptr: dport);
64 }
65 ctx->count++;
66
67 return 0;
68}
69
70/**
71 * devm_cxl_port_enumerate_dports - enumerate downstream ports of the upstream port
72 * @port: cxl_port whose ->uport_dev is the upstream of dports to be enumerated
73 *
74 * Returns a positive number of dports enumerated or a negative error
75 * code.
76 */
77int devm_cxl_port_enumerate_dports(struct cxl_port *port)
78{
79 struct pci_bus *bus = cxl_port_to_pci_bus(port);
80 struct cxl_walk_context ctx;
81 int type;
82
83 if (!bus)
84 return -ENXIO;
85
86 if (pci_is_root_bus(pbus: bus))
87 type = PCI_EXP_TYPE_ROOT_PORT;
88 else
89 type = PCI_EXP_TYPE_DOWNSTREAM;
90
91 ctx = (struct cxl_walk_context) {
92 .port = port,
93 .bus = bus,
94 .type = type,
95 };
96 pci_walk_bus(top: bus, cb: match_add_dports, userdata: &ctx);
97
98 if (ctx.count == 0)
99 return -ENODEV;
100 if (ctx.error)
101 return ctx.error;
102 return ctx.count;
103}
104EXPORT_SYMBOL_NS_GPL(devm_cxl_port_enumerate_dports, "CXL");
105
106static int cxl_dvsec_mem_range_valid(struct cxl_dev_state *cxlds, int id)
107{
108 struct pci_dev *pdev = to_pci_dev(cxlds->dev);
109 int d = cxlds->cxl_dvsec;
110 bool valid = false;
111 int rc, i;
112 u32 temp;
113
114 if (id > CXL_DVSEC_RANGE_MAX)
115 return -EINVAL;
116
117 /* Check MEM INFO VALID bit first, give up after 1s */
118 i = 1;
119 do {
120 rc = pci_read_config_dword(dev: pdev,
121 where: d + CXL_DVSEC_RANGE_SIZE_LOW(id),
122 val: &temp);
123 if (rc)
124 return rc;
125
126 valid = FIELD_GET(CXL_DVSEC_MEM_INFO_VALID, temp);
127 if (valid)
128 break;
129 msleep(msecs: 1000);
130 } while (i--);
131
132 if (!valid) {
133 dev_err(&pdev->dev,
134 "Timeout awaiting memory range %d valid after 1s.\n",
135 id);
136 return -ETIMEDOUT;
137 }
138
139 return 0;
140}
141
142static int cxl_dvsec_mem_range_active(struct cxl_dev_state *cxlds, int id)
143{
144 struct pci_dev *pdev = to_pci_dev(cxlds->dev);
145 int d = cxlds->cxl_dvsec;
146 bool active = false;
147 int rc, i;
148 u32 temp;
149
150 if (id > CXL_DVSEC_RANGE_MAX)
151 return -EINVAL;
152
153 /* Check MEM ACTIVE bit, up to 60s timeout by default */
154 for (i = media_ready_timeout; i; i--) {
155 rc = pci_read_config_dword(
156 dev: pdev, where: d + CXL_DVSEC_RANGE_SIZE_LOW(id), val: &temp);
157 if (rc)
158 return rc;
159
160 active = FIELD_GET(CXL_DVSEC_MEM_ACTIVE, temp);
161 if (active)
162 break;
163 msleep(msecs: 1000);
164 }
165
166 if (!active) {
167 dev_err(&pdev->dev,
168 "timeout awaiting memory active after %d seconds\n",
169 media_ready_timeout);
170 return -ETIMEDOUT;
171 }
172
173 return 0;
174}
175
176/*
177 * Wait up to @media_ready_timeout for the device to report memory
178 * active.
179 */
180int cxl_await_media_ready(struct cxl_dev_state *cxlds)
181{
182 struct pci_dev *pdev = to_pci_dev(cxlds->dev);
183 int d = cxlds->cxl_dvsec;
184 int rc, i, hdm_count;
185 u64 md_status;
186 u16 cap;
187
188 rc = pci_read_config_word(dev: pdev,
189 where: d + CXL_DVSEC_CAP_OFFSET, val: &cap);
190 if (rc)
191 return rc;
192
193 hdm_count = FIELD_GET(CXL_DVSEC_HDM_COUNT_MASK, cap);
194 for (i = 0; i < hdm_count; i++) {
195 rc = cxl_dvsec_mem_range_valid(cxlds, id: i);
196 if (rc)
197 return rc;
198 }
199
200 for (i = 0; i < hdm_count; i++) {
201 rc = cxl_dvsec_mem_range_active(cxlds, id: i);
202 if (rc)
203 return rc;
204 }
205
206 md_status = readq(addr: cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET);
207 if (!CXLMDEV_READY(md_status))
208 return -EIO;
209
210 return 0;
211}
212EXPORT_SYMBOL_NS_GPL(cxl_await_media_ready, "CXL");
213
214static int cxl_set_mem_enable(struct cxl_dev_state *cxlds, u16 val)
215{
216 struct pci_dev *pdev = to_pci_dev(cxlds->dev);
217 int d = cxlds->cxl_dvsec;
218 u16 ctrl;
219 int rc;
220
221 rc = pci_read_config_word(dev: pdev, where: d + CXL_DVSEC_CTRL_OFFSET, val: &ctrl);
222 if (rc < 0)
223 return rc;
224
225 if ((ctrl & CXL_DVSEC_MEM_ENABLE) == val)
226 return 1;
227 ctrl &= ~CXL_DVSEC_MEM_ENABLE;
228 ctrl |= val;
229
230 rc = pci_write_config_word(dev: pdev, where: d + CXL_DVSEC_CTRL_OFFSET, val: ctrl);
231 if (rc < 0)
232 return rc;
233
234 return 0;
235}
236
237static void clear_mem_enable(void *cxlds)
238{
239 cxl_set_mem_enable(cxlds, val: 0);
240}
241
242static int devm_cxl_enable_mem(struct device *host, struct cxl_dev_state *cxlds)
243{
244 int rc;
245
246 rc = cxl_set_mem_enable(cxlds, CXL_DVSEC_MEM_ENABLE);
247 if (rc < 0)
248 return rc;
249 if (rc > 0)
250 return 0;
251 return devm_add_action_or_reset(host, clear_mem_enable, cxlds);
252}
253
254/* require dvsec ranges to be covered by a locked platform window */
255static int dvsec_range_allowed(struct device *dev, const void *arg)
256{
257 const struct range *dev_range = arg;
258 struct cxl_decoder *cxld;
259
260 if (!is_root_decoder(dev))
261 return 0;
262
263 cxld = to_cxl_decoder(dev);
264
265 if (!(cxld->flags & CXL_DECODER_F_RAM))
266 return 0;
267
268 return range_contains(r1: &cxld->hpa_range, r2: dev_range);
269}
270
271static void disable_hdm(void *_cxlhdm)
272{
273 u32 global_ctrl;
274 struct cxl_hdm *cxlhdm = _cxlhdm;
275 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
276
277 global_ctrl = readl(addr: hdm + CXL_HDM_DECODER_CTRL_OFFSET);
278 writel(val: global_ctrl & ~CXL_HDM_DECODER_ENABLE,
279 addr: hdm + CXL_HDM_DECODER_CTRL_OFFSET);
280}
281
282static int devm_cxl_enable_hdm(struct device *host, struct cxl_hdm *cxlhdm)
283{
284 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
285 u32 global_ctrl;
286
287 global_ctrl = readl(addr: hdm + CXL_HDM_DECODER_CTRL_OFFSET);
288 writel(val: global_ctrl | CXL_HDM_DECODER_ENABLE,
289 addr: hdm + CXL_HDM_DECODER_CTRL_OFFSET);
290
291 return devm_add_action_or_reset(host, disable_hdm, cxlhdm);
292}
293
294int cxl_dvsec_rr_decode(struct cxl_dev_state *cxlds,
295 struct cxl_endpoint_dvsec_info *info)
296{
297 struct pci_dev *pdev = to_pci_dev(cxlds->dev);
298 struct device *dev = cxlds->dev;
299 int hdm_count, rc, i, ranges = 0;
300 int d = cxlds->cxl_dvsec;
301 u16 cap, ctrl;
302
303 if (!d) {
304 dev_dbg(dev, "No DVSEC Capability\n");
305 return -ENXIO;
306 }
307
308 rc = pci_read_config_word(dev: pdev, where: d + CXL_DVSEC_CAP_OFFSET, val: &cap);
309 if (rc)
310 return rc;
311
312 if (!(cap & CXL_DVSEC_MEM_CAPABLE)) {
313 dev_dbg(dev, "Not MEM Capable\n");
314 return -ENXIO;
315 }
316
317 /*
318 * It is not allowed by spec for MEM.capable to be set and have 0 legacy
319 * HDM decoders (values > 2 are also undefined as of CXL 2.0). As this
320 * driver is for a spec defined class code which must be CXL.mem
321 * capable, there is no point in continuing to enable CXL.mem.
322 */
323 hdm_count = FIELD_GET(CXL_DVSEC_HDM_COUNT_MASK, cap);
324 if (!hdm_count || hdm_count > 2)
325 return -EINVAL;
326
327 /*
328 * The current DVSEC values are moot if the memory capability is
329 * disabled, and they will remain moot after the HDM Decoder
330 * capability is enabled.
331 */
332 rc = pci_read_config_word(dev: pdev, where: d + CXL_DVSEC_CTRL_OFFSET, val: &ctrl);
333 if (rc)
334 return rc;
335
336 info->mem_enabled = FIELD_GET(CXL_DVSEC_MEM_ENABLE, ctrl);
337 if (!info->mem_enabled)
338 return 0;
339
340 for (i = 0; i < hdm_count; i++) {
341 u64 base, size;
342 u32 temp;
343
344 rc = cxl_dvsec_mem_range_valid(cxlds, id: i);
345 if (rc)
346 return rc;
347
348 rc = pci_read_config_dword(
349 dev: pdev, where: d + CXL_DVSEC_RANGE_SIZE_HIGH(i), val: &temp);
350 if (rc)
351 return rc;
352
353 size = (u64)temp << 32;
354
355 rc = pci_read_config_dword(
356 dev: pdev, where: d + CXL_DVSEC_RANGE_SIZE_LOW(i), val: &temp);
357 if (rc)
358 return rc;
359
360 size |= temp & CXL_DVSEC_MEM_SIZE_LOW_MASK;
361 if (!size) {
362 continue;
363 }
364
365 rc = pci_read_config_dword(
366 dev: pdev, where: d + CXL_DVSEC_RANGE_BASE_HIGH(i), val: &temp);
367 if (rc)
368 return rc;
369
370 base = (u64)temp << 32;
371
372 rc = pci_read_config_dword(
373 dev: pdev, where: d + CXL_DVSEC_RANGE_BASE_LOW(i), val: &temp);
374 if (rc)
375 return rc;
376
377 base |= temp & CXL_DVSEC_MEM_BASE_LOW_MASK;
378
379 info->dvsec_range[ranges++] = (struct range) {
380 .start = base,
381 .end = base + size - 1
382 };
383 }
384
385 info->ranges = ranges;
386
387 return 0;
388}
389EXPORT_SYMBOL_NS_GPL(cxl_dvsec_rr_decode, "CXL");
390
391/**
392 * cxl_hdm_decode_init() - Setup HDM decoding for the endpoint
393 * @cxlds: Device state
394 * @cxlhdm: Mapped HDM decoder Capability
395 * @info: Cached DVSEC range registers info
396 *
397 * Try to enable the endpoint's HDM Decoder Capability
398 */
399int cxl_hdm_decode_init(struct cxl_dev_state *cxlds, struct cxl_hdm *cxlhdm,
400 struct cxl_endpoint_dvsec_info *info)
401{
402 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
403 struct cxl_port *port = cxlhdm->port;
404 struct device *dev = cxlds->dev;
405 struct cxl_port *root;
406 int i, rc, allowed;
407 u32 global_ctrl = 0;
408
409 if (hdm)
410 global_ctrl = readl(addr: hdm + CXL_HDM_DECODER_CTRL_OFFSET);
411
412 /*
413 * If the HDM Decoder Capability is already enabled then assume
414 * that some other agent like platform firmware set it up.
415 */
416 if (global_ctrl & CXL_HDM_DECODER_ENABLE || (!hdm && info->mem_enabled))
417 return devm_cxl_enable_mem(host: &port->dev, cxlds);
418
419 /*
420 * If the HDM Decoder Capability does not exist and DVSEC was
421 * not setup, the DVSEC based emulation cannot be used.
422 */
423 if (!hdm)
424 return -ENODEV;
425
426 /* The HDM Decoder Capability exists but is globally disabled. */
427
428 /*
429 * If the DVSEC CXL Range registers are not enabled, just
430 * enable and use the HDM Decoder Capability registers.
431 */
432 if (!info->mem_enabled) {
433 rc = devm_cxl_enable_hdm(host: &port->dev, cxlhdm);
434 if (rc)
435 return rc;
436
437 return devm_cxl_enable_mem(host: &port->dev, cxlds);
438 }
439
440 /*
441 * Per CXL 2.0 Section 8.1.3.8.3 and 8.1.3.8.4 DVSEC CXL Range 1 Base
442 * [High,Low] when HDM operation is enabled the range register values
443 * are ignored by the device, but the spec also recommends matching the
444 * DVSEC Range 1,2 to HDM Decoder Range 0,1. So, non-zero info->ranges
445 * are expected even though Linux does not require or maintain that
446 * match. Check if at least one DVSEC range is enabled and allowed by
447 * the platform. That is, the DVSEC range must be covered by a locked
448 * platform window (CFMWS). Fail otherwise as the endpoint's decoders
449 * cannot be used.
450 */
451
452 root = to_cxl_port(dev: port->dev.parent);
453 while (!is_cxl_root(port: root) && is_cxl_port(dev: root->dev.parent))
454 root = to_cxl_port(dev: root->dev.parent);
455 if (!is_cxl_root(port: root)) {
456 dev_err(dev, "Failed to acquire root port for HDM enable\n");
457 return -ENODEV;
458 }
459
460 for (i = 0, allowed = 0; i < info->ranges; i++) {
461 struct device *cxld_dev;
462
463 cxld_dev = device_find_child(parent: &root->dev, data: &info->dvsec_range[i],
464 match: dvsec_range_allowed);
465 if (!cxld_dev) {
466 dev_dbg(dev, "DVSEC Range%d denied by platform\n", i);
467 continue;
468 }
469 dev_dbg(dev, "DVSEC Range%d allowed by platform\n", i);
470 put_device(dev: cxld_dev);
471 allowed++;
472 }
473
474 if (!allowed) {
475 dev_err(dev, "Range register decodes outside platform defined CXL ranges.\n");
476 return -ENXIO;
477 }
478
479 return 0;
480}
481EXPORT_SYMBOL_NS_GPL(cxl_hdm_decode_init, "CXL");
482
483#define CXL_DOE_TABLE_ACCESS_REQ_CODE 0x000000ff
484#define CXL_DOE_TABLE_ACCESS_REQ_CODE_READ 0
485#define CXL_DOE_TABLE_ACCESS_TABLE_TYPE 0x0000ff00
486#define CXL_DOE_TABLE_ACCESS_TABLE_TYPE_CDATA 0
487#define CXL_DOE_TABLE_ACCESS_ENTRY_HANDLE 0xffff0000
488#define CXL_DOE_TABLE_ACCESS_LAST_ENTRY 0xffff
489#define CXL_DOE_PROTOCOL_TABLE_ACCESS 2
490
491#define CDAT_DOE_REQ(entry_handle) cpu_to_le32 \
492 (FIELD_PREP(CXL_DOE_TABLE_ACCESS_REQ_CODE, \
493 CXL_DOE_TABLE_ACCESS_REQ_CODE_READ) | \
494 FIELD_PREP(CXL_DOE_TABLE_ACCESS_TABLE_TYPE, \
495 CXL_DOE_TABLE_ACCESS_TABLE_TYPE_CDATA) | \
496 FIELD_PREP(CXL_DOE_TABLE_ACCESS_ENTRY_HANDLE, (entry_handle)))
497
498static int cxl_cdat_get_length(struct device *dev,
499 struct pci_doe_mb *doe_mb,
500 size_t *length)
501{
502 __le32 request = CDAT_DOE_REQ(0);
503 __le32 response[2];
504 int rc;
505
506 rc = pci_doe(doe_mb, PCI_VENDOR_ID_CXL,
507 CXL_DOE_PROTOCOL_TABLE_ACCESS,
508 request: &request, request_sz: sizeof(request),
509 response: &response, response_sz: sizeof(response));
510 if (rc < 0) {
511 dev_err(dev, "DOE failed: %d", rc);
512 return rc;
513 }
514 if (rc < sizeof(response))
515 return -EIO;
516
517 *length = le32_to_cpu(response[1]);
518 dev_dbg(dev, "CDAT length %zu\n", *length);
519
520 return 0;
521}
522
523static int cxl_cdat_read_table(struct device *dev,
524 struct pci_doe_mb *doe_mb,
525 struct cdat_doe_rsp *rsp, size_t *length)
526{
527 size_t received, remaining = *length;
528 unsigned int entry_handle = 0;
529 union cdat_data *data;
530 __le32 saved_dw = 0;
531
532 do {
533 __le32 request = CDAT_DOE_REQ(entry_handle);
534 int rc;
535
536 rc = pci_doe(doe_mb, PCI_VENDOR_ID_CXL,
537 CXL_DOE_PROTOCOL_TABLE_ACCESS,
538 request: &request, request_sz: sizeof(request),
539 response: rsp, response_sz: sizeof(*rsp) + remaining);
540 if (rc < 0) {
541 dev_err(dev, "DOE failed: %d", rc);
542 return rc;
543 }
544
545 if (rc < sizeof(*rsp))
546 return -EIO;
547
548 data = (union cdat_data *)rsp->data;
549 received = rc - sizeof(*rsp);
550
551 if (entry_handle == 0) {
552 if (received != sizeof(data->header))
553 return -EIO;
554 } else {
555 if (received < sizeof(data->entry) ||
556 received != le16_to_cpu(data->entry.length))
557 return -EIO;
558 }
559
560 /* Get the CXL table access header entry handle */
561 entry_handle = FIELD_GET(CXL_DOE_TABLE_ACCESS_ENTRY_HANDLE,
562 le32_to_cpu(rsp->doe_header));
563
564 /*
565 * Table Access Response Header overwrote the last DW of
566 * previous entry, so restore that DW
567 */
568 rsp->doe_header = saved_dw;
569 remaining -= received;
570 rsp = (void *)rsp + received;
571 saved_dw = rsp->doe_header;
572 } while (entry_handle != CXL_DOE_TABLE_ACCESS_LAST_ENTRY);
573
574 /* Length in CDAT header may exceed concatenation of CDAT entries */
575 *length -= remaining;
576
577 return 0;
578}
579
580static unsigned char cdat_checksum(void *buf, size_t size)
581{
582 unsigned char sum, *data = buf;
583 size_t i;
584
585 for (sum = 0, i = 0; i < size; i++)
586 sum += data[i];
587 return sum;
588}
589
590/**
591 * read_cdat_data - Read the CDAT data on this port
592 * @port: Port to read data from
593 *
594 * This call will sleep waiting for responses from the DOE mailbox.
595 */
596void read_cdat_data(struct cxl_port *port)
597{
598 struct device *uport = port->uport_dev;
599 struct device *dev = &port->dev;
600 struct pci_doe_mb *doe_mb;
601 struct pci_dev *pdev = NULL;
602 struct cxl_memdev *cxlmd;
603 struct cdat_doe_rsp *buf;
604 size_t table_length, length;
605 int rc;
606
607 if (is_cxl_memdev(dev: uport)) {
608 struct device *host;
609
610 cxlmd = to_cxl_memdev(dev: uport);
611 host = cxlmd->dev.parent;
612 if (dev_is_pci(host))
613 pdev = to_pci_dev(host);
614 } else if (dev_is_pci(uport)) {
615 pdev = to_pci_dev(uport);
616 }
617
618 if (!pdev)
619 return;
620
621 doe_mb = pci_find_doe_mailbox(pdev, PCI_VENDOR_ID_CXL,
622 CXL_DOE_PROTOCOL_TABLE_ACCESS);
623 if (!doe_mb) {
624 dev_dbg(dev, "No CDAT mailbox\n");
625 return;
626 }
627
628 port->cdat_available = true;
629
630 if (cxl_cdat_get_length(dev, doe_mb, length: &length)) {
631 dev_dbg(dev, "No CDAT length\n");
632 return;
633 }
634
635 /*
636 * The begin of the CDAT buffer needs space for additional 4
637 * bytes for the DOE header. Table data starts afterwards.
638 */
639 buf = devm_kzalloc(dev, size: sizeof(*buf) + length, GFP_KERNEL);
640 if (!buf)
641 goto err;
642
643 table_length = length;
644
645 rc = cxl_cdat_read_table(dev, doe_mb, rsp: buf, length: &length);
646 if (rc)
647 goto err;
648
649 if (table_length != length)
650 dev_warn(dev, "Malformed CDAT table length (%zu:%zu), discarding trailing data\n",
651 table_length, length);
652
653 if (cdat_checksum(buf: buf->data, size: length))
654 goto err;
655
656 port->cdat.table = buf->data;
657 port->cdat.length = length;
658
659 return;
660err:
661 /* Don't leave table data allocated on error */
662 devm_kfree(dev, p: buf);
663 dev_err(dev, "Failed to read/validate CDAT.\n");
664}
665EXPORT_SYMBOL_NS_GPL(read_cdat_data, "CXL");
666
667static void __cxl_handle_cor_ras(struct cxl_dev_state *cxlds,
668 void __iomem *ras_base)
669{
670 void __iomem *addr;
671 u32 status;
672
673 if (!ras_base)
674 return;
675
676 addr = ras_base + CXL_RAS_CORRECTABLE_STATUS_OFFSET;
677 status = readl(addr);
678 if (status & CXL_RAS_CORRECTABLE_STATUS_MASK) {
679 writel(val: status & CXL_RAS_CORRECTABLE_STATUS_MASK, addr);
680 trace_cxl_aer_correctable_error(cxlmd: cxlds->cxlmd, status);
681 }
682}
683
684static void cxl_handle_endpoint_cor_ras(struct cxl_dev_state *cxlds)
685{
686 return __cxl_handle_cor_ras(cxlds, ras_base: cxlds->regs.ras);
687}
688
689/* CXL spec rev3.0 8.2.4.16.1 */
690static void header_log_copy(void __iomem *ras_base, u32 *log)
691{
692 void __iomem *addr;
693 u32 *log_addr;
694 int i, log_u32_size = CXL_HEADERLOG_SIZE / sizeof(u32);
695
696 addr = ras_base + CXL_RAS_HEADER_LOG_OFFSET;
697 log_addr = log;
698
699 for (i = 0; i < log_u32_size; i++) {
700 *log_addr = readl(addr);
701 log_addr++;
702 addr += sizeof(u32);
703 }
704}
705
706/*
707 * Log the state of the RAS status registers and prepare them to log the
708 * next error status. Return 1 if reset needed.
709 */
710static bool __cxl_handle_ras(struct cxl_dev_state *cxlds,
711 void __iomem *ras_base)
712{
713 u32 hl[CXL_HEADERLOG_SIZE_U32];
714 void __iomem *addr;
715 u32 status;
716 u32 fe;
717
718 if (!ras_base)
719 return false;
720
721 addr = ras_base + CXL_RAS_UNCORRECTABLE_STATUS_OFFSET;
722 status = readl(addr);
723 if (!(status & CXL_RAS_UNCORRECTABLE_STATUS_MASK))
724 return false;
725
726 /* If multiple errors, log header points to first error from ctrl reg */
727 if (hweight32(status) > 1) {
728 void __iomem *rcc_addr =
729 ras_base + CXL_RAS_CAP_CONTROL_OFFSET;
730
731 fe = BIT(FIELD_GET(CXL_RAS_CAP_CONTROL_FE_MASK,
732 readl(rcc_addr)));
733 } else {
734 fe = status;
735 }
736
737 header_log_copy(ras_base, log: hl);
738 trace_cxl_aer_uncorrectable_error(cxlmd: cxlds->cxlmd, status, fe, hl);
739 writel(val: status & CXL_RAS_UNCORRECTABLE_STATUS_MASK, addr);
740
741 return true;
742}
743
744static bool cxl_handle_endpoint_ras(struct cxl_dev_state *cxlds)
745{
746 return __cxl_handle_ras(cxlds, ras_base: cxlds->regs.ras);
747}
748
749#ifdef CONFIG_PCIEAER_CXL
750
751static void cxl_dport_map_rch_aer(struct cxl_dport *dport)
752{
753 resource_size_t aer_phys;
754 struct device *host;
755 u16 aer_cap;
756
757 aer_cap = cxl_rcrb_to_aer(dev: dport->dport_dev, rcrb: dport->rcrb.base);
758 if (aer_cap) {
759 host = dport->reg_map.host;
760 aer_phys = aer_cap + dport->rcrb.base;
761 dport->regs.dport_aer = devm_cxl_iomap_block(dev: host, addr: aer_phys,
762 length: sizeof(struct aer_capability_regs));
763 }
764}
765
766static void cxl_dport_map_ras(struct cxl_dport *dport)
767{
768 struct cxl_register_map *map = &dport->reg_map;
769 struct device *dev = dport->dport_dev;
770
771 if (!map->component_map.ras.valid)
772 dev_dbg(dev, "RAS registers not found\n");
773 else if (cxl_map_component_regs(map, regs: &dport->regs.component,
774 BIT(CXL_CM_CAP_CAP_ID_RAS)))
775 dev_dbg(dev, "Failed to map RAS capability.\n");
776}
777
778static void cxl_disable_rch_root_ints(struct cxl_dport *dport)
779{
780 void __iomem *aer_base = dport->regs.dport_aer;
781 u32 aer_cmd_mask, aer_cmd;
782
783 if (!aer_base)
784 return;
785
786 /*
787 * Disable RCH root port command interrupts.
788 * CXL 3.0 12.2.1.1 - RCH Downstream Port-detected Errors
789 *
790 * This sequence may not be necessary. CXL spec states disabling
791 * the root cmd register's interrupts is required. But, PCI spec
792 * shows these are disabled by default on reset.
793 */
794 aer_cmd_mask = (PCI_ERR_ROOT_CMD_COR_EN |
795 PCI_ERR_ROOT_CMD_NONFATAL_EN |
796 PCI_ERR_ROOT_CMD_FATAL_EN);
797 aer_cmd = readl(addr: aer_base + PCI_ERR_ROOT_COMMAND);
798 aer_cmd &= ~aer_cmd_mask;
799 writel(val: aer_cmd, addr: aer_base + PCI_ERR_ROOT_COMMAND);
800}
801
802/**
803 * cxl_dport_init_ras_reporting - Setup CXL RAS report on this dport
804 * @dport: the cxl_dport that needs to be initialized
805 * @host: host device for devm operations
806 */
807void cxl_dport_init_ras_reporting(struct cxl_dport *dport, struct device *host)
808{
809 dport->reg_map.host = host;
810 cxl_dport_map_ras(dport);
811
812 if (dport->rch) {
813 struct pci_host_bridge *host_bridge = to_pci_host_bridge(dport->dport_dev);
814
815 if (!host_bridge->native_aer)
816 return;
817
818 cxl_dport_map_rch_aer(dport);
819 cxl_disable_rch_root_ints(dport);
820 }
821}
822EXPORT_SYMBOL_NS_GPL(cxl_dport_init_ras_reporting, "CXL");
823
824static void cxl_handle_rdport_cor_ras(struct cxl_dev_state *cxlds,
825 struct cxl_dport *dport)
826{
827 return __cxl_handle_cor_ras(cxlds, ras_base: dport->regs.ras);
828}
829
830static bool cxl_handle_rdport_ras(struct cxl_dev_state *cxlds,
831 struct cxl_dport *dport)
832{
833 return __cxl_handle_ras(cxlds, ras_base: dport->regs.ras);
834}
835
836/*
837 * Copy the AER capability registers using 32 bit read accesses.
838 * This is necessary because RCRB AER capability is MMIO mapped. Clear the
839 * status after copying.
840 *
841 * @aer_base: base address of AER capability block in RCRB
842 * @aer_regs: destination for copying AER capability
843 */
844static bool cxl_rch_get_aer_info(void __iomem *aer_base,
845 struct aer_capability_regs *aer_regs)
846{
847 int read_cnt = sizeof(struct aer_capability_regs) / sizeof(u32);
848 u32 *aer_regs_buf = (u32 *)aer_regs;
849 int n;
850
851 if (!aer_base)
852 return false;
853
854 /* Use readl() to guarantee 32-bit accesses */
855 for (n = 0; n < read_cnt; n++)
856 aer_regs_buf[n] = readl(addr: aer_base + n * sizeof(u32));
857
858 writel(val: aer_regs->uncor_status, addr: aer_base + PCI_ERR_UNCOR_STATUS);
859 writel(val: aer_regs->cor_status, addr: aer_base + PCI_ERR_COR_STATUS);
860
861 return true;
862}
863
864/* Get AER severity. Return false if there is no error. */
865static bool cxl_rch_get_aer_severity(struct aer_capability_regs *aer_regs,
866 int *severity)
867{
868 if (aer_regs->uncor_status & ~aer_regs->uncor_mask) {
869 if (aer_regs->uncor_status & PCI_ERR_ROOT_FATAL_RCV)
870 *severity = AER_FATAL;
871 else
872 *severity = AER_NONFATAL;
873 return true;
874 }
875
876 if (aer_regs->cor_status & ~aer_regs->cor_mask) {
877 *severity = AER_CORRECTABLE;
878 return true;
879 }
880
881 return false;
882}
883
884static void cxl_handle_rdport_errors(struct cxl_dev_state *cxlds)
885{
886 struct pci_dev *pdev = to_pci_dev(cxlds->dev);
887 struct aer_capability_regs aer_regs;
888 struct cxl_dport *dport;
889 int severity;
890
891 struct cxl_port *port __free(put_cxl_port) =
892 cxl_pci_find_port(pdev, dport: &dport);
893 if (!port)
894 return;
895
896 if (!cxl_rch_get_aer_info(aer_base: dport->regs.dport_aer, aer_regs: &aer_regs))
897 return;
898
899 if (!cxl_rch_get_aer_severity(aer_regs: &aer_regs, severity: &severity))
900 return;
901
902 pci_print_aer(dev: pdev, aer_severity: severity, aer: &aer_regs);
903
904 if (severity == AER_CORRECTABLE)
905 cxl_handle_rdport_cor_ras(cxlds, dport);
906 else
907 cxl_handle_rdport_ras(cxlds, dport);
908}
909
910#else
911static void cxl_handle_rdport_errors(struct cxl_dev_state *cxlds) { }
912#endif
913
914void cxl_cor_error_detected(struct pci_dev *pdev)
915{
916 struct cxl_dev_state *cxlds = pci_get_drvdata(pdev);
917 struct device *dev = &cxlds->cxlmd->dev;
918
919 scoped_guard(device, dev) {
920 if (!dev->driver) {
921 dev_warn(&pdev->dev,
922 "%s: memdev disabled, abort error handling\n",
923 dev_name(dev));
924 return;
925 }
926
927 if (cxlds->rcd)
928 cxl_handle_rdport_errors(cxlds);
929
930 cxl_handle_endpoint_cor_ras(cxlds);
931 }
932}
933EXPORT_SYMBOL_NS_GPL(cxl_cor_error_detected, "CXL");
934
935pci_ers_result_t cxl_error_detected(struct pci_dev *pdev,
936 pci_channel_state_t state)
937{
938 struct cxl_dev_state *cxlds = pci_get_drvdata(pdev);
939 struct cxl_memdev *cxlmd = cxlds->cxlmd;
940 struct device *dev = &cxlmd->dev;
941 bool ue;
942
943 scoped_guard(device, dev) {
944 if (!dev->driver) {
945 dev_warn(&pdev->dev,
946 "%s: memdev disabled, abort error handling\n",
947 dev_name(dev));
948 return PCI_ERS_RESULT_DISCONNECT;
949 }
950
951 if (cxlds->rcd)
952 cxl_handle_rdport_errors(cxlds);
953 /*
954 * A frozen channel indicates an impending reset which is fatal to
955 * CXL.mem operation, and will likely crash the system. On the off
956 * chance the situation is recoverable dump the status of the RAS
957 * capability registers and bounce the active state of the memdev.
958 */
959 ue = cxl_handle_endpoint_ras(cxlds);
960 }
961
962
963 switch (state) {
964 case pci_channel_io_normal:
965 if (ue) {
966 device_release_driver(dev);
967 return PCI_ERS_RESULT_NEED_RESET;
968 }
969 return PCI_ERS_RESULT_CAN_RECOVER;
970 case pci_channel_io_frozen:
971 dev_warn(&pdev->dev,
972 "%s: frozen state error detected, disable CXL.mem\n",
973 dev_name(dev));
974 device_release_driver(dev);
975 return PCI_ERS_RESULT_NEED_RESET;
976 case pci_channel_io_perm_failure:
977 dev_warn(&pdev->dev,
978 "failure state error detected, request disconnect\n");
979 return PCI_ERS_RESULT_DISCONNECT;
980 }
981 return PCI_ERS_RESULT_NEED_RESET;
982}
983EXPORT_SYMBOL_NS_GPL(cxl_error_detected, "CXL");
984
985static int cxl_flit_size(struct pci_dev *pdev)
986{
987 if (cxl_pci_flit_256(pdev))
988 return 256;
989
990 return 68;
991}
992
993/**
994 * cxl_pci_get_latency - calculate the link latency for the PCIe link
995 * @pdev: PCI device
996 *
997 * return: calculated latency or 0 for no latency
998 *
999 * CXL Memory Device SW Guide v1.0 2.11.4 Link latency calculation
1000 * Link latency = LinkPropagationLatency + FlitLatency + RetimerLatency
1001 * LinkProgationLatency is negligible, so 0 will be used
1002 * RetimerLatency is assumed to be negligible and 0 will be used
1003 * FlitLatency = FlitSize / LinkBandwidth
1004 * FlitSize is defined by spec. CXL rev3.0 4.2.1.
1005 * 68B flit is used up to 32GT/s. >32GT/s, 256B flit size is used.
1006 * The FlitLatency is converted to picoseconds.
1007 */
1008long cxl_pci_get_latency(struct pci_dev *pdev)
1009{
1010 long bw;
1011
1012 bw = pcie_link_speed_mbps(pdev);
1013 if (bw < 0)
1014 return 0;
1015 bw /= BITS_PER_BYTE;
1016
1017 return cxl_flit_size(pdev) * MEGA / bw;
1018}
1019
1020static int __cxl_endpoint_decoder_reset_detected(struct device *dev, void *data)
1021{
1022 struct cxl_port *port = data;
1023 struct cxl_decoder *cxld;
1024 struct cxl_hdm *cxlhdm;
1025 void __iomem *hdm;
1026 u32 ctrl;
1027
1028 if (!is_endpoint_decoder(dev))
1029 return 0;
1030
1031 cxld = to_cxl_decoder(dev);
1032 if ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)
1033 return 0;
1034
1035 cxlhdm = dev_get_drvdata(dev: &port->dev);
1036 hdm = cxlhdm->regs.hdm_decoder;
1037 ctrl = readl(addr: hdm + CXL_HDM_DECODER0_CTRL_OFFSET(cxld->id));
1038
1039 return !FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl);
1040}
1041
1042bool cxl_endpoint_decoder_reset_detected(struct cxl_port *port)
1043{
1044 return device_for_each_child(parent: &port->dev, data: port,
1045 fn: __cxl_endpoint_decoder_reset_detected);
1046}
1047EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_reset_detected, "CXL");
1048
1049int cxl_pci_get_bandwidth(struct pci_dev *pdev, struct access_coordinate *c)
1050{
1051 int speed, bw;
1052 u16 lnksta;
1053 u32 width;
1054
1055 speed = pcie_link_speed_mbps(pdev);
1056 if (speed < 0)
1057 return speed;
1058 speed /= BITS_PER_BYTE;
1059
1060 pcie_capability_read_word(dev: pdev, PCI_EXP_LNKSTA, val: &lnksta);
1061 width = FIELD_GET(PCI_EXP_LNKSTA_NLW, lnksta);
1062 bw = speed * width;
1063
1064 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
1065 c[i].read_bandwidth = bw;
1066 c[i].write_bandwidth = bw;
1067 }
1068
1069 return 0;
1070}
1071
1072/*
1073 * Set max timeout such that platforms will optimize GPF flow to avoid
1074 * the implied worst-case scenario delays. On a sane platform, all
1075 * devices should always complete GPF within the energy budget of
1076 * the GPF flow. The kernel does not have enough information to pick
1077 * anything better than "maximize timeouts and hope it works".
1078 *
1079 * A misbehaving device could block forward progress of GPF for all
1080 * the other devices, exhausting the energy budget of the platform.
1081 * However, the spec seems to assume that moving on from slow to respond
1082 * devices is a virtue. It is not possible to know that, in actuality,
1083 * the slow to respond device is *the* most critical device in the
1084 * system to wait.
1085 */
1086#define GPF_TIMEOUT_BASE_MAX 2
1087#define GPF_TIMEOUT_SCALE_MAX 7 /* 10 seconds */
1088
1089u16 cxl_gpf_get_dvsec(struct device *dev)
1090{
1091 struct pci_dev *pdev;
1092 bool is_port = true;
1093 u16 dvsec;
1094
1095 if (!dev_is_pci(dev))
1096 return 0;
1097
1098 pdev = to_pci_dev(dev);
1099 if (pci_pcie_type(dev: pdev) == PCI_EXP_TYPE_ENDPOINT)
1100 is_port = false;
1101
1102 dvsec = pci_find_dvsec_capability(dev: pdev, PCI_VENDOR_ID_CXL,
1103 dvsec: is_port ? CXL_DVSEC_PORT_GPF : CXL_DVSEC_DEVICE_GPF);
1104 if (!dvsec)
1105 dev_warn(dev, "%s GPF DVSEC not present\n",
1106 is_port ? "Port" : "Device");
1107 return dvsec;
1108}
1109EXPORT_SYMBOL_NS_GPL(cxl_gpf_get_dvsec, "CXL");
1110
1111static int update_gpf_port_dvsec(struct pci_dev *pdev, int dvsec, int phase)
1112{
1113 u64 base, scale;
1114 int rc, offset;
1115 u16 ctrl;
1116
1117 switch (phase) {
1118 case 1:
1119 offset = CXL_DVSEC_PORT_GPF_PHASE_1_CONTROL_OFFSET;
1120 base = CXL_DVSEC_PORT_GPF_PHASE_1_TMO_BASE_MASK;
1121 scale = CXL_DVSEC_PORT_GPF_PHASE_1_TMO_SCALE_MASK;
1122 break;
1123 case 2:
1124 offset = CXL_DVSEC_PORT_GPF_PHASE_2_CONTROL_OFFSET;
1125 base = CXL_DVSEC_PORT_GPF_PHASE_2_TMO_BASE_MASK;
1126 scale = CXL_DVSEC_PORT_GPF_PHASE_2_TMO_SCALE_MASK;
1127 break;
1128 default:
1129 return -EINVAL;
1130 }
1131
1132 rc = pci_read_config_word(dev: pdev, where: dvsec + offset, val: &ctrl);
1133 if (rc)
1134 return rc;
1135
1136 if (FIELD_GET(base, ctrl) == GPF_TIMEOUT_BASE_MAX &&
1137 FIELD_GET(scale, ctrl) == GPF_TIMEOUT_SCALE_MAX)
1138 return 0;
1139
1140 ctrl = FIELD_PREP(base, GPF_TIMEOUT_BASE_MAX);
1141 ctrl |= FIELD_PREP(scale, GPF_TIMEOUT_SCALE_MAX);
1142
1143 rc = pci_write_config_word(dev: pdev, where: dvsec + offset, val: ctrl);
1144 if (!rc)
1145 pci_dbg(pdev, "Port GPF phase %d timeout: %d0 secs\n",
1146 phase, GPF_TIMEOUT_BASE_MAX);
1147
1148 return rc;
1149}
1150
1151int cxl_gpf_port_setup(struct cxl_dport *dport)
1152{
1153 if (!dport)
1154 return -EINVAL;
1155
1156 if (!dport->gpf_dvsec) {
1157 struct pci_dev *pdev;
1158 int dvsec;
1159
1160 dvsec = cxl_gpf_get_dvsec(dport->dport_dev);
1161 if (!dvsec)
1162 return -EINVAL;
1163
1164 dport->gpf_dvsec = dvsec;
1165 pdev = to_pci_dev(dport->dport_dev);
1166 update_gpf_port_dvsec(pdev, dvsec: dport->gpf_dvsec, phase: 1);
1167 update_gpf_port_dvsec(pdev, dvsec: dport->gpf_dvsec, phase: 2);
1168 }
1169
1170 return 0;
1171}
1172

Provided by KDAB

Privacy Policy
Improve your Profiling and Debugging skills
Find out more

source code of linux/drivers/cxl/core/pci.c