1	// SPDX-License-Identifier: GPL-2.0-only
2	/* Copyright(c) 2020 Intel Corporation. All rights reserved. */
3	#include <linux/platform_device.h>
4	#include <linux/memregion.h>
5	#include <linux/workqueue.h>
6	#include <linux/debugfs.h>
7	#include <linux/device.h>
8	#include <linux/module.h>
9	#include <linux/pci.h>
10	#include <linux/slab.h>
11	#include <linux/idr.h>
12	#include <linux/node.h>
13	#include <cxl/einj.h>
14	#include <cxlmem.h>
15	#include <cxlpci.h>
16	#include <cxl.h>
17	#include "core.h"
18
19	/**
20	* DOC: cxl core
21	*
22	* The CXL core provides a set of interfaces that can be consumed by CXL aware
23	* drivers. The interfaces allow for creation, modification, and destruction of
24	* regions, memory devices, ports, and decoders. CXL aware drivers must register
25	* with the CXL core via these interfaces in order to be able to participate in
26	* cross-device interleave coordination. The CXL core also establishes and
27	* maintains the bridge to the nvdimm subsystem.
28	*
29	* CXL core introduces sysfs hierarchy to control the devices that are
30	* instantiated by the core.
31	*/
32
33	static DEFINE_IDA(cxl_port_ida);
34	static DEFINE_XARRAY(cxl_root_buses);
35
36	/*
37	* The terminal device in PCI is NULL and @platform_bus
38	* for platform devices (for cxl_test)
39	*/
40	static bool is_cxl_host_bridge(struct device *dev)
41	{
42	return (!dev || dev == &platform_bus);
43	}
44
45	int cxl_num_decoders_committed(struct cxl_port *port)
46	{
47	lockdep_assert_held(&cxl_rwsem.region);
48
49	return port->commit_end + 1;
50	}
51
52	static ssize_t devtype_show(struct device *dev, struct device_attribute *attr,
53	char *buf)
54	{
55	return sysfs_emit(buf, fmt: "%s\n", dev->type->name);
56	}
57	static DEVICE_ATTR_RO(devtype);
58
59	static int cxl_device_id(const struct device *dev)
60	{
61	if (dev->type == &cxl_nvdimm_bridge_type)
62	return CXL_DEVICE_NVDIMM_BRIDGE;
63	if (dev->type == &cxl_nvdimm_type)
64	return CXL_DEVICE_NVDIMM;
65	if (dev->type == CXL_PMEM_REGION_TYPE())
66	return CXL_DEVICE_PMEM_REGION;
67	if (dev->type == CXL_DAX_REGION_TYPE())
68	return CXL_DEVICE_DAX_REGION;
69	if (is_cxl_port(dev)) {
70	if (is_cxl_root(port: to_cxl_port(dev)))
71	return CXL_DEVICE_ROOT;
72	return CXL_DEVICE_PORT;
73	}
74	if (is_cxl_memdev(dev))
75	return CXL_DEVICE_MEMORY_EXPANDER;
76	if (dev->type == CXL_REGION_TYPE())
77	return CXL_DEVICE_REGION;
78	if (dev->type == &cxl_pmu_type)
79	return CXL_DEVICE_PMU;
80	return 0;
81	}
82
83	static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
84	char *buf)
85	{
86	return sysfs_emit(buf, CXL_MODALIAS_FMT "\n", cxl_device_id(dev));
87	}
88	static DEVICE_ATTR_RO(modalias);
89
90	static struct attribute *cxl_base_attributes[] = {
91	&dev_attr_devtype.attr,
92	&dev_attr_modalias.attr,
93	NULL,
94	};
95
96	struct attribute_group cxl_base_attribute_group = {
97	.attrs = cxl_base_attributes,
98	};
99
100	static ssize_t start_show(struct device *dev, struct device_attribute *attr,
101	char *buf)
102	{
103	struct cxl_decoder *cxld = to_cxl_decoder(dev);
104
105	return sysfs_emit(buf, fmt: "%#llx\n", cxld->hpa_range.start);
106	}
107	static DEVICE_ATTR_ADMIN_RO(start);
108
109	static ssize_t size_show(struct device *dev, struct device_attribute *attr,
110	char *buf)
111	{
112	struct cxl_decoder *cxld = to_cxl_decoder(dev);
113
114	return sysfs_emit(buf, fmt: "%#llx\n", range_len(range: &cxld->hpa_range));
115	}
116	static DEVICE_ATTR_RO(size);
117
118	#define CXL_DECODER_FLAG_ATTR(name, flag) \
119	static ssize_t name##_show(struct device *dev, \
120	struct device_attribute *attr, char *buf) \
121	{ \
122	struct cxl_decoder *cxld = to_cxl_decoder(dev); \
123	\
124	return sysfs_emit(buf, "%s\n", \
125	(cxld->flags & (flag)) ? "1" : "0"); \
126	} \
127	static DEVICE_ATTR_RO(name)
128
129	CXL_DECODER_FLAG_ATTR(cap_pmem, CXL_DECODER_F_PMEM);
130	CXL_DECODER_FLAG_ATTR(cap_ram, CXL_DECODER_F_RAM);
131	CXL_DECODER_FLAG_ATTR(cap_type2, CXL_DECODER_F_TYPE2);
132	CXL_DECODER_FLAG_ATTR(cap_type3, CXL_DECODER_F_TYPE3);
133	CXL_DECODER_FLAG_ATTR(locked, CXL_DECODER_F_LOCK);
134
135	static ssize_t target_type_show(struct device *dev,
136	struct device_attribute *attr, char *buf)
137	{
138	struct cxl_decoder *cxld = to_cxl_decoder(dev);
139
140	switch (cxld->target_type) {
141	case CXL_DECODER_DEVMEM:
142	return sysfs_emit(buf, fmt: "accelerator\n");
143	case CXL_DECODER_HOSTONLYMEM:
144	return sysfs_emit(buf, fmt: "expander\n");
145	}
146	return -ENXIO;
147	}
148	static DEVICE_ATTR_RO(target_type);
149
150	static ssize_t emit_target_list(struct cxl_switch_decoder *cxlsd, char *buf)
151	{
152	struct cxl_decoder *cxld = &cxlsd->cxld;
153	ssize_t offset = 0;
154	int i, rc = 0;
155
156	for (i = 0; i < cxld->interleave_ways; i++) {
157	struct cxl_dport *dport = cxlsd->target[i];
158	struct cxl_dport *next = NULL;
159
160	if (!dport)
161	break;
162
163	if (i + 1 < cxld->interleave_ways)
164	next = cxlsd->target[i + 1];
165	rc = sysfs_emit_at(buf, at: offset, fmt: "%d%s", dport->port_id,
166	next ? "," : "");
167	if (rc < 0)
168	return rc;
169	offset += rc;
170	}
171
172	return offset;
173	}
174
175	static ssize_t target_list_show(struct device *dev,
176	struct device_attribute *attr, char *buf)
177	{
178	struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
179	ssize_t offset;
180	int rc;
181
182	guard(rwsem_read)(T: &cxl_rwsem.region);
183	rc = emit_target_list(cxlsd, buf);
184	if (rc < 0)
185	return rc;
186	offset = rc;
187
188	rc = sysfs_emit_at(buf, at: offset, fmt: "\n");
189	if (rc < 0)
190	return rc;
191
192	return offset + rc;
193	}
194	static DEVICE_ATTR_RO(target_list);
195
196	static ssize_t mode_show(struct device *dev, struct device_attribute *attr,
197	char *buf)
198	{
199	struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
200	struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
201	struct cxl_dev_state *cxlds = cxlmd->cxlds;
202	/* without @cxl_rwsem.dpa, make sure @part is not reloaded */
203	int part = READ_ONCE(cxled->part);
204	const char *desc;
205
206	if (part < 0)
207	desc = "none";
208	else
209	desc = cxlds->part[part].res.name;
210
211	return sysfs_emit(buf, fmt: "%s\n", desc);
212	}
213
214	static ssize_t mode_store(struct device *dev, struct device_attribute *attr,
215	const char *buf, size_t len)
216	{
217	struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
218	enum cxl_partition_mode mode;
219	ssize_t rc;
220
221	if (sysfs_streq(s1: buf, s2: "pmem"))
222	mode = CXL_PARTMODE_PMEM;
223	else if (sysfs_streq(s1: buf, s2: "ram"))
224	mode = CXL_PARTMODE_RAM;
225	else
226	return -EINVAL;
227
228	rc = cxl_dpa_set_part(cxled, mode);
229	if (rc)
230	return rc;
231
232	return len;
233	}
234	static DEVICE_ATTR_RW(mode);
235
236	static ssize_t dpa_resource_show(struct device *dev, struct device_attribute *attr,
237	char *buf)
238	{
239	struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
240
241	guard(rwsem_read)(T: &cxl_rwsem.dpa);
242	return sysfs_emit(buf, fmt: "%#llx\n", (u64)cxl_dpa_resource_start(cxled));
243	}
244	static DEVICE_ATTR_RO(dpa_resource);
245
246	static ssize_t dpa_size_show(struct device *dev, struct device_attribute *attr,
247	char *buf)
248	{
249	struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
250	resource_size_t size = cxl_dpa_size(cxled);
251
252	return sysfs_emit(buf, fmt: "%pa\n", &size);
253	}
254
255	static ssize_t dpa_size_store(struct device *dev, struct device_attribute *attr,
256	const char *buf, size_t len)
257	{
258	struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
259	unsigned long long size;
260	ssize_t rc;
261
262	rc = kstrtoull(s: buf, base: 0, res: &size);
263	if (rc)
264	return rc;
265
266	if (!IS_ALIGNED(size, SZ_256M))
267	return -EINVAL;
268
269	rc = cxl_dpa_free(cxled);
270	if (rc)
271	return rc;
272
273	if (size == 0)
274	return len;
275
276	rc = cxl_dpa_alloc(cxled, size);
277	if (rc)
278	return rc;
279
280	return len;
281	}
282	static DEVICE_ATTR_RW(dpa_size);
283
284	static ssize_t interleave_granularity_show(struct device *dev,
285	struct device_attribute *attr,
286	char *buf)
287	{
288	struct cxl_decoder *cxld = to_cxl_decoder(dev);
289
290	return sysfs_emit(buf, fmt: "%d\n", cxld->interleave_granularity);
291	}
292
293	static DEVICE_ATTR_RO(interleave_granularity);
294
295	static ssize_t interleave_ways_show(struct device *dev,
296	struct device_attribute *attr, char *buf)
297	{
298	struct cxl_decoder *cxld = to_cxl_decoder(dev);
299
300	return sysfs_emit(buf, fmt: "%d\n", cxld->interleave_ways);
301	}
302
303	static DEVICE_ATTR_RO(interleave_ways);
304
305	static ssize_t qos_class_show(struct device *dev,
306	struct device_attribute *attr, char *buf)
307	{
308	struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
309
310	return sysfs_emit(buf, fmt: "%d\n", cxlrd->qos_class);
311	}
312	static DEVICE_ATTR_RO(qos_class);
313
314	static struct attribute *cxl_decoder_base_attrs[] = {
315	&dev_attr_start.attr,
316	&dev_attr_size.attr,
317	&dev_attr_locked.attr,
318	&dev_attr_interleave_granularity.attr,
319	&dev_attr_interleave_ways.attr,
320	NULL,
321	};
322
323	static struct attribute_group cxl_decoder_base_attribute_group = {
324	.attrs = cxl_decoder_base_attrs,
325	};
326
327	static struct attribute *cxl_decoder_root_attrs[] = {
328	&dev_attr_cap_pmem.attr,
329	&dev_attr_cap_ram.attr,
330	&dev_attr_cap_type2.attr,
331	&dev_attr_cap_type3.attr,
332	&dev_attr_target_list.attr,
333	&dev_attr_qos_class.attr,
334	SET_CXL_REGION_ATTR(create_pmem_region)
335	SET_CXL_REGION_ATTR(create_ram_region)
336	SET_CXL_REGION_ATTR(delete_region)
337	NULL,
338	};
339
340	static bool can_create_pmem(struct cxl_root_decoder *cxlrd)
341	{
342	unsigned long flags = CXL_DECODER_F_TYPE3 | CXL_DECODER_F_PMEM;
343
344	return (cxlrd->cxlsd.cxld.flags & flags) == flags;
345	}
346
347	static bool can_create_ram(struct cxl_root_decoder *cxlrd)
348	{
349	unsigned long flags = CXL_DECODER_F_TYPE3 | CXL_DECODER_F_RAM;
350
351	return (cxlrd->cxlsd.cxld.flags & flags) == flags;
352	}
353
354	static umode_t cxl_root_decoder_visible(struct kobject *kobj, struct attribute *a, int n)
355	{
356	struct device *dev = kobj_to_dev(kobj);
357	struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
358
359	if (a == CXL_REGION_ATTR(create_pmem_region) && !can_create_pmem(cxlrd))
360	return 0;
361
362	if (a == CXL_REGION_ATTR(create_ram_region) && !can_create_ram(cxlrd))
363	return 0;
364
365	if (a == CXL_REGION_ATTR(delete_region) &&
366	!(can_create_pmem(cxlrd) || can_create_ram(cxlrd)))
367	return 0;
368
369	return a->mode;
370	}
371
372	static struct attribute_group cxl_decoder_root_attribute_group = {
373	.attrs = cxl_decoder_root_attrs,
374	.is_visible = cxl_root_decoder_visible,
375	};
376
377	static const struct attribute_group *cxl_decoder_root_attribute_groups[] = {
378	&cxl_decoder_root_attribute_group,
379	&cxl_decoder_base_attribute_group,
380	&cxl_base_attribute_group,
381	NULL,
382	};
383
384	static struct attribute *cxl_decoder_switch_attrs[] = {
385	&dev_attr_target_type.attr,
386	&dev_attr_target_list.attr,
387	SET_CXL_REGION_ATTR(region)
388	NULL,
389	};
390
391	static struct attribute_group cxl_decoder_switch_attribute_group = {
392	.attrs = cxl_decoder_switch_attrs,
393	};
394
395	static const struct attribute_group *cxl_decoder_switch_attribute_groups[] = {
396	&cxl_decoder_switch_attribute_group,
397	&cxl_decoder_base_attribute_group,
398	&cxl_base_attribute_group,
399	NULL,
400	};
401
402	static struct attribute *cxl_decoder_endpoint_attrs[] = {
403	&dev_attr_target_type.attr,
404	&dev_attr_mode.attr,
405	&dev_attr_dpa_size.attr,
406	&dev_attr_dpa_resource.attr,
407	SET_CXL_REGION_ATTR(region)
408	NULL,
409	};
410
411	static struct attribute_group cxl_decoder_endpoint_attribute_group = {
412	.attrs = cxl_decoder_endpoint_attrs,
413	};
414
415	static const struct attribute_group *cxl_decoder_endpoint_attribute_groups[] = {
416	&cxl_decoder_base_attribute_group,
417	&cxl_decoder_endpoint_attribute_group,
418	&cxl_base_attribute_group,
419	NULL,
420	};
421
422	static void __cxl_decoder_release(struct cxl_decoder *cxld)
423	{
424	struct cxl_port *port = to_cxl_port(dev: cxld->dev.parent);
425
426	ida_free(&port->decoder_ida, id: cxld->id);
427	put_device(dev: &port->dev);
428	}
429
430	static void cxl_endpoint_decoder_release(struct device *dev)
431	{
432	struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
433
434	__cxl_decoder_release(cxld: &cxled->cxld);
435	kfree(objp: cxled);
436	}
437
438	static void cxl_switch_decoder_release(struct device *dev)
439	{
440	struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
441
442	__cxl_decoder_release(cxld: &cxlsd->cxld);
443	kfree(objp: cxlsd);
444	}
445
446	struct cxl_root_decoder *to_cxl_root_decoder(struct device *dev)
447	{
448	if (dev_WARN_ONCE(dev, !is_root_decoder(dev),
449	"not a cxl_root_decoder device\n"))
450	return NULL;
451	return container_of(dev, struct cxl_root_decoder, cxlsd.cxld.dev);
452	}
453	EXPORT_SYMBOL_NS_GPL(to_cxl_root_decoder, "CXL");
454
455	static void cxl_root_decoder_release(struct device *dev)
456	{
457	struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
458
459	if (atomic_read(v: &cxlrd->region_id) >= 0)
460	memregion_free(id: atomic_read(v: &cxlrd->region_id));
461	__cxl_decoder_release(cxld: &cxlrd->cxlsd.cxld);
462	kfree(objp: cxlrd);
463	}
464
465	static const struct device_type cxl_decoder_endpoint_type = {
466	.name = "cxl_decoder_endpoint",
467	.release = cxl_endpoint_decoder_release,
468	.groups = cxl_decoder_endpoint_attribute_groups,
469	};
470
471	static const struct device_type cxl_decoder_switch_type = {
472	.name = "cxl_decoder_switch",
473	.release = cxl_switch_decoder_release,
474	.groups = cxl_decoder_switch_attribute_groups,
475	};
476
477	static const struct device_type cxl_decoder_root_type = {
478	.name = "cxl_decoder_root",
479	.release = cxl_root_decoder_release,
480	.groups = cxl_decoder_root_attribute_groups,
481	};
482
483	bool is_endpoint_decoder(struct device *dev)
484	{
485	return dev->type == &cxl_decoder_endpoint_type;
486	}
487	EXPORT_SYMBOL_NS_GPL(is_endpoint_decoder, "CXL");
488
489	bool is_root_decoder(struct device *dev)
490	{
491	return dev->type == &cxl_decoder_root_type;
492	}
493	EXPORT_SYMBOL_NS_GPL(is_root_decoder, "CXL");
494
495	bool is_switch_decoder(struct device *dev)
496	{
497	return is_root_decoder(dev) || dev->type == &cxl_decoder_switch_type;
498	}
499	EXPORT_SYMBOL_NS_GPL(is_switch_decoder, "CXL");
500
501	struct cxl_decoder *to_cxl_decoder(struct device *dev)
502	{
503	if (dev_WARN_ONCE(dev,
504	!is_switch_decoder(dev) && !is_endpoint_decoder(dev),
505	"not a cxl_decoder device\n"))
506	return NULL;
507	return container_of(dev, struct cxl_decoder, dev);
508	}
509	EXPORT_SYMBOL_NS_GPL(to_cxl_decoder, "CXL");
510
511	struct cxl_endpoint_decoder *to_cxl_endpoint_decoder(struct device *dev)
512	{
513	if (dev_WARN_ONCE(dev, !is_endpoint_decoder(dev),
514	"not a cxl_endpoint_decoder device\n"))
515	return NULL;
516	return container_of(dev, struct cxl_endpoint_decoder, cxld.dev);
517	}
518	EXPORT_SYMBOL_NS_GPL(to_cxl_endpoint_decoder, "CXL");
519
520	struct cxl_switch_decoder *to_cxl_switch_decoder(struct device *dev)
521	{
522	if (dev_WARN_ONCE(dev, !is_switch_decoder(dev),
523	"not a cxl_switch_decoder device\n"))
524	return NULL;
525	return container_of(dev, struct cxl_switch_decoder, cxld.dev);
526	}
527	EXPORT_SYMBOL_NS_GPL(to_cxl_switch_decoder, "CXL");
528
529	static void cxl_ep_release(struct cxl_ep *ep)
530	{
531	put_device(dev: ep->ep);
532	kfree(objp: ep);
533	}
534
535	static void cxl_ep_remove(struct cxl_port *port, struct cxl_ep *ep)
536	{
537	if (!ep)
538	return;
539	xa_erase(&port->endpoints, index: (unsigned long) ep->ep);
540	cxl_ep_release(ep);
541	}
542
543	static void cxl_port_release(struct device *dev)
544	{
545	struct cxl_port *port = to_cxl_port(dev);
546	unsigned long index;
547	struct cxl_ep *ep;
548
549	xa_for_each(&port->endpoints, index, ep)
550	cxl_ep_remove(port, ep);
551	xa_destroy(&port->endpoints);
552	xa_destroy(&port->dports);
553	xa_destroy(&port->regions);
554	ida_free(&cxl_port_ida, id: port->id);
555	if (is_cxl_root(port))
556	kfree(objp: to_cxl_root(port));
557	else
558	kfree(objp: port);
559	}
560
561	static ssize_t decoders_committed_show(struct device *dev,
562	struct device_attribute *attr, char *buf)
563	{
564	struct cxl_port *port = to_cxl_port(dev);
565
566	guard(rwsem_read)(T: &cxl_rwsem.region);
567	return sysfs_emit(buf, fmt: "%d\n", cxl_num_decoders_committed(port));
568	}
569
570	static DEVICE_ATTR_RO(decoders_committed);
571
572	static struct attribute *cxl_port_attrs[] = {
573	&dev_attr_decoders_committed.attr,
574	NULL,
575	};
576
577	static struct attribute_group cxl_port_attribute_group = {
578	.attrs = cxl_port_attrs,
579	};
580
581	static const struct attribute_group *cxl_port_attribute_groups[] = {
582	&cxl_base_attribute_group,
583	&cxl_port_attribute_group,
584	NULL,
585	};
586
587	static const struct device_type cxl_port_type = {
588	.name = "cxl_port",
589	.release = cxl_port_release,
590	.groups = cxl_port_attribute_groups,
591	};
592
593	bool is_cxl_port(const struct device *dev)
594	{
595	return dev->type == &cxl_port_type;
596	}
597	EXPORT_SYMBOL_NS_GPL(is_cxl_port, "CXL");
598
599	struct cxl_port *to_cxl_port(const struct device *dev)
600	{
601	if (dev_WARN_ONCE(dev, dev->type != &cxl_port_type,
602	"not a cxl_port device\n"))
603	return NULL;
604	return container_of(dev, struct cxl_port, dev);
605	}
606	EXPORT_SYMBOL_NS_GPL(to_cxl_port, "CXL");
607
608	struct cxl_port *parent_port_of(struct cxl_port *port)
609	{
610	if (!port || !port->parent_dport)
611	return NULL;
612	return port->parent_dport->port;
613	}
614
615	static void unregister_port(void *_port)
616	{
617	struct cxl_port *port = _port;
618	struct cxl_port *parent = parent_port_of(port);
619	struct device *lock_dev;
620
621	/*
622	* CXL root port's and the first level of ports are unregistered
623	* under the platform firmware device lock, all other ports are
624	* unregistered while holding their parent port lock.
625	*/
626	if (!parent)
627	lock_dev = port->uport_dev;
628	else if (is_cxl_root(port: parent))
629	lock_dev = parent->uport_dev;
630	else
631	lock_dev = &parent->dev;
632
633	device_lock_assert(dev: lock_dev);
634	port->dead = true;
635	device_unregister(dev: &port->dev);
636	}
637
638	static void cxl_unlink_uport(void *_port)
639	{
640	struct cxl_port *port = _port;
641
642	sysfs_remove_link(kobj: &port->dev.kobj, name: "uport");
643	}
644
645	static int devm_cxl_link_uport(struct device *host, struct cxl_port *port)
646	{
647	int rc;
648
649	rc = sysfs_create_link(kobj: &port->dev.kobj, target: &port->uport_dev->kobj,
650	name: "uport");
651	if (rc)
652	return rc;
653	return devm_add_action_or_reset(host, cxl_unlink_uport, port);
654	}
655
656	static void cxl_unlink_parent_dport(void *_port)
657	{
658	struct cxl_port *port = _port;
659
660	sysfs_remove_link(kobj: &port->dev.kobj, name: "parent_dport");
661	}
662
663	static int devm_cxl_link_parent_dport(struct device *host,
664	struct cxl_port *port,
665	struct cxl_dport *parent_dport)
666	{
667	int rc;
668
669	if (!parent_dport)
670	return 0;
671
672	rc = sysfs_create_link(kobj: &port->dev.kobj, target: &parent_dport->dport_dev->kobj,
673	name: "parent_dport");
674	if (rc)
675	return rc;
676	return devm_add_action_or_reset(host, cxl_unlink_parent_dport, port);
677	}
678
679	static struct lock_class_key cxl_port_key;
680
681	static struct cxl_port *cxl_port_alloc(struct device *uport_dev,
682	struct cxl_dport *parent_dport)
683	{
684	struct cxl_root *cxl_root __free(kfree) = NULL;
685	struct cxl_port *port, *_port __free(kfree) = NULL;
686	struct device *dev;
687	int rc;
688
689	/* No parent_dport, root cxl_port */
690	if (!parent_dport) {
691	cxl_root = kzalloc(sizeof(*cxl_root), GFP_KERNEL);
692	if (!cxl_root)
693	return ERR_PTR(error: -ENOMEM);
694	} else {
695	_port = kzalloc(sizeof(*port), GFP_KERNEL);
696	if (!_port)
697	return ERR_PTR(error: -ENOMEM);
698	}
699
700	rc = ida_alloc(ida: &cxl_port_ida, GFP_KERNEL);
701	if (rc < 0)
702	return ERR_PTR(error: rc);
703
704	if (cxl_root)
705	port = &no_free_ptr(cxl_root)->port;
706	else
707	port = no_free_ptr(_port);
708
709	port->id = rc;
710	port->uport_dev = uport_dev;
711
712	/*
713	* The top-level cxl_port "cxl_root" does not have a cxl_port as
714	* its parent and it does not have any corresponding component
715	* registers as its decode is described by a fixed platform
716	* description.
717	*/
718	dev = &port->dev;
719	if (parent_dport) {
720	struct cxl_port *parent_port = parent_dport->port;
721	struct cxl_port *iter;
722
723	dev->parent = &parent_port->dev;
724	port->depth = parent_port->depth + 1;
725	port->parent_dport = parent_dport;
726
727	/*
728	* walk to the host bridge, or the first ancestor that knows
729	* the host bridge
730	*/
731	iter = port;
732	while (!iter->host_bridge &&
733	!is_cxl_root(port: to_cxl_port(iter->dev.parent)))
734	iter = to_cxl_port(iter->dev.parent);
735	if (iter->host_bridge)
736	port->host_bridge = iter->host_bridge;
737	else if (parent_dport->rch)
738	port->host_bridge = parent_dport->dport_dev;
739	else
740	port->host_bridge = iter->uport_dev;
741	dev_dbg(uport_dev, "host-bridge: %s\n",
742	dev_name(port->host_bridge));
743	} else
744	dev->parent = uport_dev;
745
746	ida_init(ida: &port->decoder_ida);
747	port->hdm_end = -1;
748	port->commit_end = -1;
749	xa_init(xa: &port->dports);
750	xa_init(xa: &port->endpoints);
751	xa_init(xa: &port->regions);
752	port->component_reg_phys = CXL_RESOURCE_NONE;
753
754	device_initialize(dev);
755	lockdep_set_class_and_subclass(&dev->mutex, &cxl_port_key, port->depth);
756	device_set_pm_not_required(dev);
757	dev->bus = &cxl_bus_type;
758	dev->type = &cxl_port_type;
759
760	return port;
761	}
762
763	static int cxl_setup_comp_regs(struct device *host, struct cxl_register_map *map,
764	resource_size_t component_reg_phys)
765	{
766	*map = (struct cxl_register_map) {
767	.host = host,
768	.reg_type = CXL_REGLOC_RBI_EMPTY,
769	.resource = component_reg_phys,
770	};
771
772	if (component_reg_phys == CXL_RESOURCE_NONE)
773	return 0;
774
775	map->reg_type = CXL_REGLOC_RBI_COMPONENT;
776	map->max_size = CXL_COMPONENT_REG_BLOCK_SIZE;
777
778	return cxl_setup_regs(map);
779	}
780
781	static int cxl_port_setup_regs(struct cxl_port *port,
782	resource_size_t component_reg_phys)
783	{
784	if (dev_is_platform(port->uport_dev))
785	return 0;
786	return cxl_setup_comp_regs(host: &port->dev, map: &port->reg_map,
787	component_reg_phys);
788	}
789
790	static int cxl_dport_setup_regs(struct device *host, struct cxl_dport *dport,
791	resource_size_t component_reg_phys)
792	{
793	int rc;
794
795	if (dev_is_platform(dport->dport_dev))
796	return 0;
797
798	/*
799	* use @dport->dport_dev for the context for error messages during
800	* register probing, and fixup @host after the fact, since @host may be
801	* NULL.
802	*/
803	rc = cxl_setup_comp_regs(host: dport->dport_dev, map: &dport->reg_map,
804	component_reg_phys);
805	dport->reg_map.host = host;
806	return rc;
807	}
808
809	DEFINE_SHOW_ATTRIBUTE(einj_cxl_available_error_type);
810
811	static int cxl_einj_inject(void *data, u64 type)
812	{
813	struct cxl_dport *dport = data;
814
815	if (dport->rch)
816	return einj_cxl_inject_rch_error(rcrb: dport->rcrb.base, type);
817
818	return einj_cxl_inject_error(to_pci_dev(dport->dport_dev), type);
819	}
820	DEFINE_DEBUGFS_ATTRIBUTE(cxl_einj_inject_fops, NULL, cxl_einj_inject,
821	"0x%llx\n");
822
823	static void cxl_debugfs_create_dport_dir(struct cxl_dport *dport)
824	{
825	struct dentry *dir;
826
827	if (!einj_cxl_is_initialized())
828	return;
829
830	/*
831	* dport_dev needs to be a PCIe port for CXL 2.0+ ports because
832	* EINJ expects a dport SBDF to be specified for 2.0 error injection.
833	*/
834	if (!dport->rch && !dev_is_pci(dport->dport_dev))
835	return;
836
837	dir = cxl_debugfs_create_dir(dir: dev_name(dev: dport->dport_dev));
838
839	debugfs_create_file("einj_inject", 0200, dir, dport,
840	&cxl_einj_inject_fops);
841	}
842
843	static int cxl_port_add(struct cxl_port *port,
844	resource_size_t component_reg_phys,
845	struct cxl_dport *parent_dport)
846	{
847	struct device *dev __free(put_device) = &port->dev;
848	int rc;
849
850	if (is_cxl_memdev(dev: port->uport_dev)) {
851	struct cxl_memdev *cxlmd = to_cxl_memdev(dev: port->uport_dev);
852	struct cxl_dev_state *cxlds = cxlmd->cxlds;
853
854	rc = dev_set_name(dev, name: "endpoint%d", port->id);
855	if (rc)
856	return rc;
857
858	/*
859	* The endpoint driver already enumerated the component and RAS
860	* registers. Reuse that enumeration while prepping them to be
861	* mapped by the cxl_port driver.
862	*/
863	port->reg_map = cxlds->reg_map;
864	port->reg_map.host = &port->dev;
865	cxlmd->endpoint = port;
866	} else if (parent_dport) {
867	rc = dev_set_name(dev, name: "port%d", port->id);
868	if (rc)
869	return rc;
870
871	port->component_reg_phys = component_reg_phys;
872	} else {
873	rc = dev_set_name(dev, name: "root%d", port->id);
874	if (rc)
875	return rc;
876	}
877
878	rc = device_add(dev);
879	if (rc)
880	return rc;
881
882	/* Inhibit the cleanup function invoked */
883	dev = NULL;
884	return 0;
885	}
886
887	static struct cxl_port *__devm_cxl_add_port(struct device *host,
888	struct device *uport_dev,
889	resource_size_t component_reg_phys,
890	struct cxl_dport *parent_dport)
891	{
892	struct cxl_port *port;
893	int rc;
894
895	port = cxl_port_alloc(uport_dev, parent_dport);
896	if (IS_ERR(ptr: port))
897	return port;
898
899	rc = cxl_port_add(port, component_reg_phys, parent_dport);
900	if (rc)
901	return ERR_PTR(error: rc);
902
903	rc = devm_add_action_or_reset(host, unregister_port, port);
904	if (rc)
905	return ERR_PTR(error: rc);
906
907	rc = devm_cxl_link_uport(host, port);
908	if (rc)
909	return ERR_PTR(error: rc);
910
911	rc = devm_cxl_link_parent_dport(host, port, parent_dport);
912	if (rc)
913	return ERR_PTR(error: rc);
914
915	if (parent_dport && dev_is_pci(uport_dev))
916	port->pci_latency = cxl_pci_get_latency(to_pci_dev(uport_dev));
917
918	return port;
919	}
920
921	/**
922	* devm_cxl_add_port - register a cxl_port in CXL memory decode hierarchy
923	* @host: host device for devm operations
924	* @uport_dev: "physical" device implementing this upstream port
925	* @component_reg_phys: (optional) for configurable cxl_port instances
926	* @parent_dport: next hop up in the CXL memory decode hierarchy
927	*/
928	struct cxl_port *devm_cxl_add_port(struct device *host,
929	struct device *uport_dev,
930	resource_size_t component_reg_phys,
931	struct cxl_dport *parent_dport)
932	{
933	struct cxl_port *port, *parent_port;
934
935	port = __devm_cxl_add_port(host, uport_dev, component_reg_phys,
936	parent_dport);
937
938	parent_port = parent_dport ? parent_dport->port : NULL;
939	if (IS_ERR(ptr: port)) {
940	dev_dbg(uport_dev, "Failed to add%s%s%s: %ld\n",
941	parent_port ? " port to " : "",
942	parent_port ? dev_name(&parent_port->dev) : "",
943	parent_port ? "" : " root port",
944	PTR_ERR(port));
945	} else {
946	dev_dbg(uport_dev, "%s added%s%s%s\n",
947	dev_name(&port->dev),
948	parent_port ? " to " : "",
949	parent_port ? dev_name(&parent_port->dev) : "",
950	parent_port ? "" : " (root port)");
951	}
952
953	return port;
954	}
955	EXPORT_SYMBOL_NS_GPL(devm_cxl_add_port, "CXL");
956
957	struct cxl_root *devm_cxl_add_root(struct device *host,
958	const struct cxl_root_ops *ops)
959	{
960	struct cxl_root *cxl_root;
961	struct cxl_port *port;
962
963	port = devm_cxl_add_port(host, host, CXL_RESOURCE_NONE, NULL);
964	if (IS_ERR(ptr: port))
965	return ERR_CAST(ptr: port);
966
967	cxl_root = to_cxl_root(port);
968	cxl_root->ops = ops;
969	return cxl_root;
970	}
971	EXPORT_SYMBOL_NS_GPL(devm_cxl_add_root, "CXL");
972
973	struct pci_bus *cxl_port_to_pci_bus(struct cxl_port *port)
974	{
975	/* There is no pci_bus associated with a CXL platform-root port */
976	if (is_cxl_root(port))
977	return NULL;
978
979	if (dev_is_pci(port->uport_dev)) {
980	struct pci_dev *pdev = to_pci_dev(port->uport_dev);
981
982	return pdev->subordinate;
983	}
984
985	return xa_load(&cxl_root_buses, index: (unsigned long)port->uport_dev);
986	}
987	EXPORT_SYMBOL_NS_GPL(cxl_port_to_pci_bus, "CXL");
988
989	static void unregister_pci_bus(void *uport_dev)
990	{
991	xa_erase(&cxl_root_buses, index: (unsigned long)uport_dev);
992	}
993
994	int devm_cxl_register_pci_bus(struct device *host, struct device *uport_dev,
995	struct pci_bus *bus)
996	{
997	int rc;
998
999	if (dev_is_pci(uport_dev))
1000	return -EINVAL;
1001
1002	rc = xa_insert(xa: &cxl_root_buses, index: (unsigned long)uport_dev, entry: bus,
1003	GFP_KERNEL);
1004	if (rc)
1005	return rc;
1006	return devm_add_action_or_reset(host, unregister_pci_bus, uport_dev);
1007	}
1008	EXPORT_SYMBOL_NS_GPL(devm_cxl_register_pci_bus, "CXL");
1009
1010	static bool dev_is_cxl_root_child(struct device *dev)
1011	{
1012	struct cxl_port *port, *parent;
1013
1014	if (!is_cxl_port(dev))
1015	return false;
1016
1017	port = to_cxl_port(dev);
1018	if (is_cxl_root(port))
1019	return false;
1020
1021	parent = to_cxl_port(port->dev.parent);
1022	if (is_cxl_root(port: parent))
1023	return true;
1024
1025	return false;
1026	}
1027
1028	struct cxl_root *find_cxl_root(struct cxl_port *port)
1029	{
1030	struct cxl_port *iter = port;
1031
1032	while (iter && !is_cxl_root(port: iter))
1033	iter = to_cxl_port(iter->dev.parent);
1034
1035	if (!iter)
1036	return NULL;
1037	get_device(dev: &iter->dev);
1038	return to_cxl_root(port: iter);
1039	}
1040	EXPORT_SYMBOL_NS_GPL(find_cxl_root, "CXL");
1041
1042	static struct cxl_dport *find_dport(struct cxl_port *port, int id)
1043	{
1044	struct cxl_dport *dport;
1045	unsigned long index;
1046
1047	device_lock_assert(dev: &port->dev);
1048	xa_for_each(&port->dports, index, dport)
1049	if (dport->port_id == id)
1050	return dport;
1051	return NULL;
1052	}
1053
1054	static int add_dport(struct cxl_port *port, struct cxl_dport *dport)
1055	{
1056	struct cxl_dport *dup;
1057	int rc;
1058
1059	device_lock_assert(dev: &port->dev);
1060	dup = find_dport(port, id: dport->port_id);
1061	if (dup) {
1062	dev_err(&port->dev,
1063	"unable to add dport%d-%s non-unique port id (%s)\n",
1064	dport->port_id, dev_name(dport->dport_dev),
1065	dev_name(dup->dport_dev));
1066	return -EBUSY;
1067	}
1068
1069	rc = xa_insert(xa: &port->dports, index: (unsigned long)dport->dport_dev, entry: dport,
1070	GFP_KERNEL);
1071	if (rc)
1072	return rc;
1073
1074	port->nr_dports++;
1075	return 0;
1076	}
1077
1078	/*
1079	* Since root-level CXL dports cannot be enumerated by PCI they are not
1080	* enumerated by the common port driver that acquires the port lock over
1081	* dport add/remove. Instead, root dports are manually added by a
1082	* platform driver and cond_cxl_root_lock() is used to take the missing
1083	* port lock in that case.
1084	*/
1085	static void cond_cxl_root_lock(struct cxl_port *port)
1086	{
1087	if (is_cxl_root(port))
1088	device_lock(dev: &port->dev);
1089	}
1090
1091	static void cond_cxl_root_unlock(struct cxl_port *port)
1092	{
1093	if (is_cxl_root(port))
1094	device_unlock(dev: &port->dev);
1095	}
1096
1097	static void cxl_dport_remove(void *data)
1098	{
1099	struct cxl_dport *dport = data;
1100	struct cxl_port *port = dport->port;
1101
1102	xa_erase(&port->dports, index: (unsigned long) dport->dport_dev);
1103	put_device(dev: dport->dport_dev);
1104	}
1105
1106	static void cxl_dport_unlink(void *data)
1107	{
1108	struct cxl_dport *dport = data;
1109	struct cxl_port *port = dport->port;
1110	char link_name[CXL_TARGET_STRLEN];
1111
1112	sprintf(buf: link_name, fmt: "dport%d", dport->port_id);
1113	sysfs_remove_link(kobj: &port->dev.kobj, name: link_name);
1114	}
1115
1116	static struct cxl_dport *
1117	__devm_cxl_add_dport(struct cxl_port *port, struct device *dport_dev,
1118	int port_id, resource_size_t component_reg_phys,
1119	resource_size_t rcrb)
1120	{
1121	char link_name[CXL_TARGET_STRLEN];
1122	struct cxl_dport *dport;
1123	struct device *host;
1124	int rc;
1125
1126	if (is_cxl_root(port))
1127	host = port->uport_dev;
1128	else
1129	host = &port->dev;
1130
1131	if (!host->driver) {
1132	dev_WARN_ONCE(&port->dev, 1, "dport:%s bad devm context\n",
1133	dev_name(dport_dev));
1134	return ERR_PTR(error: -ENXIO);
1135	}
1136
1137	if (snprintf(buf: link_name, CXL_TARGET_STRLEN, fmt: "dport%d", port_id) >=
1138	CXL_TARGET_STRLEN)
1139	return ERR_PTR(error: -EINVAL);
1140
1141	dport = devm_kzalloc(dev: host, size: sizeof(*dport), GFP_KERNEL);
1142	if (!dport)
1143	return ERR_PTR(error: -ENOMEM);
1144
1145	dport->dport_dev = dport_dev;
1146	dport->port_id = port_id;
1147	dport->port = port;
1148
1149	if (rcrb == CXL_RESOURCE_NONE) {
1150	rc = cxl_dport_setup_regs(host: &port->dev, dport,
1151	component_reg_phys);
1152	if (rc)
1153	return ERR_PTR(error: rc);
1154	} else {
1155	dport->rcrb.base = rcrb;
1156	component_reg_phys = __rcrb_to_component(dev: dport_dev, ri: &dport->rcrb,
1157	which: CXL_RCRB_DOWNSTREAM);
1158	if (component_reg_phys == CXL_RESOURCE_NONE) {
1159	dev_warn(dport_dev, "Invalid Component Registers in RCRB");
1160	return ERR_PTR(error: -ENXIO);
1161	}
1162
1163	/*
1164	* RCH @dport is not ready to map until associated with its
1165	* memdev
1166	*/
1167	rc = cxl_dport_setup_regs(NULL, dport, component_reg_phys);
1168	if (rc)
1169	return ERR_PTR(error: rc);
1170
1171	dport->rch = true;
1172	}
1173
1174	if (component_reg_phys != CXL_RESOURCE_NONE)
1175	dev_dbg(dport_dev, "Component Registers found for dport: %pa\n",
1176	&component_reg_phys);
1177
1178	cond_cxl_root_lock(port);
1179	rc = add_dport(port, dport);
1180	cond_cxl_root_unlock(port);
1181	if (rc)
1182	return ERR_PTR(error: rc);
1183
1184	/*
1185	* Setup port register if this is the first dport showed up. Having
1186	* a dport also means that there is at least 1 active link.
1187	*/
1188	if (port->nr_dports == 1 &&
1189	port->component_reg_phys != CXL_RESOURCE_NONE) {
1190	rc = cxl_port_setup_regs(port, component_reg_phys: port->component_reg_phys);
1191	if (rc) {
1192	xa_erase(&port->dports, index: (unsigned long)dport->dport_dev);
1193	return ERR_PTR(error: rc);
1194	}
1195	port->component_reg_phys = CXL_RESOURCE_NONE;
1196	}
1197
1198	get_device(dev: dport_dev);
1199	rc = devm_add_action_or_reset(host, cxl_dport_remove, dport);
1200	if (rc)
1201	return ERR_PTR(error: rc);
1202
1203	rc = sysfs_create_link(kobj: &port->dev.kobj, target: &dport_dev->kobj, name: link_name);
1204	if (rc)
1205	return ERR_PTR(error: rc);
1206
1207	rc = devm_add_action_or_reset(host, cxl_dport_unlink, dport);
1208	if (rc)
1209	return ERR_PTR(error: rc);
1210
1211	if (dev_is_pci(dport_dev))
1212	dport->link_latency = cxl_pci_get_latency(to_pci_dev(dport_dev));
1213
1214	cxl_debugfs_create_dport_dir(dport);
1215
1216	return dport;
1217	}
1218
1219	/**
1220	* devm_cxl_add_dport - append VH downstream port data to a cxl_port
1221	* @port: the cxl_port that references this dport
1222	* @dport_dev: firmware or PCI device representing the dport
1223	* @port_id: identifier for this dport in a decoder's target list
1224	* @component_reg_phys: optional location of CXL component registers
1225	*
1226	* Note that dports are appended to the devm release action's of the
1227	* either the port's host (for root ports), or the port itself (for
1228	* switch ports)
1229	*/
1230	struct cxl_dport *devm_cxl_add_dport(struct cxl_port *port,
1231	struct device *dport_dev, int port_id,
1232	resource_size_t component_reg_phys)
1233	{
1234	struct cxl_dport *dport;
1235
1236	dport = __devm_cxl_add_dport(port, dport_dev, port_id,
1237	component_reg_phys, CXL_RESOURCE_NONE);
1238	if (IS_ERR(ptr: dport)) {
1239	dev_dbg(dport_dev, "failed to add dport to %s: %ld\n",
1240	dev_name(&port->dev), PTR_ERR(dport));
1241	} else {
1242	dev_dbg(dport_dev, "dport added to %s\n",
1243	dev_name(&port->dev));
1244	}
1245
1246	return dport;
1247	}
1248	EXPORT_SYMBOL_NS_GPL(devm_cxl_add_dport, "CXL");
1249
1250	/**
1251	* devm_cxl_add_rch_dport - append RCH downstream port data to a cxl_port
1252	* @port: the cxl_port that references this dport
1253	* @dport_dev: firmware or PCI device representing the dport
1254	* @port_id: identifier for this dport in a decoder's target list
1255	* @rcrb: mandatory location of a Root Complex Register Block
1256	*
1257	* See CXL 3.0 9.11.8 CXL Devices Attached to an RCH
1258	*/
1259	struct cxl_dport *devm_cxl_add_rch_dport(struct cxl_port *port,
1260	struct device *dport_dev, int port_id,
1261	resource_size_t rcrb)
1262	{
1263	struct cxl_dport *dport;
1264
1265	if (rcrb == CXL_RESOURCE_NONE) {
1266	dev_dbg(&port->dev, "failed to add RCH dport, missing RCRB\n");
1267	return ERR_PTR(error: -EINVAL);
1268	}
1269
1270	dport = __devm_cxl_add_dport(port, dport_dev, port_id,
1271	CXL_RESOURCE_NONE, rcrb);
1272	if (IS_ERR(ptr: dport)) {
1273	dev_dbg(dport_dev, "failed to add RCH dport to %s: %ld\n",
1274	dev_name(&port->dev), PTR_ERR(dport));
1275	} else {
1276	dev_dbg(dport_dev, "RCH dport added to %s\n",
1277	dev_name(&port->dev));
1278	}
1279
1280	return dport;
1281	}
1282	EXPORT_SYMBOL_NS_GPL(devm_cxl_add_rch_dport, "CXL");
1283
1284	static int add_ep(struct cxl_ep *new)
1285	{
1286	struct cxl_port *port = new->dport->port;
1287
1288	guard(device)(T: &port->dev);
1289	if (port->dead)
1290	return -ENXIO;
1291
1292	return xa_insert(xa: &port->endpoints, index: (unsigned long)new->ep,
1293	entry: new, GFP_KERNEL);
1294	}
1295
1296	/**
1297	* cxl_add_ep - register an endpoint's interest in a port
1298	* @dport: the dport that routes to @ep_dev
1299	* @ep_dev: device representing the endpoint
1300	*
1301	* Intermediate CXL ports are scanned based on the arrival of endpoints.
1302	* When those endpoints depart the port can be destroyed once all
1303	* endpoints that care about that port have been removed.
1304	*/
1305	static int cxl_add_ep(struct cxl_dport *dport, struct device *ep_dev)
1306	{
1307	struct cxl_ep *ep;
1308	int rc;
1309
1310	ep = kzalloc(sizeof(*ep), GFP_KERNEL);
1311	if (!ep)
1312	return -ENOMEM;
1313
1314	ep->ep = get_device(dev: ep_dev);
1315	ep->dport = dport;
1316
1317	rc = add_ep(new: ep);
1318	if (rc)
1319	cxl_ep_release(ep);
1320	return rc;
1321	}
1322
1323	struct cxl_find_port_ctx {
1324	const struct device *dport_dev;
1325	const struct cxl_port *parent_port;
1326	struct cxl_dport **dport;
1327	};
1328
1329	static int match_port_by_dport(struct device *dev, const void *data)
1330	{
1331	const struct cxl_find_port_ctx *ctx = data;
1332	struct cxl_dport *dport;
1333	struct cxl_port *port;
1334
1335	if (!is_cxl_port(dev))
1336	return 0;
1337	if (ctx->parent_port && dev->parent != &ctx->parent_port->dev)
1338	return 0;
1339
1340	port = to_cxl_port(dev);
1341	dport = cxl_find_dport_by_dev(port, dport_dev: ctx->dport_dev);
1342	if (ctx->dport)
1343	*ctx->dport = dport;
1344	return dport != NULL;
1345	}
1346
1347	static struct cxl_port *__find_cxl_port(struct cxl_find_port_ctx *ctx)
1348	{
1349	struct device *dev;
1350
1351	if (!ctx->dport_dev)
1352	return NULL;
1353
1354	dev = bus_find_device(bus: &cxl_bus_type, NULL, data: ctx, match: match_port_by_dport);
1355	if (dev)
1356	return to_cxl_port(dev);
1357	return NULL;
1358	}
1359
1360	static struct cxl_port *find_cxl_port(struct device *dport_dev,
1361	struct cxl_dport **dport)
1362	{
1363	struct cxl_find_port_ctx ctx = {
1364	.dport_dev = dport_dev,
1365	.dport = dport,
1366	};
1367	struct cxl_port *port;
1368
1369	port = __find_cxl_port(ctx: &ctx);
1370	return port;
1371	}
1372
1373	/*
1374	* All users of grandparent() are using it to walk PCIe-like switch port
1375	* hierarchy. A PCIe switch is comprised of a bridge device representing the
1376	* upstream switch port and N bridges representing downstream switch ports. When
1377	* bridges stack the grand-parent of a downstream switch port is another
1378	* downstream switch port in the immediate ancestor switch.
1379	*/
1380	static struct device *grandparent(struct device *dev)
1381	{
1382	if (dev && dev->parent)
1383	return dev->parent->parent;
1384	return NULL;
1385	}
1386
1387	static struct device *endpoint_host(struct cxl_port *endpoint)
1388	{
1389	struct cxl_port *port = to_cxl_port(endpoint->dev.parent);
1390
1391	if (is_cxl_root(port))
1392	return port->uport_dev;
1393	return &port->dev;
1394	}
1395
1396	static void delete_endpoint(void *data)
1397	{
1398	struct cxl_memdev *cxlmd = data;
1399	struct cxl_port *endpoint = cxlmd->endpoint;
1400	struct device *host = endpoint_host(endpoint);
1401
1402	scoped_guard(device, host) {
1403	if (host->driver && !endpoint->dead) {
1404	devm_release_action(dev: host, action: cxl_unlink_parent_dport, data: endpoint);
1405	devm_release_action(dev: host, action: cxl_unlink_uport, data: endpoint);
1406	devm_release_action(dev: host, action: unregister_port, data: endpoint);
1407	}
1408	cxlmd->endpoint = NULL;
1409	}
1410	put_device(dev: &endpoint->dev);
1411	put_device(dev: host);
1412	}
1413
1414	int cxl_endpoint_autoremove(struct cxl_memdev *cxlmd, struct cxl_port *endpoint)
1415	{
1416	struct device *host = endpoint_host(endpoint);
1417	struct device *dev = &cxlmd->dev;
1418
1419	get_device(dev: host);
1420	get_device(dev: &endpoint->dev);
1421	cxlmd->depth = endpoint->depth;
1422	return devm_add_action_or_reset(dev, delete_endpoint, cxlmd);
1423	}
1424	EXPORT_SYMBOL_NS_GPL(cxl_endpoint_autoremove, "CXL");
1425
1426	/*
1427	* The natural end of life of a non-root 'cxl_port' is when its parent port goes
1428	* through a ->remove() event ("top-down" unregistration). The unnatural trigger
1429	* for a port to be unregistered is when all memdevs beneath that port have gone
1430	* through ->remove(). This "bottom-up" removal selectively removes individual
1431	* child ports manually. This depends on devm_cxl_add_port() to not change is
1432	* devm action registration order, and for dports to have already been
1433	* destroyed by del_dports().
1434	*/
1435	static void delete_switch_port(struct cxl_port *port)
1436	{
1437	devm_release_action(dev: port->dev.parent, action: cxl_unlink_parent_dport, data: port);
1438	devm_release_action(dev: port->dev.parent, action: cxl_unlink_uport, data: port);
1439	devm_release_action(dev: port->dev.parent, action: unregister_port, data: port);
1440	}
1441
1442	static void del_dport(struct cxl_dport *dport)
1443	{
1444	struct cxl_port *port = dport->port;
1445
1446	devm_release_action(dev: &port->dev, action: cxl_dport_unlink, data: dport);
1447	devm_release_action(dev: &port->dev, action: cxl_dport_remove, data: dport);
1448	devm_kfree(dev: &port->dev, p: dport);
1449	}
1450
1451	static void del_dports(struct cxl_port *port)
1452	{
1453	struct cxl_dport *dport;
1454	unsigned long index;
1455
1456	device_lock_assert(dev: &port->dev);
1457
1458	xa_for_each(&port->dports, index, dport)
1459	del_dport(dport);
1460	}
1461
1462	struct detach_ctx {
1463	struct cxl_memdev *cxlmd;
1464	int depth;
1465	};
1466
1467	static int port_has_memdev(struct device *dev, const void *data)
1468	{
1469	const struct detach_ctx *ctx = data;
1470	struct cxl_port *port;
1471
1472	if (!is_cxl_port(dev))
1473	return 0;
1474
1475	port = to_cxl_port(dev);
1476	if (port->depth != ctx->depth)
1477	return 0;
1478
1479	return !!cxl_ep_load(port, cxlmd: ctx->cxlmd);
1480	}
1481
1482	static void cxl_detach_ep(void *data)
1483	{
1484	struct cxl_memdev *cxlmd = data;
1485
1486	for (int i = cxlmd->depth - 1; i >= 1; i--) {
1487	struct cxl_port *port, *parent_port;
1488	struct detach_ctx ctx = {
1489	.cxlmd = cxlmd,
1490	.depth = i,
1491	};
1492	struct cxl_ep *ep;
1493	bool died = false;
1494
1495	struct device *dev __free(put_device) =
1496	bus_find_device(bus: &cxl_bus_type, NULL, data: &ctx, match: port_has_memdev);
1497	if (!dev)
1498	continue;
1499	port = to_cxl_port(dev);
1500
1501	parent_port = to_cxl_port(port->dev.parent);
1502	device_lock(dev: &parent_port->dev);
1503	device_lock(dev: &port->dev);
1504	ep = cxl_ep_load(port, cxlmd);
1505	dev_dbg(&cxlmd->dev, "disconnect %s from %s\n",
1506	ep ? dev_name(ep->ep) : "", dev_name(&port->dev));
1507	cxl_ep_remove(port, ep);
1508	if (ep && !port->dead && xa_empty(xa: &port->endpoints) &&
1509	!is_cxl_root(port: parent_port) && parent_port->dev.driver) {
1510	/*
1511	* This was the last ep attached to a dynamically
1512	* enumerated port. Block new cxl_add_ep() and garbage
1513	* collect the port.
1514	*/
1515	died = true;
1516	port->dead = true;
1517	del_dports(port);
1518	}
1519	device_unlock(dev: &port->dev);
1520
1521	if (died) {
1522	dev_dbg(&cxlmd->dev, "delete %s\n",
1523	dev_name(&port->dev));
1524	delete_switch_port(port);
1525	}
1526	device_unlock(dev: &parent_port->dev);
1527	}
1528	}
1529
1530	static resource_size_t find_component_registers(struct device *dev)
1531	{
1532	struct cxl_register_map map;
1533	struct pci_dev *pdev;
1534
1535	/*
1536	* Theoretically, CXL component registers can be hosted on a
1537	* non-PCI device, in practice, only cxl_test hits this case.
1538	*/
1539	if (!dev_is_pci(dev))
1540	return CXL_RESOURCE_NONE;
1541
1542	pdev = to_pci_dev(dev);
1543
1544	cxl_find_regblock(pdev, type: CXL_REGLOC_RBI_COMPONENT, map: &map);
1545	return map.resource;
1546	}
1547
1548	static int match_port_by_uport(struct device *dev, const void *data)
1549	{
1550	const struct device *uport_dev = data;
1551	struct cxl_port *port;
1552
1553	if (!is_cxl_port(dev))
1554	return 0;
1555
1556	port = to_cxl_port(dev);
1557	return uport_dev == port->uport_dev;
1558	}
1559
1560	/*
1561	* Function takes a device reference on the port device. Caller should do a
1562	* put_device() when done.
1563	*/
1564	static struct cxl_port *find_cxl_port_by_uport(struct device *uport_dev)
1565	{
1566	struct device *dev;
1567
1568	dev = bus_find_device(bus: &cxl_bus_type, NULL, data: uport_dev, match: match_port_by_uport);
1569	if (dev)
1570	return to_cxl_port(dev);
1571	return NULL;
1572	}
1573
1574	static int update_decoder_targets(struct device *dev, void *data)
1575	{
1576	struct cxl_dport *dport = data;
1577	struct cxl_switch_decoder *cxlsd;
1578	struct cxl_decoder *cxld;
1579	int i;
1580
1581	if (!is_switch_decoder(dev))
1582	return 0;
1583
1584	cxlsd = to_cxl_switch_decoder(dev);
1585	cxld = &cxlsd->cxld;
1586	guard(rwsem_write)(T: &cxl_rwsem.region);
1587
1588	for (i = 0; i < cxld->interleave_ways; i++) {
1589	if (cxld->target_map[i] == dport->port_id) {
1590	cxlsd->target[i] = dport;
1591	dev_dbg(dev, "dport%d found in target list, index %d\n",
1592	dport->port_id, i);
1593	return 0;
1594	}
1595	}
1596
1597	return 0;
1598	}
1599
1600	DEFINE_FREE(del_cxl_dport, struct cxl_dport *, if (!IS_ERR_OR_NULL(_T)) del_dport(_T))
1601	static struct cxl_dport *cxl_port_add_dport(struct cxl_port *port,
1602	struct device *dport_dev)
1603	{
1604	struct cxl_dport *dport;
1605	int rc;
1606
1607	device_lock_assert(dev: &port->dev);
1608	if (!port->dev.driver)
1609	return ERR_PTR(error: -ENXIO);
1610
1611	dport = cxl_find_dport_by_dev(port, dport_dev);
1612	if (dport) {
1613	dev_dbg(&port->dev, "dport%d:%s already exists\n",
1614	dport->port_id, dev_name(dport_dev));
1615	return ERR_PTR(error: -EBUSY);
1616	}
1617
1618	struct cxl_dport *new_dport __free(del_cxl_dport) =
1619	devm_cxl_add_dport_by_dev(port, dport_dev);
1620	if (IS_ERR(ptr: new_dport))
1621	return new_dport;
1622
1623	cxl_switch_parse_cdat(dport: new_dport);
1624
1625	if (ida_is_empty(ida: &port->decoder_ida)) {
1626	rc = devm_cxl_switch_port_decoders_setup(port);
1627	if (rc)
1628	return ERR_PTR(error: rc);
1629	dev_dbg(&port->dev, "first dport%d:%s added with decoders\n",
1630	new_dport->port_id, dev_name(dport_dev));
1631	return no_free_ptr(new_dport);
1632	}
1633
1634	/* New dport added, update the decoder targets */
1635	device_for_each_child(parent: &port->dev, data: new_dport, fn: update_decoder_targets);
1636
1637	dev_dbg(&port->dev, "dport%d:%s added\n", new_dport->port_id,
1638	dev_name(dport_dev));
1639
1640	return no_free_ptr(new_dport);
1641	}
1642
1643	static struct cxl_dport *devm_cxl_create_port(struct device *ep_dev,
1644	struct cxl_port *parent_port,
1645	struct cxl_dport *parent_dport,
1646	struct device *uport_dev,
1647	struct device *dport_dev)
1648	{
1649	resource_size_t component_reg_phys;
1650
1651	device_lock_assert(dev: &parent_port->dev);
1652	if (!parent_port->dev.driver) {
1653	dev_warn(ep_dev,
1654	"port %s:%s:%s disabled, failed to enumerate CXL.mem\n",
1655	dev_name(&parent_port->dev), dev_name(uport_dev),
1656	dev_name(dport_dev));
1657	}
1658
1659	struct cxl_port *port __free(put_cxl_port) =
1660	find_cxl_port_by_uport(uport_dev);
1661	if (!port) {
1662	component_reg_phys = find_component_registers(dev: uport_dev);
1663	port = devm_cxl_add_port(&parent_port->dev, uport_dev,
1664	component_reg_phys, parent_dport);
1665	if (IS_ERR(ptr: port))
1666	return ERR_CAST(ptr: port);
1667
1668	/*
1669	* retry to make sure a port is found. a port device
1670	* reference is taken.
1671	*/
1672	port = find_cxl_port_by_uport(uport_dev);
1673	if (!port)
1674	return ERR_PTR(error: -ENODEV);
1675
1676	dev_dbg(ep_dev, "created port %s:%s\n",
1677	dev_name(&port->dev), dev_name(port->uport_dev));
1678	} else {
1679	/*
1680	* Port was created before right before this function is
1681	* called. Signal the caller to deal with it.
1682	*/
1683	return ERR_PTR(error: -EAGAIN);
1684	}
1685
1686	guard(device)(T: &port->dev);
1687	return cxl_port_add_dport(port, dport_dev);
1688	}
1689
1690	static int add_port_attach_ep(struct cxl_memdev *cxlmd,
1691	struct device *uport_dev,
1692	struct device *dport_dev)
1693	{
1694	struct device *dparent = grandparent(dev: dport_dev);
1695	struct cxl_dport *dport, *parent_dport;
1696	int rc;
1697
1698	if (is_cxl_host_bridge(dev: dparent)) {
1699	/*
1700	* The iteration reached the topology root without finding the
1701	* CXL-root 'cxl_port' on a previous iteration, fail for now to
1702	* be re-probed after platform driver attaches.
1703	*/
1704	dev_dbg(&cxlmd->dev, "%s is a root dport\n",
1705	dev_name(dport_dev));
1706	return -ENXIO;
1707	}
1708
1709	struct cxl_port *parent_port __free(put_cxl_port) =
1710	find_cxl_port_by_uport(uport_dev: dparent->parent);
1711	if (!parent_port) {
1712	/* iterate to create this parent_port */
1713	return -EAGAIN;
1714	}
1715
1716	scoped_guard(device, &parent_port->dev) {
1717	parent_dport = cxl_find_dport_by_dev(port: parent_port, dport_dev: dparent);
1718	if (!parent_dport) {
1719	parent_dport = cxl_port_add_dport(port: parent_port, dport_dev: dparent);
1720	if (IS_ERR(ptr: parent_dport))
1721	return PTR_ERR(ptr: parent_dport);
1722	}
1723
1724	dport = devm_cxl_create_port(ep_dev: &cxlmd->dev, parent_port,
1725	parent_dport, uport_dev,
1726	dport_dev);
1727	if (IS_ERR(ptr: dport)) {
1728	/* Port already exists, restart iteration */
1729	if (PTR_ERR(ptr: dport) == -EAGAIN)
1730	return 0;
1731	return PTR_ERR(ptr: dport);
1732	}
1733	}
1734
1735	rc = cxl_add_ep(dport, ep_dev: &cxlmd->dev);
1736	if (rc == -EBUSY) {
1737	/*
1738	* "can't" happen, but this error code means
1739	* something to the caller, so translate it.
1740	*/
1741	rc = -ENXIO;
1742	}
1743
1744	return rc;
1745	}
1746
1747	static struct cxl_dport *find_or_add_dport(struct cxl_port *port,
1748	struct device *dport_dev)
1749	{
1750	struct cxl_dport *dport;
1751
1752	device_lock_assert(dev: &port->dev);
1753	dport = cxl_find_dport_by_dev(port, dport_dev);
1754	if (!dport) {
1755	dport = cxl_port_add_dport(port, dport_dev);
1756	if (IS_ERR(ptr: dport))
1757	return dport;
1758
1759	/* New dport added, restart iteration */
1760	return ERR_PTR(error: -EAGAIN);
1761	}
1762
1763	return dport;
1764	}
1765
1766	int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd)
1767	{
1768	struct device *dev = &cxlmd->dev;
1769	struct device *iter;
1770	int rc;
1771
1772	/*
1773	* Skip intermediate port enumeration in the RCH case, there
1774	* are no ports in between a host bridge and an endpoint.
1775	*/
1776	if (cxlmd->cxlds->rcd)
1777	return 0;
1778
1779	rc = devm_add_action_or_reset(&cxlmd->dev, cxl_detach_ep, cxlmd);
1780	if (rc)
1781	return rc;
1782
1783	/*
1784	* Scan for and add all cxl_ports in this device's ancestry.
1785	* Repeat until no more ports are added. Abort if a port add
1786	* attempt fails.
1787	*/
1788	retry:
1789	for (iter = dev; iter; iter = grandparent(dev: iter)) {
1790	struct device *dport_dev = grandparent(dev: iter);
1791	struct device *uport_dev;
1792	struct cxl_dport *dport;
1793
1794	if (is_cxl_host_bridge(dev: dport_dev))
1795	return 0;
1796
1797	uport_dev = dport_dev->parent;
1798	if (!uport_dev) {
1799	dev_warn(dev, "at %s no parent for dport: %s\n",
1800	dev_name(iter), dev_name(dport_dev));
1801	return -ENXIO;
1802	}
1803
1804	dev_dbg(dev, "scan: iter: %s dport_dev: %s parent: %s\n",
1805	dev_name(iter), dev_name(dport_dev),
1806	dev_name(uport_dev));
1807	struct cxl_port *port __free(put_cxl_port) =
1808	find_cxl_port_by_uport(uport_dev);
1809	if (port) {
1810	dev_dbg(&cxlmd->dev,
1811	"found already registered port %s:%s\n",
1812	dev_name(&port->dev),
1813	dev_name(port->uport_dev));
1814
1815	/*
1816	* RP port enumerated by cxl_acpi without dport will
1817	* have the dport added here.
1818	*/
1819	scoped_guard(device, &port->dev) {
1820	dport = find_or_add_dport(port, dport_dev);
1821	if (IS_ERR(ptr: dport)) {
1822	if (PTR_ERR(ptr: dport) == -EAGAIN)
1823	goto retry;
1824	return PTR_ERR(ptr: dport);
1825	}
1826	}
1827
1828	rc = cxl_add_ep(dport, ep_dev: &cxlmd->dev);
1829
1830	/*
1831	* If the endpoint already exists in the port's list,
1832	* that's ok, it was added on a previous pass.
1833	* Otherwise, retry in add_port_attach_ep() after taking
1834	* the parent_port lock as the current port may be being
1835	* reaped.
1836	*/
1837	if (rc && rc != -EBUSY)
1838	return rc;
1839
1840	cxl_gpf_port_setup(dport);
1841
1842	/* Any more ports to add between this one and the root? */
1843	if (!dev_is_cxl_root_child(dev: &port->dev))
1844	continue;
1845
1846	return 0;
1847	}
1848
1849	rc = add_port_attach_ep(cxlmd, uport_dev, dport_dev);
1850	/* port missing, try to add parent */
1851	if (rc == -EAGAIN)
1852	continue;
1853	/* failed to add ep or port */
1854	if (rc)
1855	return rc;
1856	/* port added, new descendants possible, start over */
1857	goto retry;
1858	}
1859
1860	return 0;
1861	}
1862	EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_ports, "CXL");
1863
1864	struct cxl_port *cxl_pci_find_port(struct pci_dev *pdev,
1865	struct cxl_dport **dport)
1866	{
1867	return find_cxl_port(dport_dev: pdev->dev.parent, dport);
1868	}
1869	EXPORT_SYMBOL_NS_GPL(cxl_pci_find_port, "CXL");
1870
1871	struct cxl_port *cxl_mem_find_port(struct cxl_memdev *cxlmd,
1872	struct cxl_dport **dport)
1873	{
1874	return find_cxl_port(dport_dev: grandparent(dev: &cxlmd->dev), dport);
1875	}
1876	EXPORT_SYMBOL_NS_GPL(cxl_mem_find_port, "CXL");
1877
1878	static int decoder_populate_targets(struct cxl_switch_decoder *cxlsd,
1879	struct cxl_port *port)
1880	{
1881	struct cxl_decoder *cxld = &cxlsd->cxld;
1882	int i;
1883
1884	device_lock_assert(dev: &port->dev);
1885
1886	if (xa_empty(xa: &port->dports))
1887	return 0;
1888
1889	guard(rwsem_write)(T: &cxl_rwsem.region);
1890	for (i = 0; i < cxlsd->cxld.interleave_ways; i++) {
1891	struct cxl_dport *dport = find_dport(port, id: cxld->target_map[i]);
1892
1893	if (!dport) {
1894	/* dport may be activated later */
1895	continue;
1896	}
1897	cxlsd->target[i] = dport;
1898	}
1899
1900	return 0;
1901	}
1902
1903	static struct lock_class_key cxl_decoder_key;
1904
1905	/**
1906	* cxl_decoder_init - Common decoder setup / initialization
1907	* @port: owning port of this decoder
1908	* @cxld: common decoder properties to initialize
1909	*
1910	* A port may contain one or more decoders. Each of those decoders
1911	* enable some address space for CXL.mem utilization. A decoder is
1912	* expected to be configured by the caller before registering via
1913	* cxl_decoder_add()
1914	*/
1915	static int cxl_decoder_init(struct cxl_port *port, struct cxl_decoder *cxld)
1916	{
1917	struct device *dev;
1918	int rc;
1919
1920	rc = ida_alloc(ida: &port->decoder_ida, GFP_KERNEL);
1921	if (rc < 0)
1922	return rc;
1923
1924	/* need parent to stick around to release the id */
1925	get_device(dev: &port->dev);
1926	cxld->id = rc;
1927
1928	dev = &cxld->dev;
1929	device_initialize(dev);
1930	lockdep_set_class(&dev->mutex, &cxl_decoder_key);
1931	device_set_pm_not_required(dev);
1932	dev->parent = &port->dev;
1933	dev->bus = &cxl_bus_type;
1934
1935	/* Pre initialize an "empty" decoder */
1936	cxld->interleave_ways = 1;
1937	cxld->interleave_granularity = PAGE_SIZE;
1938	cxld->target_type = CXL_DECODER_HOSTONLYMEM;
1939	cxld->hpa_range = (struct range) {
1940	.start = 0,
1941	.end = -1,
1942	};
1943
1944	return 0;
1945	}
1946
1947	static int cxl_switch_decoder_init(struct cxl_port *port,
1948	struct cxl_switch_decoder *cxlsd,
1949	int nr_targets)
1950	{
1951	if (nr_targets > CXL_DECODER_MAX_INTERLEAVE)
1952	return -EINVAL;
1953
1954	cxlsd->nr_targets = nr_targets;
1955	return cxl_decoder_init(port, cxld: &cxlsd->cxld);
1956	}
1957
1958	/**
1959	* cxl_root_decoder_alloc - Allocate a root level decoder
1960	* @port: owning CXL root of this decoder
1961	* @nr_targets: static number of downstream targets
1962	*
1963	* Return: A new cxl decoder to be registered by cxl_decoder_add(). A
1964	* 'CXL root' decoder is one that decodes from a top-level / static platform
1965	* firmware description of CXL resources into a CXL standard decode
1966	* topology.
1967	*/
1968	struct cxl_root_decoder *cxl_root_decoder_alloc(struct cxl_port *port,
1969	unsigned int nr_targets)
1970	{
1971	struct cxl_root_decoder *cxlrd;
1972	struct cxl_switch_decoder *cxlsd;
1973	struct cxl_decoder *cxld;
1974	int rc;
1975
1976	if (!is_cxl_root(port))
1977	return ERR_PTR(error: -EINVAL);
1978
1979	cxlrd = kzalloc(struct_size(cxlrd, cxlsd.target, nr_targets),
1980	GFP_KERNEL);
1981	if (!cxlrd)
1982	return ERR_PTR(error: -ENOMEM);
1983
1984	cxlsd = &cxlrd->cxlsd;
1985	rc = cxl_switch_decoder_init(port, cxlsd, nr_targets);
1986	if (rc) {
1987	kfree(objp: cxlrd);
1988	return ERR_PTR(error: rc);
1989	}
1990
1991	mutex_init(&cxlrd->range_lock);
1992
1993	cxld = &cxlsd->cxld;
1994	cxld->dev.type = &cxl_decoder_root_type;
1995	/*
1996	* cxl_root_decoder_release() special cases negative ids to
1997	* detect memregion_alloc() failures.
1998	*/
1999	atomic_set(v: &cxlrd->region_id, i: -1);
2000	rc = memregion_alloc(GFP_KERNEL);
2001	if (rc < 0) {
2002	put_device(dev: &cxld->dev);
2003	return ERR_PTR(error: rc);
2004	}
2005
2006	atomic_set(v: &cxlrd->region_id, i: rc);
2007	cxlrd->qos_class = CXL_QOS_CLASS_INVALID;
2008	return cxlrd;
2009	}
2010	EXPORT_SYMBOL_NS_GPL(cxl_root_decoder_alloc, "CXL");
2011
2012	/**
2013	* cxl_switch_decoder_alloc - Allocate a switch level decoder
2014	* @port: owning CXL switch port of this decoder
2015	* @nr_targets: max number of dynamically addressable downstream targets
2016	*
2017	* Return: A new cxl decoder to be registered by cxl_decoder_add(). A
2018	* 'switch' decoder is any decoder that can be enumerated by PCIe
2019	* topology and the HDM Decoder Capability. This includes the decoders
2020	* that sit between Switch Upstream Ports / Switch Downstream Ports and
2021	* Host Bridges / Root Ports.
2022	*/
2023	struct cxl_switch_decoder *cxl_switch_decoder_alloc(struct cxl_port *port,
2024	unsigned int nr_targets)
2025	{
2026	struct cxl_switch_decoder *cxlsd;
2027	struct cxl_decoder *cxld;
2028	int rc;
2029
2030	if (is_cxl_root(port) || is_cxl_endpoint(port))
2031	return ERR_PTR(error: -EINVAL);
2032
2033	cxlsd = kzalloc(struct_size(cxlsd, target, nr_targets), GFP_KERNEL);
2034	if (!cxlsd)
2035	return ERR_PTR(error: -ENOMEM);
2036
2037	rc = cxl_switch_decoder_init(port, cxlsd, nr_targets);
2038	if (rc) {
2039	kfree(objp: cxlsd);
2040	return ERR_PTR(error: rc);
2041	}
2042
2043	cxld = &cxlsd->cxld;
2044	cxld->dev.type = &cxl_decoder_switch_type;
2045	return cxlsd;
2046	}
2047	EXPORT_SYMBOL_NS_GPL(cxl_switch_decoder_alloc, "CXL");
2048
2049	/**
2050	* cxl_endpoint_decoder_alloc - Allocate an endpoint decoder
2051	* @port: owning port of this decoder
2052	*
2053	* Return: A new cxl decoder to be registered by cxl_decoder_add()
2054	*/
2055	struct cxl_endpoint_decoder *cxl_endpoint_decoder_alloc(struct cxl_port *port)
2056	{
2057	struct cxl_endpoint_decoder *cxled;
2058	struct cxl_decoder *cxld;
2059	int rc;
2060
2061	if (!is_cxl_endpoint(port))
2062	return ERR_PTR(error: -EINVAL);
2063
2064	cxled = kzalloc(sizeof(*cxled), GFP_KERNEL);
2065	if (!cxled)
2066	return ERR_PTR(error: -ENOMEM);
2067
2068	cxled->pos = -1;
2069	cxled->part = -1;
2070	cxld = &cxled->cxld;
2071	rc = cxl_decoder_init(port, cxld);
2072	if (rc) {
2073	kfree(objp: cxled);
2074	return ERR_PTR(error: rc);
2075	}
2076
2077	cxld->dev.type = &cxl_decoder_endpoint_type;
2078	return cxled;
2079	}
2080	EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_alloc, "CXL");
2081
2082	/**
2083	* cxl_decoder_add_locked - Add a decoder with targets
2084	* @cxld: The cxl decoder allocated by cxl_<type>_decoder_alloc()
2085	*
2086	* Certain types of decoders may not have any targets. The main example of this
2087	* is an endpoint device. A more awkward example is a hostbridge whose root
2088	* ports get hot added (technically possible, though unlikely).
2089	*
2090	* This is the locked variant of cxl_decoder_add().
2091	*
2092	* Context: Process context. Expects the device lock of the port that owns the
2093	* @cxld to be held.
2094	*
2095	* Return: Negative error code if the decoder wasn't properly configured; else
2096	* returns 0.
2097	*/
2098	int cxl_decoder_add_locked(struct cxl_decoder *cxld)
2099	{
2100	struct cxl_port *port;
2101	struct device *dev;
2102	int rc;
2103
2104	if (WARN_ON_ONCE(!cxld))
2105	return -EINVAL;
2106
2107	if (WARN_ON_ONCE(IS_ERR(cxld)))
2108	return PTR_ERR(ptr: cxld);
2109
2110	if (cxld->interleave_ways < 1)
2111	return -EINVAL;
2112
2113	dev = &cxld->dev;
2114
2115	port = to_cxl_port(cxld->dev.parent);
2116	if (!is_endpoint_decoder(dev)) {
2117	struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
2118
2119	rc = decoder_populate_targets(cxlsd, port);
2120	if (rc && (cxld->flags & CXL_DECODER_F_ENABLE)) {
2121	dev_err(&port->dev,
2122	"Failed to populate active decoder targets\n");
2123	return rc;
2124	}
2125	}
2126
2127	rc = dev_set_name(dev, name: "decoder%d.%d", port->id, cxld->id);
2128	if (rc)
2129	return rc;
2130
2131	return device_add(dev);
2132	}
2133	EXPORT_SYMBOL_NS_GPL(cxl_decoder_add_locked, "CXL");
2134
2135	/**
2136	* cxl_decoder_add - Add a decoder with targets
2137	* @cxld: The cxl decoder allocated by cxl_<type>_decoder_alloc()
2138	*
2139	* This is the unlocked variant of cxl_decoder_add_locked().
2140	* See cxl_decoder_add_locked().
2141	*
2142	* Context: Process context. Takes and releases the device lock of the port that
2143	* owns the @cxld.
2144	*/
2145	int cxl_decoder_add(struct cxl_decoder *cxld)
2146	{
2147	struct cxl_port *port;
2148
2149	if (WARN_ON_ONCE(!cxld))
2150	return -EINVAL;
2151
2152	if (WARN_ON_ONCE(IS_ERR(cxld)))
2153	return PTR_ERR(ptr: cxld);
2154
2155	port = to_cxl_port(cxld->dev.parent);
2156
2157	guard(device)(T: &port->dev);
2158	return cxl_decoder_add_locked(cxld);
2159	}
2160	EXPORT_SYMBOL_NS_GPL(cxl_decoder_add, "CXL");
2161
2162	static void cxld_unregister(void *dev)
2163	{
2164	if (is_endpoint_decoder(dev))
2165	cxl_decoder_detach(NULL, cxled: to_cxl_endpoint_decoder(dev), pos: -1,
2166	mode: DETACH_INVALIDATE);
2167
2168	device_unregister(dev);
2169	}
2170
2171	int cxl_decoder_autoremove(struct device *host, struct cxl_decoder *cxld)
2172	{
2173	return devm_add_action_or_reset(host, cxld_unregister, &cxld->dev);
2174	}
2175	EXPORT_SYMBOL_NS_GPL(cxl_decoder_autoremove, "CXL");
2176
2177	/**
2178	* __cxl_driver_register - register a driver for the cxl bus
2179	* @cxl_drv: cxl driver structure to attach
2180	* @owner: owning module/driver
2181	* @modname: KBUILD_MODNAME for parent driver
2182	*/
2183	int __cxl_driver_register(struct cxl_driver *cxl_drv, struct module *owner,
2184	const char *modname)
2185	{
2186	if (!cxl_drv->probe) {
2187	pr_debug("%s ->probe() must be specified\n", modname);
2188	return -EINVAL;
2189	}
2190
2191	if (!cxl_drv->name) {
2192	pr_debug("%s ->name must be specified\n", modname);
2193	return -EINVAL;
2194	}
2195
2196	if (!cxl_drv->id) {
2197	pr_debug("%s ->id must be specified\n", modname);
2198	return -EINVAL;
2199	}
2200
2201	cxl_drv->drv.bus = &cxl_bus_type;
2202	cxl_drv->drv.owner = owner;
2203	cxl_drv->drv.mod_name = modname;
2204	cxl_drv->drv.name = cxl_drv->name;
2205
2206	return driver_register(drv: &cxl_drv->drv);
2207	}
2208	EXPORT_SYMBOL_NS_GPL(__cxl_driver_register, "CXL");
2209
2210	void cxl_driver_unregister(struct cxl_driver *cxl_drv)
2211	{
2212	driver_unregister(drv: &cxl_drv->drv);
2213	}
2214	EXPORT_SYMBOL_NS_GPL(cxl_driver_unregister, "CXL");
2215
2216	static int cxl_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)
2217	{
2218	return add_uevent_var(env, format: "MODALIAS=" CXL_MODALIAS_FMT,
2219	cxl_device_id(dev));
2220	}
2221
2222	static int cxl_bus_match(struct device *dev, const struct device_driver *drv)
2223	{
2224	return cxl_device_id(dev) == to_cxl_drv(drv)->id;
2225	}
2226
2227	static int cxl_bus_probe(struct device *dev)
2228	{
2229	int rc;
2230
2231	rc = to_cxl_drv(dev->driver)->probe(dev);
2232	dev_dbg(dev, "probe: %d\n", rc);
2233	return rc;
2234	}
2235
2236	static void cxl_bus_remove(struct device *dev)
2237	{
2238	struct cxl_driver *cxl_drv = to_cxl_drv(dev->driver);
2239
2240	if (cxl_drv->remove)
2241	cxl_drv->remove(dev);
2242	}
2243
2244	static struct workqueue_struct *cxl_bus_wq;
2245
2246	static int cxl_rescan_attach(struct device *dev, void *data)
2247	{
2248	int rc = device_attach(dev);
2249
2250	dev_vdbg(dev, "rescan: %s\n", rc ? "attach" : "detached");
2251
2252	return 0;
2253	}
2254
2255	static void cxl_bus_rescan_queue(struct work_struct *w)
2256	{
2257	bus_for_each_dev(bus: &cxl_bus_type, NULL, NULL, fn: cxl_rescan_attach);
2258	}
2259
2260	void cxl_bus_rescan(void)
2261	{
2262	static DECLARE_WORK(rescan_work, cxl_bus_rescan_queue);
2263
2264	queue_work(wq: cxl_bus_wq, work: &rescan_work);
2265	}
2266	EXPORT_SYMBOL_NS_GPL(cxl_bus_rescan, "CXL");
2267
2268	void cxl_bus_drain(void)
2269	{
2270	drain_workqueue(wq: cxl_bus_wq);
2271	}
2272	EXPORT_SYMBOL_NS_GPL(cxl_bus_drain, "CXL");
2273
2274	bool schedule_cxl_memdev_detach(struct cxl_memdev *cxlmd)
2275	{
2276	return queue_work(wq: cxl_bus_wq, work: &cxlmd->detach_work);
2277	}
2278	EXPORT_SYMBOL_NS_GPL(schedule_cxl_memdev_detach, "CXL");
2279
2280	static void add_latency(struct access_coordinate *c, long latency)
2281	{
2282	for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
2283	c[i].write_latency += latency;
2284	c[i].read_latency += latency;
2285	}
2286	}
2287
2288	static bool coordinates_valid(struct access_coordinate *c)
2289	{
2290	for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
2291	if (c[i].read_bandwidth && c[i].write_bandwidth &&
2292	c[i].read_latency && c[i].write_latency)
2293	continue;
2294	return false;
2295	}
2296
2297	return true;
2298	}
2299
2300	static void set_min_bandwidth(struct access_coordinate *c, unsigned int bw)
2301	{
2302	for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
2303	c[i].write_bandwidth = min(c[i].write_bandwidth, bw);
2304	c[i].read_bandwidth = min(c[i].read_bandwidth, bw);
2305	}
2306	}
2307
2308	static void set_access_coordinates(struct access_coordinate *out,
2309	struct access_coordinate *in)
2310	{
2311	for (int i = 0; i < ACCESS_COORDINATE_MAX; i++)
2312	out[i] = in[i];
2313	}
2314
2315	static bool parent_port_is_cxl_root(struct cxl_port *port)
2316	{
2317	return is_cxl_root(port: to_cxl_port(port->dev.parent));
2318	}
2319
2320	/**
2321	* cxl_endpoint_get_perf_coordinates - Retrieve performance numbers stored in dports
2322	* of CXL path
2323	* @port: endpoint cxl_port
2324	* @coord: output performance data
2325	*
2326	* Return: errno on failure, 0 on success.
2327	*/
2328	int cxl_endpoint_get_perf_coordinates(struct cxl_port *port,
2329	struct access_coordinate *coord)
2330	{
2331	struct cxl_memdev *cxlmd = to_cxl_memdev(dev: port->uport_dev);
2332	struct access_coordinate c[] = {
2333	{
2334	.read_bandwidth = UINT_MAX,
2335	.write_bandwidth = UINT_MAX,
2336	},
2337	{
2338	.read_bandwidth = UINT_MAX,
2339	.write_bandwidth = UINT_MAX,
2340	},
2341	};
2342	struct cxl_port *iter = port;
2343	struct cxl_dport *dport;
2344	struct pci_dev *pdev;
2345	struct device *dev;
2346	unsigned int bw;
2347	bool is_cxl_root;
2348
2349	if (!is_cxl_endpoint(port))
2350	return -EINVAL;
2351
2352	/*
2353	* Skip calculation for RCD. Expectation is HMAT already covers RCD case
2354	* since RCH does not support hotplug.
2355	*/
2356	if (cxlmd->cxlds->rcd)
2357	return 0;
2358
2359	/*
2360	* Exit the loop when the parent port of the current iter port is cxl
2361	* root. The iterative loop starts at the endpoint and gathers the
2362	* latency of the CXL link from the current device/port to the connected
2363	* downstream port each iteration.
2364	*/
2365	do {
2366	dport = iter->parent_dport;
2367	iter = to_cxl_port(iter->dev.parent);
2368	is_cxl_root = parent_port_is_cxl_root(port: iter);
2369
2370	/*
2371	* There's no valid access_coordinate for a root port since RPs do not
2372	* have CDAT and therefore needs to be skipped.
2373	*/
2374	if (!is_cxl_root) {
2375	if (!coordinates_valid(c: dport->coord))
2376	return -EINVAL;
2377	cxl_coordinates_combine(out: c, c1: c, c2: dport->coord);
2378	}
2379	add_latency(c, latency: dport->link_latency);
2380	} while (!is_cxl_root);
2381
2382	dport = iter->parent_dport;
2383	/* Retrieve HB coords */
2384	if (!coordinates_valid(c: dport->coord))
2385	return -EINVAL;
2386	cxl_coordinates_combine(out: c, c1: c, c2: dport->coord);
2387
2388	dev = port->uport_dev->parent;
2389	if (!dev_is_pci(dev))
2390	return -ENODEV;
2391
2392	/* Get the calculated PCI paths bandwidth */
2393	pdev = to_pci_dev(dev);
2394	bw = pcie_bandwidth_available(dev: pdev, NULL, NULL, NULL);
2395	if (bw == 0)
2396	return -ENXIO;
2397	bw /= BITS_PER_BYTE;
2398
2399	set_min_bandwidth(c, bw);
2400	set_access_coordinates(out: coord, in: c);
2401
2402	return 0;
2403	}
2404	EXPORT_SYMBOL_NS_GPL(cxl_endpoint_get_perf_coordinates, "CXL");
2405
2406	int cxl_port_get_switch_dport_bandwidth(struct cxl_port *port,
2407	struct access_coordinate *c)
2408	{
2409	struct cxl_dport *dport = port->parent_dport;
2410
2411	/* Check this port is connected to a switch DSP and not an RP */
2412	if (parent_port_is_cxl_root(port: to_cxl_port(port->dev.parent)))
2413	return -ENODEV;
2414
2415	if (!coordinates_valid(c: dport->coord))
2416	return -EINVAL;
2417
2418	for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
2419	c[i].read_bandwidth = dport->coord[i].read_bandwidth;
2420	c[i].write_bandwidth = dport->coord[i].write_bandwidth;
2421	}
2422
2423	return 0;
2424	}
2425
2426	/* for user tooling to ensure port disable work has completed */
2427	static ssize_t flush_store(const struct bus_type *bus, const char *buf, size_t count)
2428	{
2429	if (sysfs_streq(s1: buf, s2: "1")) {
2430	flush_workqueue(cxl_bus_wq);
2431	return count;
2432	}
2433
2434	return -EINVAL;
2435	}
2436
2437	static BUS_ATTR_WO(flush);
2438
2439	static struct attribute *cxl_bus_attributes[] = {
2440	&bus_attr_flush.attr,
2441	NULL,
2442	};
2443
2444	static struct attribute_group cxl_bus_attribute_group = {
2445	.attrs = cxl_bus_attributes,
2446	};
2447
2448	static const struct attribute_group *cxl_bus_attribute_groups[] = {
2449	&cxl_bus_attribute_group,
2450	NULL,
2451	};
2452
2453	const struct bus_type cxl_bus_type = {
2454	.name = "cxl",
2455	.uevent = cxl_bus_uevent,
2456	.match = cxl_bus_match,
2457	.probe = cxl_bus_probe,
2458	.remove = cxl_bus_remove,
2459	.bus_groups = cxl_bus_attribute_groups,
2460	};
2461	EXPORT_SYMBOL_NS_GPL(cxl_bus_type, "CXL");
2462
2463	static struct dentry *cxl_debugfs;
2464
2465	struct dentry *cxl_debugfs_create_dir(const char *dir)
2466	{
2467	return debugfs_create_dir(name: dir, parent: cxl_debugfs);
2468	}
2469	EXPORT_SYMBOL_NS_GPL(cxl_debugfs_create_dir, "CXL");
2470
2471	static __init int cxl_core_init(void)
2472	{
2473	int rc;
2474
2475	cxl_debugfs = debugfs_create_dir(name: "cxl", NULL);
2476
2477	if (einj_cxl_is_initialized())
2478	debugfs_create_file("einj_types", 0400, cxl_debugfs, NULL,
2479	&einj_cxl_available_error_type_fops);
2480
2481	cxl_mbox_init();
2482
2483	rc = cxl_memdev_init();
2484	if (rc)
2485	return rc;
2486
2487	cxl_bus_wq = alloc_ordered_workqueue("cxl_port", 0);
2488	if (!cxl_bus_wq) {
2489	rc = -ENOMEM;
2490	goto err_wq;
2491	}
2492
2493	rc = bus_register(bus: &cxl_bus_type);
2494	if (rc)
2495	goto err_bus;
2496
2497	rc = cxl_region_init();
2498	if (rc)
2499	goto err_region;
2500
2501	rc = cxl_ras_init();
2502	if (rc)
2503	goto err_ras;
2504
2505	return 0;
2506
2507	err_ras:
2508	cxl_region_exit();
2509	err_region:
2510	bus_unregister(bus: &cxl_bus_type);
2511	err_bus:
2512	destroy_workqueue(wq: cxl_bus_wq);
2513	err_wq:
2514	cxl_memdev_exit();
2515	return rc;
2516	}
2517
2518	static void cxl_core_exit(void)
2519	{
2520	cxl_ras_exit();
2521	cxl_region_exit();
2522	bus_unregister(bus: &cxl_bus_type);
2523	destroy_workqueue(wq: cxl_bus_wq);
2524	cxl_memdev_exit();
2525	debugfs_remove_recursive(dentry: cxl_debugfs);
2526	}
2527
2528	subsys_initcall(cxl_core_init);
2529	module_exit(cxl_core_exit);
2530	MODULE_DESCRIPTION("CXL: Core Compute Express Link support");
2531	MODULE_LICENSE("GPL v2");
2532	MODULE_IMPORT_NS("CXL");
2533