1	// SPDX-License-Identifier: GPL-2.0-only
2	/* Copyright(c) 2022 Intel Corporation. All rights reserved. */
3	#include <linux/memregion.h>
4	#include <linux/genalloc.h>
5	#include <linux/debugfs.h>
6	#include <linux/device.h>
7	#include <linux/module.h>
8	#include <linux/memory.h>
9	#include <linux/slab.h>
10	#include <linux/uuid.h>
11	#include <linux/sort.h>
12	#include <linux/idr.h>
13	#include <linux/memory-tiers.h>
14	#include <linux/string_choices.h>
15	#include <cxlmem.h>
16	#include <cxl.h>
17	#include "core.h"
18
19	/**
20	* DOC: cxl core region
21	*
22	* CXL Regions represent mapped memory capacity in system physical address
23	* space. Whereas the CXL Root Decoders identify the bounds of potential CXL
24	* Memory ranges, Regions represent the active mapped capacity by the HDM
25	* Decoder Capability structures throughout the Host Bridges, Switches, and
26	* Endpoints in the topology.
27	*
28	* Region configuration has ordering constraints. UUID may be set at any time
29	* but is only visible for persistent regions.
30	* 1. Interleave granularity
31	* 2. Interleave size
32	* 3. Decoder targets
33	*/
34
35	/*
36	* nodemask that sets per node when the access_coordinates for the node has
37	* been updated by the CXL memory hotplug notifier.
38	*/
39	static nodemask_t nodemask_region_seen = NODE_MASK_NONE;
40
41	static struct cxl_region *to_cxl_region(struct device *dev);
42
43	#define __ACCESS_ATTR_RO(_level, _name) { \
44	.attr = { .name = __stringify(_name), .mode = 0444 }, \
45	.show = _name##_access##_level##_show, \
46	}
47
48	#define ACCESS_DEVICE_ATTR_RO(level, name) \
49	struct device_attribute dev_attr_access##level##_##name = __ACCESS_ATTR_RO(level, name)
50
51	#define ACCESS_ATTR_RO(level, attrib) \
52	static ssize_t attrib##_access##level##_show(struct device *dev, \
53	struct device_attribute *attr, \
54	char *buf) \
55	{ \
56	struct cxl_region *cxlr = to_cxl_region(dev); \
57	\
58	if (cxlr->coord[level].attrib == 0) \
59	return -ENOENT; \
60	\
61	return sysfs_emit(buf, "%u\n", cxlr->coord[level].attrib); \
62	} \
63	static ACCESS_DEVICE_ATTR_RO(level, attrib)
64
65	ACCESS_ATTR_RO(0, read_bandwidth);
66	ACCESS_ATTR_RO(0, read_latency);
67	ACCESS_ATTR_RO(0, write_bandwidth);
68	ACCESS_ATTR_RO(0, write_latency);
69
70	#define ACCESS_ATTR_DECLARE(level, attrib) \
71	(&dev_attr_access##level##_##attrib.attr)
72
73	static struct attribute *access0_coordinate_attrs[] = {
74	ACCESS_ATTR_DECLARE(0, read_bandwidth),
75	ACCESS_ATTR_DECLARE(0, write_bandwidth),
76	ACCESS_ATTR_DECLARE(0, read_latency),
77	ACCESS_ATTR_DECLARE(0, write_latency),
78	NULL
79	};
80
81	ACCESS_ATTR_RO(1, read_bandwidth);
82	ACCESS_ATTR_RO(1, read_latency);
83	ACCESS_ATTR_RO(1, write_bandwidth);
84	ACCESS_ATTR_RO(1, write_latency);
85
86	static struct attribute *access1_coordinate_attrs[] = {
87	ACCESS_ATTR_DECLARE(1, read_bandwidth),
88	ACCESS_ATTR_DECLARE(1, write_bandwidth),
89	ACCESS_ATTR_DECLARE(1, read_latency),
90	ACCESS_ATTR_DECLARE(1, write_latency),
91	NULL
92	};
93
94	#define ACCESS_VISIBLE(level) \
95	static umode_t cxl_region_access##level##_coordinate_visible( \
96	struct kobject *kobj, struct attribute *a, int n) \
97	{ \
98	struct device *dev = kobj_to_dev(kobj); \
99	struct cxl_region *cxlr = to_cxl_region(dev); \
100	\
101	if (a == &dev_attr_access##level##_read_latency.attr && \
102	cxlr->coord[level].read_latency == 0) \
103	return 0; \
104	\
105	if (a == &dev_attr_access##level##_write_latency.attr && \
106	cxlr->coord[level].write_latency == 0) \
107	return 0; \
108	\
109	if (a == &dev_attr_access##level##_read_bandwidth.attr && \
110	cxlr->coord[level].read_bandwidth == 0) \
111	return 0; \
112	\
113	if (a == &dev_attr_access##level##_write_bandwidth.attr && \
114	cxlr->coord[level].write_bandwidth == 0) \
115	return 0; \
116	\
117	return a->mode; \
118	}
119
120	ACCESS_VISIBLE(0);
121	ACCESS_VISIBLE(1);
122
123	static const struct attribute_group cxl_region_access0_coordinate_group = {
124	.name = "access0",
125	.attrs = access0_coordinate_attrs,
126	.is_visible = cxl_region_access0_coordinate_visible,
127	};
128
129	static const struct attribute_group *get_cxl_region_access0_group(void)
130	{
131	return &cxl_region_access0_coordinate_group;
132	}
133
134	static const struct attribute_group cxl_region_access1_coordinate_group = {
135	.name = "access1",
136	.attrs = access1_coordinate_attrs,
137	.is_visible = cxl_region_access1_coordinate_visible,
138	};
139
140	static const struct attribute_group *get_cxl_region_access1_group(void)
141	{
142	return &cxl_region_access1_coordinate_group;
143	}
144
145	static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
146	char *buf)
147	{
148	struct cxl_region *cxlr = to_cxl_region(dev);
149	struct cxl_region_params *p = &cxlr->params;
150	ssize_t rc;
151
152	ACQUIRE(rwsem_read_intr, region_rwsem)(T: &cxl_rwsem.region);
153	if ((rc = ACQUIRE_ERR(rwsem_read_intr, &region_rwsem)))
154	return rc;
155	if (cxlr->mode != CXL_PARTMODE_PMEM)
156	return sysfs_emit(buf, fmt: "\n");
157	return sysfs_emit(buf, fmt: "%pUb\n", &p->uuid);
158	}
159
160	static int is_dup(struct device *match, void *data)
161	{
162	struct cxl_region_params *p;
163	struct cxl_region *cxlr;
164	uuid_t *uuid = data;
165
166	if (!is_cxl_region(dev: match))
167	return 0;
168
169	lockdep_assert_held(&cxl_rwsem.region);
170	cxlr = to_cxl_region(dev: match);
171	p = &cxlr->params;
172
173	if (uuid_equal(u1: &p->uuid, u2: uuid)) {
174	dev_dbg(match, "already has uuid: %pUb\n", uuid);
175	return -EBUSY;
176	}
177
178	return 0;
179	}
180
181	static ssize_t uuid_store(struct device *dev, struct device_attribute *attr,
182	const char *buf, size_t len)
183	{
184	struct cxl_region *cxlr = to_cxl_region(dev);
185	struct cxl_region_params *p = &cxlr->params;
186	uuid_t temp;
187	ssize_t rc;
188
189	if (len != UUID_STRING_LEN + 1)
190	return -EINVAL;
191
192	rc = uuid_parse(uuid: buf, u: &temp);
193	if (rc)
194	return rc;
195
196	if (uuid_is_null(uuid: &temp))
197	return -EINVAL;
198
199	ACQUIRE(rwsem_write_kill, region_rwsem)(T: &cxl_rwsem.region);
200	if ((rc = ACQUIRE_ERR(rwsem_write_kill, &region_rwsem)))
201	return rc;
202
203	if (uuid_equal(u1: &p->uuid, u2: &temp))
204	return len;
205
206	if (p->state >= CXL_CONFIG_ACTIVE)
207	return -EBUSY;
208
209	rc = bus_for_each_dev(bus: &cxl_bus_type, NULL, data: &temp, fn: is_dup);
210	if (rc < 0)
211	return rc;
212
213	uuid_copy(dst: &p->uuid, src: &temp);
214
215	return len;
216	}
217	static DEVICE_ATTR_RW(uuid);
218
219	static struct cxl_region_ref *cxl_rr_load(struct cxl_port *port,
220	struct cxl_region *cxlr)
221	{
222	return xa_load(&port->regions, index: (unsigned long)cxlr);
223	}
224
225	static int cxl_region_invalidate_memregion(struct cxl_region *cxlr)
226	{
227	if (!cpu_cache_has_invalidate_memregion()) {
228	if (IS_ENABLED(CONFIG_CXL_REGION_INVALIDATION_TEST)) {
229	dev_info_once(
230	&cxlr->dev,
231	"Bypassing cpu_cache_invalidate_memregion() for testing!\n");
232	return 0;
233	}
234	dev_WARN(&cxlr->dev,
235	"Failed to synchronize CPU cache state\n");
236	return -ENXIO;
237	}
238
239	if (!cxlr->params.res)
240	return -ENXIO;
241	cpu_cache_invalidate_memregion(start: cxlr->params.res->start,
242	len: resource_size(res: cxlr->params.res));
243	return 0;
244	}
245
246	static void cxl_region_decode_reset(struct cxl_region *cxlr, int count)
247	{
248	struct cxl_region_params *p = &cxlr->params;
249	int i;
250
251	if (test_bit(CXL_REGION_F_LOCK, &cxlr->flags))
252	return;
253
254	/*
255	* Before region teardown attempt to flush, evict any data cached for
256	* this region, or scream loudly about missing arch / platform support
257	* for CXL teardown.
258	*/
259	cxl_region_invalidate_memregion(cxlr);
260
261	for (i = count - 1; i >= 0; i--) {
262	struct cxl_endpoint_decoder *cxled = p->targets[i];
263	struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
264	struct cxl_port *iter = cxled_to_port(cxled);
265	struct cxl_dev_state *cxlds = cxlmd->cxlds;
266	struct cxl_ep *ep;
267
268	if (cxlds->rcd)
269	goto endpoint_reset;
270
271	while (!is_cxl_root(port: to_cxl_port(dev: iter->dev.parent)))
272	iter = to_cxl_port(dev: iter->dev.parent);
273
274	for (ep = cxl_ep_load(port: iter, cxlmd); iter;
275	iter = ep->next, ep = cxl_ep_load(port: iter, cxlmd)) {
276	struct cxl_region_ref *cxl_rr;
277	struct cxl_decoder *cxld;
278
279	cxl_rr = cxl_rr_load(port: iter, cxlr);
280	cxld = cxl_rr->decoder;
281	if (cxld->reset)
282	cxld->reset(cxld);
283	set_bit(CXL_REGION_F_NEEDS_RESET, addr: &cxlr->flags);
284	}
285
286	endpoint_reset:
287	cxled->cxld.reset(&cxled->cxld);
288	set_bit(CXL_REGION_F_NEEDS_RESET, addr: &cxlr->flags);
289	}
290
291	/* all decoders associated with this region have been torn down */
292	clear_bit(CXL_REGION_F_NEEDS_RESET, addr: &cxlr->flags);
293	}
294
295	static int commit_decoder(struct cxl_decoder *cxld)
296	{
297	struct cxl_switch_decoder *cxlsd = NULL;
298
299	if (cxld->commit)
300	return cxld->commit(cxld);
301
302	if (is_switch_decoder(dev: &cxld->dev))
303	cxlsd = to_cxl_switch_decoder(dev: &cxld->dev);
304
305	if (dev_WARN_ONCE(&cxld->dev, !cxlsd || cxlsd->nr_targets > 1,
306	"->commit() is required\n"))
307	return -ENXIO;
308	return 0;
309	}
310
311	static int cxl_region_decode_commit(struct cxl_region *cxlr)
312	{
313	struct cxl_region_params *p = &cxlr->params;
314	int i, rc = 0;
315
316	for (i = 0; i < p->nr_targets; i++) {
317	struct cxl_endpoint_decoder *cxled = p->targets[i];
318	struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
319	struct cxl_region_ref *cxl_rr;
320	struct cxl_decoder *cxld;
321	struct cxl_port *iter;
322	struct cxl_ep *ep;
323
324	/* commit bottom up */
325	for (iter = cxled_to_port(cxled); !is_cxl_root(port: iter);
326	iter = to_cxl_port(dev: iter->dev.parent)) {
327	cxl_rr = cxl_rr_load(port: iter, cxlr);
328	cxld = cxl_rr->decoder;
329	rc = commit_decoder(cxld);
330	if (rc)
331	break;
332	}
333
334	if (rc) {
335	/* programming @iter failed, teardown */
336	for (ep = cxl_ep_load(port: iter, cxlmd); ep && iter;
337	iter = ep->next, ep = cxl_ep_load(port: iter, cxlmd)) {
338	cxl_rr = cxl_rr_load(port: iter, cxlr);
339	cxld = cxl_rr->decoder;
340	if (cxld->reset)
341	cxld->reset(cxld);
342	}
343
344	cxled->cxld.reset(&cxled->cxld);
345	goto err;
346	}
347	}
348
349	return 0;
350
351	err:
352	/* undo the targets that were successfully committed */
353	cxl_region_decode_reset(cxlr, count: i);
354	return rc;
355	}
356
357	static int queue_reset(struct cxl_region *cxlr)
358	{
359	struct cxl_region_params *p = &cxlr->params;
360	int rc;
361
362	ACQUIRE(rwsem_write_kill, rwsem)(T: &cxl_rwsem.region);
363	if ((rc = ACQUIRE_ERR(rwsem_write_kill, &rwsem)))
364	return rc;
365
366	/* Already in the requested state? */
367	if (p->state < CXL_CONFIG_COMMIT)
368	return 0;
369
370	p->state = CXL_CONFIG_RESET_PENDING;
371
372	return 0;
373	}
374
375	static int __commit(struct cxl_region *cxlr)
376	{
377	struct cxl_region_params *p = &cxlr->params;
378	int rc;
379
380	ACQUIRE(rwsem_write_kill, rwsem)(T: &cxl_rwsem.region);
381	if ((rc = ACQUIRE_ERR(rwsem_write_kill, &rwsem)))
382	return rc;
383
384	/* Already in the requested state? */
385	if (p->state >= CXL_CONFIG_COMMIT)
386	return 0;
387
388	/* Not ready to commit? */
389	if (p->state < CXL_CONFIG_ACTIVE)
390	return -ENXIO;
391
392	/*
393	* Invalidate caches before region setup to drop any speculative
394	* consumption of this address space
395	*/
396	rc = cxl_region_invalidate_memregion(cxlr);
397	if (rc)
398	return rc;
399
400	rc = cxl_region_decode_commit(cxlr);
401	if (rc)
402	return rc;
403
404	p->state = CXL_CONFIG_COMMIT;
405
406	return 0;
407	}
408
409	static ssize_t commit_store(struct device *dev, struct device_attribute *attr,
410	const char *buf, size_t len)
411	{
412	struct cxl_region *cxlr = to_cxl_region(dev);
413	struct cxl_region_params *p = &cxlr->params;
414	bool commit;
415	ssize_t rc;
416
417	rc = kstrtobool(s: buf, res: &commit);
418	if (rc)
419	return rc;
420
421	if (commit) {
422	rc = __commit(cxlr);
423	if (rc)
424	return rc;
425	return len;
426	}
427
428	if (test_bit(CXL_REGION_F_LOCK, &cxlr->flags))
429	return -EPERM;
430
431	rc = queue_reset(cxlr);
432	if (rc)
433	return rc;
434
435	/*
436	* Unmap the region and depend the reset-pending state to ensure
437	* it does not go active again until post reset
438	*/
439	device_release_driver(dev: &cxlr->dev);
440
441	/*
442	* With the reset pending take cxl_rwsem.region unconditionally
443	* to ensure the reset gets handled before returning.
444	*/
445	guard(rwsem_write)(T: &cxl_rwsem.region);
446
447	/*
448	* Revalidate that the reset is still pending in case another
449	* thread already handled this reset.
450	*/
451	if (p->state == CXL_CONFIG_RESET_PENDING) {
452	cxl_region_decode_reset(cxlr, count: p->interleave_ways);
453	p->state = CXL_CONFIG_ACTIVE;
454	}
455
456	return len;
457	}
458
459	static ssize_t commit_show(struct device *dev, struct device_attribute *attr,
460	char *buf)
461	{
462	struct cxl_region *cxlr = to_cxl_region(dev);
463	struct cxl_region_params *p = &cxlr->params;
464	ssize_t rc;
465
466	ACQUIRE(rwsem_read_intr, rwsem)(T: &cxl_rwsem.region);
467	if ((rc = ACQUIRE_ERR(rwsem_read_intr, &rwsem)))
468	return rc;
469	return sysfs_emit(buf, fmt: "%d\n", p->state >= CXL_CONFIG_COMMIT);
470	}
471	static DEVICE_ATTR_RW(commit);
472
473	static ssize_t interleave_ways_show(struct device *dev,
474	struct device_attribute *attr, char *buf)
475	{
476	struct cxl_region *cxlr = to_cxl_region(dev);
477	struct cxl_region_params *p = &cxlr->params;
478	int rc;
479
480	ACQUIRE(rwsem_read_intr, rwsem)(T: &cxl_rwsem.region);
481	if ((rc = ACQUIRE_ERR(rwsem_read_intr, &rwsem)))
482	return rc;
483	return sysfs_emit(buf, fmt: "%d\n", p->interleave_ways);
484	}
485
486	static const struct attribute_group *get_cxl_region_target_group(void);
487
488	static ssize_t interleave_ways_store(struct device *dev,
489	struct device_attribute *attr,
490	const char *buf, size_t len)
491	{
492	struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev: dev->parent);
493	struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
494	struct cxl_region *cxlr = to_cxl_region(dev);
495	struct cxl_region_params *p = &cxlr->params;
496	unsigned int val, save;
497	int rc;
498	u8 iw;
499
500	rc = kstrtouint(s: buf, base: 0, res: &val);
501	if (rc)
502	return rc;
503
504	rc = ways_to_eiw(ways: val, eiw: &iw);
505	if (rc)
506	return rc;
507
508	/*
509	* Even for x3, x6, and x12 interleaves the region interleave must be a
510	* power of 2 multiple of the host bridge interleave.
511	*/
512	if (!is_power_of_2(n: val / cxld->interleave_ways) ||
513	(val % cxld->interleave_ways)) {
514	dev_dbg(&cxlr->dev, "invalid interleave: %d\n", val);
515	return -EINVAL;
516	}
517
518	ACQUIRE(rwsem_write_kill, rwsem)(T: &cxl_rwsem.region);
519	if ((rc = ACQUIRE_ERR(rwsem_write_kill, &rwsem)))
520	return rc;
521
522	if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE)
523	return -EBUSY;
524
525	save = p->interleave_ways;
526	p->interleave_ways = val;
527	rc = sysfs_update_group(kobj: &cxlr->dev.kobj, grp: get_cxl_region_target_group());
528	if (rc) {
529	p->interleave_ways = save;
530	return rc;
531	}
532
533	return len;
534	}
535	static DEVICE_ATTR_RW(interleave_ways);
536
537	static ssize_t interleave_granularity_show(struct device *dev,
538	struct device_attribute *attr,
539	char *buf)
540	{
541	struct cxl_region *cxlr = to_cxl_region(dev);
542	struct cxl_region_params *p = &cxlr->params;
543	int rc;
544
545	ACQUIRE(rwsem_read_intr, rwsem)(T: &cxl_rwsem.region);
546	if ((rc = ACQUIRE_ERR(rwsem_read_intr, &rwsem)))
547	return rc;
548	return sysfs_emit(buf, fmt: "%d\n", p->interleave_granularity);
549	}
550
551	static ssize_t interleave_granularity_store(struct device *dev,
552	struct device_attribute *attr,
553	const char *buf, size_t len)
554	{
555	struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev: dev->parent);
556	struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
557	struct cxl_region *cxlr = to_cxl_region(dev);
558	struct cxl_region_params *p = &cxlr->params;
559	int rc, val;
560	u16 ig;
561
562	rc = kstrtoint(s: buf, base: 0, res: &val);
563	if (rc)
564	return rc;
565
566	rc = granularity_to_eig(granularity: val, eig: &ig);
567	if (rc)
568	return rc;
569
570	/*
571	* When the host-bridge is interleaved, disallow region granularity !=
572	* root granularity. Regions with a granularity less than the root
573	* interleave result in needing multiple endpoints to support a single
574	* slot in the interleave (possible to support in the future). Regions
575	* with a granularity greater than the root interleave result in invalid
576	* DPA translations (invalid to support).
577	*/
578	if (cxld->interleave_ways > 1 && val != cxld->interleave_granularity)
579	return -EINVAL;
580
581	ACQUIRE(rwsem_write_kill, rwsem)(T: &cxl_rwsem.region);
582	if ((rc = ACQUIRE_ERR(rwsem_write_kill, &rwsem)))
583	return rc;
584
585	if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE)
586	return -EBUSY;
587
588	p->interleave_granularity = val;
589
590	return len;
591	}
592	static DEVICE_ATTR_RW(interleave_granularity);
593
594	static ssize_t resource_show(struct device *dev, struct device_attribute *attr,
595	char *buf)
596	{
597	struct cxl_region *cxlr = to_cxl_region(dev);
598	struct cxl_region_params *p = &cxlr->params;
599	u64 resource = -1ULL;
600	int rc;
601
602	ACQUIRE(rwsem_read_intr, rwsem)(T: &cxl_rwsem.region);
603	if ((rc = ACQUIRE_ERR(rwsem_read_intr, &rwsem)))
604	return rc;
605
606	if (p->res)
607	resource = p->res->start;
608	return sysfs_emit(buf, fmt: "%#llx\n", resource);
609	}
610	static DEVICE_ATTR_RO(resource);
611
612	static ssize_t mode_show(struct device *dev, struct device_attribute *attr,
613	char *buf)
614	{
615	struct cxl_region *cxlr = to_cxl_region(dev);
616	const char *desc;
617
618	if (cxlr->mode == CXL_PARTMODE_RAM)
619	desc = "ram";
620	else if (cxlr->mode == CXL_PARTMODE_PMEM)
621	desc = "pmem";
622	else
623	desc = "";
624
625	return sysfs_emit(buf, fmt: "%s\n", desc);
626	}
627	static DEVICE_ATTR_RO(mode);
628
629	static int alloc_hpa(struct cxl_region *cxlr, resource_size_t size)
630	{
631	struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev: cxlr->dev.parent);
632	struct cxl_region_params *p = &cxlr->params;
633	struct resource *res;
634	u64 remainder = 0;
635
636	lockdep_assert_held_write(&cxl_rwsem.region);
637
638	/* Nothing to do... */
639	if (p->res && resource_size(res: p->res) == size)
640	return 0;
641
642	/* To change size the old size must be freed first */
643	if (p->res)
644	return -EBUSY;
645
646	if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE)
647	return -EBUSY;
648
649	/* ways, granularity and uuid (if PMEM) need to be set before HPA */
650	if (!p->interleave_ways || !p->interleave_granularity ||
651	(cxlr->mode == CXL_PARTMODE_PMEM && uuid_is_null(uuid: &p->uuid)))
652	return -ENXIO;
653
654	div64_u64_rem(dividend: size, divisor: (u64)SZ_256M * p->interleave_ways, remainder: &remainder);
655	if (remainder)
656	return -EINVAL;
657
658	res = alloc_free_mem_region(base: cxlrd->res, size, SZ_256M,
659	name: dev_name(dev: &cxlr->dev));
660	if (IS_ERR(ptr: res)) {
661	dev_dbg(&cxlr->dev,
662	"HPA allocation error (%ld) for size:%pap in %s %pr\n",
663	PTR_ERR(res), &size, cxlrd->res->name, cxlrd->res);
664	return PTR_ERR(ptr: res);
665	}
666
667	p->res = res;
668	p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
669
670	return 0;
671	}
672
673	static void cxl_region_iomem_release(struct cxl_region *cxlr)
674	{
675	struct cxl_region_params *p = &cxlr->params;
676
677	if (device_is_registered(dev: &cxlr->dev))
678	lockdep_assert_held_write(&cxl_rwsem.region);
679	if (p->res) {
680	/*
681	* Autodiscovered regions may not have been able to insert their
682	* resource.
683	*/
684	if (p->res->parent)
685	remove_resource(old: p->res);
686	kfree(objp: p->res);
687	p->res = NULL;
688	}
689	}
690
691	static int free_hpa(struct cxl_region *cxlr)
692	{
693	struct cxl_region_params *p = &cxlr->params;
694
695	lockdep_assert_held_write(&cxl_rwsem.region);
696
697	if (!p->res)
698	return 0;
699
700	if (p->state >= CXL_CONFIG_ACTIVE)
701	return -EBUSY;
702
703	cxl_region_iomem_release(cxlr);
704	p->state = CXL_CONFIG_IDLE;
705	return 0;
706	}
707
708	static ssize_t size_store(struct device *dev, struct device_attribute *attr,
709	const char *buf, size_t len)
710	{
711	struct cxl_region *cxlr = to_cxl_region(dev);
712	u64 val;
713	int rc;
714
715	rc = kstrtou64(s: buf, base: 0, res: &val);
716	if (rc)
717	return rc;
718
719	ACQUIRE(rwsem_write_kill, rwsem)(T: &cxl_rwsem.region);
720	if ((rc = ACQUIRE_ERR(rwsem_write_kill, &rwsem)))
721	return rc;
722
723	if (val)
724	rc = alloc_hpa(cxlr, size: val);
725	else
726	rc = free_hpa(cxlr);
727
728	if (rc)
729	return rc;
730
731	return len;
732	}
733
734	static ssize_t size_show(struct device *dev, struct device_attribute *attr,
735	char *buf)
736	{
737	struct cxl_region *cxlr = to_cxl_region(dev);
738	struct cxl_region_params *p = &cxlr->params;
739	u64 size = 0;
740	ssize_t rc;
741
742	ACQUIRE(rwsem_read_intr, rwsem)(T: &cxl_rwsem.region);
743	if ((rc = ACQUIRE_ERR(rwsem_read_intr, &rwsem)))
744	return rc;
745	if (p->res)
746	size = resource_size(res: p->res);
747	return sysfs_emit(buf, fmt: "%#llx\n", size);
748	}
749	static DEVICE_ATTR_RW(size);
750
751	static ssize_t extended_linear_cache_size_show(struct device *dev,
752	struct device_attribute *attr,
753	char *buf)
754	{
755	struct cxl_region *cxlr = to_cxl_region(dev);
756	struct cxl_region_params *p = &cxlr->params;
757	ssize_t rc;
758
759	ACQUIRE(rwsem_read_intr, rwsem)(T: &cxl_rwsem.region);
760	if ((rc = ACQUIRE_ERR(rwsem_read_intr, &rwsem)))
761	return rc;
762	return sysfs_emit(buf, fmt: "%pap\n", &p->cache_size);
763	}
764	static DEVICE_ATTR_RO(extended_linear_cache_size);
765
766	static struct attribute *cxl_region_attrs[] = {
767	&dev_attr_uuid.attr,
768	&dev_attr_commit.attr,
769	&dev_attr_interleave_ways.attr,
770	&dev_attr_interleave_granularity.attr,
771	&dev_attr_resource.attr,
772	&dev_attr_size.attr,
773	&dev_attr_mode.attr,
774	&dev_attr_extended_linear_cache_size.attr,
775	NULL,
776	};
777
778	static umode_t cxl_region_visible(struct kobject *kobj, struct attribute *a,
779	int n)
780	{
781	struct device *dev = kobj_to_dev(kobj);
782	struct cxl_region *cxlr = to_cxl_region(dev);
783
784	/*
785	* Support tooling that expects to find a 'uuid' attribute for all
786	* regions regardless of mode.
787	*/
788	if (a == &dev_attr_uuid.attr && cxlr->mode != CXL_PARTMODE_PMEM)
789	return 0444;
790
791	/*
792	* Don't display extended linear cache attribute if there is no
793	* extended linear cache.
794	*/
795	if (a == &dev_attr_extended_linear_cache_size.attr &&
796	cxlr->params.cache_size == 0)
797	return 0;
798
799	return a->mode;
800	}
801
802	static const struct attribute_group cxl_region_group = {
803	.attrs = cxl_region_attrs,
804	.is_visible = cxl_region_visible,
805	};
806
807	static size_t show_targetN(struct cxl_region *cxlr, char *buf, int pos)
808	{
809	struct cxl_region_params *p = &cxlr->params;
810	struct cxl_endpoint_decoder *cxled;
811	int rc;
812
813	ACQUIRE(rwsem_read_intr, rwsem)(T: &cxl_rwsem.region);
814	if ((rc = ACQUIRE_ERR(rwsem_read_intr, &rwsem)))
815	return rc;
816
817	if (pos >= p->interleave_ways) {
818	dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
819	p->interleave_ways);
820	return -ENXIO;
821	}
822
823	cxled = p->targets[pos];
824	if (!cxled)
825	return sysfs_emit(buf, fmt: "\n");
826	return sysfs_emit(buf, fmt: "%s\n", dev_name(dev: &cxled->cxld.dev));
827	}
828
829	static int check_commit_order(struct device *dev, void *data)
830	{
831	struct cxl_decoder *cxld = to_cxl_decoder(dev);
832
833	/*
834	* if port->commit_end is not the only free decoder, then out of
835	* order shutdown has occurred, block further allocations until
836	* that is resolved
837	*/
838	if (((cxld->flags & CXL_DECODER_F_ENABLE) == 0))
839	return -EBUSY;
840	return 0;
841	}
842
843	static int match_free_decoder(struct device *dev, const void *data)
844	{
845	struct cxl_port *port = to_cxl_port(dev: dev->parent);
846	struct cxl_decoder *cxld;
847	int rc;
848
849	if (!is_switch_decoder(dev))
850	return 0;
851
852	cxld = to_cxl_decoder(dev);
853
854	if (cxld->id != port->commit_end + 1)
855	return 0;
856
857	if (cxld->region) {
858	dev_dbg(dev->parent,
859	"next decoder to commit (%s) is already reserved (%s)\n",
860	dev_name(dev), dev_name(&cxld->region->dev));
861	return 0;
862	}
863
864	rc = device_for_each_child_reverse_from(parent: dev->parent, from: dev, NULL,
865	fn: check_commit_order);
866	if (rc) {
867	dev_dbg(dev->parent,
868	"unable to allocate %s due to out of order shutdown\n",
869	dev_name(dev));
870	return 0;
871	}
872	return 1;
873	}
874
875	static bool spa_maps_hpa(const struct cxl_region_params *p,
876	const struct range *range)
877	{
878	if (!p->res)
879	return false;
880
881	/*
882	* The extended linear cache region is constructed by a 1:1 ratio
883	* where the SPA maps equal amounts of DRAM and CXL HPA capacity with
884	* CXL decoders at the high end of the SPA range.
885	*/
886	return p->res->start + p->cache_size == range->start &&
887	p->res->end == range->end;
888	}
889
890	static int match_auto_decoder(struct device *dev, const void *data)
891	{
892	const struct cxl_region_params *p = data;
893	struct cxl_decoder *cxld;
894	struct range *r;
895
896	if (!is_switch_decoder(dev))
897	return 0;
898
899	cxld = to_cxl_decoder(dev);
900	r = &cxld->hpa_range;
901
902	if (spa_maps_hpa(p, range: r))
903	return 1;
904
905	return 0;
906	}
907
908	/**
909	* cxl_port_pick_region_decoder() - assign or lookup a decoder for a region
910	* @port: a port in the ancestry of the endpoint implied by @cxled
911	* @cxled: endpoint decoder to be, or currently, mapped by @port
912	* @cxlr: region to establish, or validate, decode @port
913	*
914	* In the region creation path cxl_port_pick_region_decoder() is an
915	* allocator to find a free port. In the region assembly path, it is
916	* recalling the decoder that platform firmware picked for validation
917	* purposes.
918	*
919	* The result is recorded in a 'struct cxl_region_ref' in @port.
920	*/
921	static struct cxl_decoder *
922	cxl_port_pick_region_decoder(struct cxl_port *port,
923	struct cxl_endpoint_decoder *cxled,
924	struct cxl_region *cxlr)
925	{
926	struct device *dev;
927
928	if (port == cxled_to_port(cxled))
929	return &cxled->cxld;
930
931	if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags))
932	dev = device_find_child(parent: &port->dev, data: &cxlr->params,
933	match: match_auto_decoder);
934	else
935	dev = device_find_child(parent: &port->dev, NULL, match: match_free_decoder);
936	if (!dev)
937	return NULL;
938	/*
939	* This decoder is pinned registered as long as the endpoint decoder is
940	* registered, and endpoint decoder unregistration holds the
941	* cxl_rwsem.region over unregister events, so no need to hold on to
942	* this extra reference.
943	*/
944	put_device(dev);
945	return to_cxl_decoder(dev);
946	}
947
948	static bool auto_order_ok(struct cxl_port *port, struct cxl_region *cxlr_iter,
949	struct cxl_decoder *cxld)
950	{
951	struct cxl_region_ref *rr = cxl_rr_load(port, cxlr: cxlr_iter);
952	struct cxl_decoder *cxld_iter = rr->decoder;
953
954	/*
955	* Allow the out of order assembly of auto-discovered regions.
956	* Per CXL Spec 3.1 8.2.4.20.12 software must commit decoders
957	* in HPA order. Confirm that the decoder with the lesser HPA
958	* starting address has the lesser id.
959	*/
960	dev_dbg(&cxld->dev, "check for HPA violation %s:%d < %s:%d\n",
961	dev_name(&cxld->dev), cxld->id,
962	dev_name(&cxld_iter->dev), cxld_iter->id);
963
964	if (cxld_iter->id > cxld->id)
965	return true;
966
967	return false;
968	}
969
970	static struct cxl_region_ref *
971	alloc_region_ref(struct cxl_port *port, struct cxl_region *cxlr,
972	struct cxl_endpoint_decoder *cxled,
973	struct cxl_decoder *cxld)
974	{
975	struct cxl_region_params *p = &cxlr->params;
976	struct cxl_region_ref *cxl_rr, *iter;
977	unsigned long index;
978	int rc;
979
980	xa_for_each(&port->regions, index, iter) {
981	struct cxl_region_params *ip = &iter->region->params;
982
983	if (!ip->res || ip->res->start < p->res->start)
984	continue;
985
986	if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
987	if (auto_order_ok(port, cxlr_iter: iter->region, cxld))
988	continue;
989	}
990	dev_dbg(&cxlr->dev, "%s: HPA order violation %s:%pr vs %pr\n",
991	dev_name(&port->dev),
992	dev_name(&iter->region->dev), ip->res, p->res);
993
994	return ERR_PTR(error: -EBUSY);
995	}
996
997	cxl_rr = kzalloc(sizeof(*cxl_rr), GFP_KERNEL);
998	if (!cxl_rr)
999	return ERR_PTR(error: -ENOMEM);
1000	cxl_rr->port = port;
1001	cxl_rr->region = cxlr;
1002	cxl_rr->nr_targets = 1;
1003	xa_init(xa: &cxl_rr->endpoints);
1004
1005	rc = xa_insert(xa: &port->regions, index: (unsigned long)cxlr, entry: cxl_rr, GFP_KERNEL);
1006	if (rc) {
1007	dev_dbg(&cxlr->dev,
1008	"%s: failed to track region reference: %d\n",
1009	dev_name(&port->dev), rc);
1010	kfree(objp: cxl_rr);
1011	return ERR_PTR(error: rc);
1012	}
1013
1014	return cxl_rr;
1015	}
1016
1017	static void cxl_rr_free_decoder(struct cxl_region_ref *cxl_rr)
1018	{
1019	struct cxl_region *cxlr = cxl_rr->region;
1020	struct cxl_decoder *cxld = cxl_rr->decoder;
1021
1022	if (!cxld)
1023	return;
1024
1025	dev_WARN_ONCE(&cxlr->dev, cxld->region != cxlr, "region mismatch\n");
1026	if (cxld->region == cxlr) {
1027	cxld->region = NULL;
1028	put_device(dev: &cxlr->dev);
1029	}
1030	}
1031
1032	static void free_region_ref(struct cxl_region_ref *cxl_rr)
1033	{
1034	struct cxl_port *port = cxl_rr->port;
1035	struct cxl_region *cxlr = cxl_rr->region;
1036
1037	cxl_rr_free_decoder(cxl_rr);
1038	xa_erase(&port->regions, index: (unsigned long)cxlr);
1039	xa_destroy(&cxl_rr->endpoints);
1040	kfree(objp: cxl_rr);
1041	}
1042
1043	static int cxl_rr_ep_add(struct cxl_region_ref *cxl_rr,
1044	struct cxl_endpoint_decoder *cxled)
1045	{
1046	int rc;
1047	struct cxl_port *port = cxl_rr->port;
1048	struct cxl_region *cxlr = cxl_rr->region;
1049	struct cxl_decoder *cxld = cxl_rr->decoder;
1050	struct cxl_ep *ep = cxl_ep_load(port, cxlmd: cxled_to_memdev(cxled));
1051
1052	if (ep) {
1053	rc = xa_insert(xa: &cxl_rr->endpoints, index: (unsigned long)cxled, entry: ep,
1054	GFP_KERNEL);
1055	if (rc)
1056	return rc;
1057	}
1058	cxl_rr->nr_eps++;
1059
1060	if (!cxld->region) {
1061	cxld->region = cxlr;
1062	get_device(dev: &cxlr->dev);
1063	}
1064
1065	return 0;
1066	}
1067
1068	static int cxl_rr_assign_decoder(struct cxl_port *port, struct cxl_region *cxlr,
1069	struct cxl_endpoint_decoder *cxled,
1070	struct cxl_region_ref *cxl_rr,
1071	struct cxl_decoder *cxld)
1072	{
1073	if (cxld->region) {
1074	dev_dbg(&cxlr->dev, "%s: %s already attached to %s\n",
1075	dev_name(&port->dev), dev_name(&cxld->dev),
1076	dev_name(&cxld->region->dev));
1077	return -EBUSY;
1078	}
1079
1080	/*
1081	* Endpoints should already match the region type, but backstop that
1082	* assumption with an assertion. Switch-decoders change mapping-type
1083	* based on what is mapped when they are assigned to a region.
1084	*/
1085	dev_WARN_ONCE(&cxlr->dev,
1086	port == cxled_to_port(cxled) &&
1087	cxld->target_type != cxlr->type,
1088	"%s:%s mismatch decoder type %d -> %d\n",
1089	dev_name(&cxled_to_memdev(cxled)->dev),
1090	dev_name(&cxld->dev), cxld->target_type, cxlr->type);
1091	cxld->target_type = cxlr->type;
1092	cxl_rr->decoder = cxld;
1093	return 0;
1094	}
1095
1096	static void cxl_region_set_lock(struct cxl_region *cxlr,
1097	struct cxl_decoder *cxld)
1098	{
1099	if (!test_bit(CXL_DECODER_F_LOCK, &cxld->flags))
1100	return;
1101
1102	set_bit(CXL_REGION_F_LOCK, addr: &cxlr->flags);
1103	clear_bit(CXL_REGION_F_NEEDS_RESET, addr: &cxlr->flags);
1104	}
1105
1106	/**
1107	* cxl_port_attach_region() - track a region's interest in a port by endpoint
1108	* @port: port to add a new region reference 'struct cxl_region_ref'
1109	* @cxlr: region to attach to @port
1110	* @cxled: endpoint decoder used to create or further pin a region reference
1111	* @pos: interleave position of @cxled in @cxlr
1112	*
1113	* The attach event is an opportunity to validate CXL decode setup
1114	* constraints and record metadata needed for programming HDM decoders,
1115	* in particular decoder target lists.
1116	*
1117	* The steps are:
1118	*
1119	* - validate that there are no other regions with a higher HPA already
1120	* associated with @port
1121	* - establish a region reference if one is not already present
1122	*
1123	* - additionally allocate a decoder instance that will host @cxlr on
1124	* @port
1125	*
1126	* - pin the region reference by the endpoint
1127	* - account for how many entries in @port's target list are needed to
1128	* cover all of the added endpoints.
1129	*/
1130	static int cxl_port_attach_region(struct cxl_port *port,
1131	struct cxl_region *cxlr,
1132	struct cxl_endpoint_decoder *cxled, int pos)
1133	{
1134	struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1135	struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
1136	struct cxl_region_ref *cxl_rr;
1137	bool nr_targets_inc = false;
1138	struct cxl_decoder *cxld;
1139	unsigned long index;
1140	int rc = -EBUSY;
1141
1142	lockdep_assert_held_write(&cxl_rwsem.region);
1143
1144	cxl_rr = cxl_rr_load(port, cxlr);
1145	if (cxl_rr) {
1146	struct cxl_ep *ep_iter;
1147	int found = 0;
1148
1149	/*
1150	* Walk the existing endpoints that have been attached to
1151	* @cxlr at @port and see if they share the same 'next' port
1152	* in the downstream direction. I.e. endpoints that share common
1153	* upstream switch.
1154	*/
1155	xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
1156	if (ep_iter == ep)
1157	continue;
1158	if (ep_iter->next == ep->next) {
1159	found++;
1160	break;
1161	}
1162	}
1163
1164	/*
1165	* New target port, or @port is an endpoint port that always
1166	* accounts its own local decode as a target.
1167	*/
1168	if (!found || !ep->next) {
1169	cxl_rr->nr_targets++;
1170	nr_targets_inc = true;
1171	}
1172	} else {
1173	struct cxl_decoder *cxld;
1174
1175	cxld = cxl_port_pick_region_decoder(port, cxled, cxlr);
1176	if (!cxld) {
1177	dev_dbg(&cxlr->dev, "%s: no decoder available\n",
1178	dev_name(&port->dev));
1179	return -EBUSY;
1180	}
1181
1182	cxl_rr = alloc_region_ref(port, cxlr, cxled, cxld);
1183	if (IS_ERR(ptr: cxl_rr)) {
1184	dev_dbg(&cxlr->dev,
1185	"%s: failed to allocate region reference\n",
1186	dev_name(&port->dev));
1187	return PTR_ERR(ptr: cxl_rr);
1188	}
1189	nr_targets_inc = true;
1190
1191	rc = cxl_rr_assign_decoder(port, cxlr, cxled, cxl_rr, cxld);
1192	if (rc)
1193	goto out_erase;
1194	}
1195	cxld = cxl_rr->decoder;
1196
1197	/*
1198	* the number of targets should not exceed the target_count
1199	* of the decoder
1200	*/
1201	if (is_switch_decoder(dev: &cxld->dev)) {
1202	struct cxl_switch_decoder *cxlsd;
1203
1204	cxlsd = to_cxl_switch_decoder(dev: &cxld->dev);
1205	if (cxl_rr->nr_targets > cxlsd->nr_targets) {
1206	dev_dbg(&cxlr->dev,
1207	"%s:%s %s add: %s:%s @ %d overflows targets: %d\n",
1208	dev_name(port->uport_dev), dev_name(&port->dev),
1209	dev_name(&cxld->dev), dev_name(&cxlmd->dev),
1210	dev_name(&cxled->cxld.dev), pos,
1211	cxlsd->nr_targets);
1212	rc = -ENXIO;
1213	goto out_erase;
1214	}
1215	}
1216
1217	cxl_region_set_lock(cxlr, cxld);
1218
1219	rc = cxl_rr_ep_add(cxl_rr, cxled);
1220	if (rc) {
1221	dev_dbg(&cxlr->dev,
1222	"%s: failed to track endpoint %s:%s reference\n",
1223	dev_name(&port->dev), dev_name(&cxlmd->dev),
1224	dev_name(&cxld->dev));
1225	goto out_erase;
1226	}
1227
1228	dev_dbg(&cxlr->dev,
1229	"%s:%s %s add: %s:%s @ %d next: %s nr_eps: %d nr_targets: %d\n",
1230	dev_name(port->uport_dev), dev_name(&port->dev),
1231	dev_name(&cxld->dev), dev_name(&cxlmd->dev),
1232	dev_name(&cxled->cxld.dev), pos,
1233	ep ? ep->next ? dev_name(ep->next->uport_dev) :
1234	dev_name(&cxlmd->dev) :
1235	"none",
1236	cxl_rr->nr_eps, cxl_rr->nr_targets);
1237
1238	return 0;
1239	out_erase:
1240	if (nr_targets_inc)
1241	cxl_rr->nr_targets--;
1242	if (cxl_rr->nr_eps == 0)
1243	free_region_ref(cxl_rr);
1244	return rc;
1245	}
1246
1247	static void cxl_port_detach_region(struct cxl_port *port,
1248	struct cxl_region *cxlr,
1249	struct cxl_endpoint_decoder *cxled)
1250	{
1251	struct cxl_region_ref *cxl_rr;
1252	struct cxl_ep *ep = NULL;
1253
1254	lockdep_assert_held_write(&cxl_rwsem.region);
1255
1256	cxl_rr = cxl_rr_load(port, cxlr);
1257	if (!cxl_rr)
1258	return;
1259
1260	/*
1261	* Endpoint ports do not carry cxl_ep references, and they
1262	* never target more than one endpoint by definition
1263	*/
1264	if (cxl_rr->decoder == &cxled->cxld)
1265	cxl_rr->nr_eps--;
1266	else
1267	ep = xa_erase(&cxl_rr->endpoints, index: (unsigned long)cxled);
1268	if (ep) {
1269	struct cxl_ep *ep_iter;
1270	unsigned long index;
1271	int found = 0;
1272
1273	cxl_rr->nr_eps--;
1274	xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
1275	if (ep_iter->next == ep->next) {
1276	found++;
1277	break;
1278	}
1279	}
1280	if (!found)
1281	cxl_rr->nr_targets--;
1282	}
1283
1284	if (cxl_rr->nr_eps == 0)
1285	free_region_ref(cxl_rr);
1286	}
1287
1288	static int check_last_peer(struct cxl_endpoint_decoder *cxled,
1289	struct cxl_ep *ep, struct cxl_region_ref *cxl_rr,
1290	int distance)
1291	{
1292	struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1293	struct cxl_region *cxlr = cxl_rr->region;
1294	struct cxl_region_params *p = &cxlr->params;
1295	struct cxl_endpoint_decoder *cxled_peer;
1296	struct cxl_port *port = cxl_rr->port;
1297	struct cxl_memdev *cxlmd_peer;
1298	struct cxl_ep *ep_peer;
1299	int pos = cxled->pos;
1300
1301	/*
1302	* If this position wants to share a dport with the last endpoint mapped
1303	* then that endpoint, at index 'position - distance', must also be
1304	* mapped by this dport.
1305	*/
1306	if (pos < distance) {
1307	dev_dbg(&cxlr->dev, "%s:%s: cannot host %s:%s at %d\n",
1308	dev_name(port->uport_dev), dev_name(&port->dev),
1309	dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1310	return -ENXIO;
1311	}
1312	cxled_peer = p->targets[pos - distance];
1313	cxlmd_peer = cxled_to_memdev(cxled: cxled_peer);
1314	ep_peer = cxl_ep_load(port, cxlmd: cxlmd_peer);
1315	if (ep->dport != ep_peer->dport) {
1316	dev_dbg(&cxlr->dev,
1317	"%s:%s: %s:%s pos %d mismatched peer %s:%s\n",
1318	dev_name(port->uport_dev), dev_name(&port->dev),
1319	dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos,
1320	dev_name(&cxlmd_peer->dev),
1321	dev_name(&cxled_peer->cxld.dev));
1322	return -ENXIO;
1323	}
1324
1325	return 0;
1326	}
1327
1328	static int check_interleave_cap(struct cxl_decoder *cxld, int iw, int ig)
1329	{
1330	struct cxl_port *port = to_cxl_port(dev: cxld->dev.parent);
1331	struct cxl_hdm *cxlhdm = dev_get_drvdata(dev: &port->dev);
1332	unsigned int interleave_mask;
1333	u8 eiw;
1334	u16 eig;
1335	int high_pos, low_pos;
1336
1337	if (!test_bit(iw, &cxlhdm->iw_cap_mask))
1338	return -ENXIO;
1339	/*
1340	* Per CXL specification r3.1(8.2.4.20.13 Decoder Protection),
1341	* if eiw < 8:
1342	* DPAOFFSET[51: eig + 8] = HPAOFFSET[51: eig + 8 + eiw]
1343	* DPAOFFSET[eig + 7: 0] = HPAOFFSET[eig + 7: 0]
1344	*
1345	* when the eiw is 0, all the bits of HPAOFFSET[51: 0] are used, the
1346	* interleave bits are none.
1347	*
1348	* if eiw >= 8:
1349	* DPAOFFSET[51: eig + 8] = HPAOFFSET[51: eig + eiw] / 3
1350	* DPAOFFSET[eig + 7: 0] = HPAOFFSET[eig + 7: 0]
1351	*
1352	* when the eiw is 8, all the bits of HPAOFFSET[51: 0] are used, the
1353	* interleave bits are none.
1354	*/
1355	ways_to_eiw(ways: iw, eiw: &eiw);
1356	if (eiw == 0 || eiw == 8)
1357	return 0;
1358
1359	granularity_to_eig(granularity: ig, eig: &eig);
1360	if (eiw > 8)
1361	high_pos = eiw + eig - 1;
1362	else
1363	high_pos = eiw + eig + 7;
1364	low_pos = eig + 8;
1365	interleave_mask = GENMASK(high_pos, low_pos);
1366	if (interleave_mask & ~cxlhdm->interleave_mask)
1367	return -ENXIO;
1368
1369	return 0;
1370	}
1371
1372	static int cxl_port_setup_targets(struct cxl_port *port,
1373	struct cxl_region *cxlr,
1374	struct cxl_endpoint_decoder *cxled)
1375	{
1376	struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev: cxlr->dev.parent);
1377	int parent_iw, parent_ig, ig, iw, rc, pos = cxled->pos;
1378	struct cxl_port *parent_port = to_cxl_port(dev: port->dev.parent);
1379	struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
1380	struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1381	struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
1382	struct cxl_region_params *p = &cxlr->params;
1383	struct cxl_decoder *cxld = cxl_rr->decoder;
1384	struct cxl_switch_decoder *cxlsd;
1385	struct cxl_port *iter = port;
1386	u16 eig, peig;
1387	u8 eiw, peiw;
1388
1389	/*
1390	* While root level decoders support x3, x6, x12, switch level
1391	* decoders only support powers of 2 up to x16.
1392	*/
1393	if (!is_power_of_2(n: cxl_rr->nr_targets)) {
1394	dev_dbg(&cxlr->dev, "%s:%s: invalid target count %d\n",
1395	dev_name(port->uport_dev), dev_name(&port->dev),
1396	cxl_rr->nr_targets);
1397	return -EINVAL;
1398	}
1399
1400	cxlsd = to_cxl_switch_decoder(dev: &cxld->dev);
1401	if (cxl_rr->nr_targets_set) {
1402	int i, distance = 1;
1403	struct cxl_region_ref *cxl_rr_iter;
1404
1405	/*
1406	* The "distance" between peer downstream ports represents which
1407	* endpoint positions in the region interleave a given port can
1408	* host.
1409	*
1410	* For example, at the root of a hierarchy the distance is
1411	* always 1 as every index targets a different host-bridge. At
1412	* each subsequent switch level those ports map every Nth region
1413	* position where N is the width of the switch == distance.
1414	*/
1415	do {
1416	cxl_rr_iter = cxl_rr_load(port: iter, cxlr);
1417	distance *= cxl_rr_iter->nr_targets;
1418	iter = to_cxl_port(dev: iter->dev.parent);
1419	} while (!is_cxl_root(port: iter));
1420	distance *= cxlrd->cxlsd.cxld.interleave_ways;
1421
1422	for (i = 0; i < cxl_rr->nr_targets_set; i++)
1423	if (ep->dport == cxlsd->target[i]) {
1424	rc = check_last_peer(cxled, ep, cxl_rr,
1425	distance);
1426	if (rc)
1427	return rc;
1428	goto out_target_set;
1429	}
1430	goto add_target;
1431	}
1432
1433	if (is_cxl_root(port: parent_port)) {
1434	/*
1435	* Root decoder IG is always set to value in CFMWS which
1436	* may be different than this region's IG. We can use the
1437	* region's IG here since interleave_granularity_store()
1438	* does not allow interleaved host-bridges with
1439	* root IG != region IG.
1440	*/
1441	parent_ig = p->interleave_granularity;
1442	parent_iw = cxlrd->cxlsd.cxld.interleave_ways;
1443	/*
1444	* For purposes of address bit routing, use power-of-2 math for
1445	* switch ports.
1446	*/
1447	if (!is_power_of_2(n: parent_iw))
1448	parent_iw /= 3;
1449	} else {
1450	struct cxl_region_ref *parent_rr;
1451	struct cxl_decoder *parent_cxld;
1452
1453	parent_rr = cxl_rr_load(port: parent_port, cxlr);
1454	parent_cxld = parent_rr->decoder;
1455	parent_ig = parent_cxld->interleave_granularity;
1456	parent_iw = parent_cxld->interleave_ways;
1457	}
1458
1459	rc = granularity_to_eig(granularity: parent_ig, eig: &peig);
1460	if (rc) {
1461	dev_dbg(&cxlr->dev, "%s:%s: invalid parent granularity: %d\n",
1462	dev_name(parent_port->uport_dev),
1463	dev_name(&parent_port->dev), parent_ig);
1464	return rc;
1465	}
1466
1467	rc = ways_to_eiw(ways: parent_iw, eiw: &peiw);
1468	if (rc) {
1469	dev_dbg(&cxlr->dev, "%s:%s: invalid parent interleave: %d\n",
1470	dev_name(parent_port->uport_dev),
1471	dev_name(&parent_port->dev), parent_iw);
1472	return rc;
1473	}
1474
1475	iw = cxl_rr->nr_targets;
1476	rc = ways_to_eiw(ways: iw, eiw: &eiw);
1477	if (rc) {
1478	dev_dbg(&cxlr->dev, "%s:%s: invalid port interleave: %d\n",
1479	dev_name(port->uport_dev), dev_name(&port->dev), iw);
1480	return rc;
1481	}
1482
1483	/*
1484	* Interleave granularity is a multiple of @parent_port granularity.
1485	* Multiplier is the parent port interleave ways.
1486	*/
1487	rc = granularity_to_eig(granularity: parent_ig * parent_iw, eig: &eig);
1488	if (rc) {
1489	dev_dbg(&cxlr->dev,
1490	"%s: invalid granularity calculation (%d * %d)\n",
1491	dev_name(&parent_port->dev), parent_ig, parent_iw);
1492	return rc;
1493	}
1494
1495	rc = eig_to_granularity(eig, granularity: &ig);
1496	if (rc) {
1497	dev_dbg(&cxlr->dev, "%s:%s: invalid interleave: %d\n",
1498	dev_name(port->uport_dev), dev_name(&port->dev),
1499	256 << eig);
1500	return rc;
1501	}
1502
1503	if (iw > 8 || iw > cxlsd->nr_targets) {
1504	dev_dbg(&cxlr->dev,
1505	"%s:%s:%s: ways: %d overflows targets: %d\n",
1506	dev_name(port->uport_dev), dev_name(&port->dev),
1507	dev_name(&cxld->dev), iw, cxlsd->nr_targets);
1508	return -ENXIO;
1509	}
1510
1511	if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1512	if (cxld->interleave_ways != iw ||
1513	(iw > 1 && cxld->interleave_granularity != ig) ||
1514	!spa_maps_hpa(p, range: &cxld->hpa_range) ||
1515	((cxld->flags & CXL_DECODER_F_ENABLE) == 0)) {
1516	dev_err(&cxlr->dev,
1517	"%s:%s %s expected iw: %d ig: %d %pr\n",
1518	dev_name(port->uport_dev), dev_name(&port->dev),
1519	__func__, iw, ig, p->res);
1520	dev_err(&cxlr->dev,
1521	"%s:%s %s got iw: %d ig: %d state: %s %#llx:%#llx\n",
1522	dev_name(port->uport_dev), dev_name(&port->dev),
1523	__func__, cxld->interleave_ways,
1524	cxld->interleave_granularity,
1525	str_enabled_disabled(cxld->flags & CXL_DECODER_F_ENABLE),
1526	cxld->hpa_range.start, cxld->hpa_range.end);
1527	return -ENXIO;
1528	}
1529	} else {
1530	rc = check_interleave_cap(cxld, iw, ig);
1531	if (rc) {
1532	dev_dbg(&cxlr->dev,
1533	"%s:%s iw: %d ig: %d is not supported\n",
1534	dev_name(port->uport_dev),
1535	dev_name(&port->dev), iw, ig);
1536	return rc;
1537	}
1538
1539	cxld->interleave_ways = iw;
1540	cxld->interleave_granularity = ig;
1541	cxld->hpa_range = (struct range) {
1542	.start = p->res->start,
1543	.end = p->res->end,
1544	};
1545	}
1546	dev_dbg(&cxlr->dev, "%s:%s iw: %d ig: %d\n", dev_name(port->uport_dev),
1547	dev_name(&port->dev), iw, ig);
1548	add_target:
1549	if (cxl_rr->nr_targets_set == cxl_rr->nr_targets) {
1550	dev_dbg(&cxlr->dev,
1551	"%s:%s: targets full trying to add %s:%s at %d\n",
1552	dev_name(port->uport_dev), dev_name(&port->dev),
1553	dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1554	return -ENXIO;
1555	}
1556	if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1557	if (cxlsd->target[cxl_rr->nr_targets_set] != ep->dport) {
1558	dev_dbg(&cxlr->dev, "%s:%s: %s expected %s at %d\n",
1559	dev_name(port->uport_dev), dev_name(&port->dev),
1560	dev_name(&cxlsd->cxld.dev),
1561	dev_name(ep->dport->dport_dev),
1562	cxl_rr->nr_targets_set);
1563	return -ENXIO;
1564	}
1565	} else {
1566	cxlsd->target[cxl_rr->nr_targets_set] = ep->dport;
1567	cxlsd->cxld.target_map[cxl_rr->nr_targets_set] = ep->dport->port_id;
1568	}
1569	cxl_rr->nr_targets_set++;
1570	out_target_set:
1571	dev_dbg(&cxlr->dev, "%s:%s target[%d] = %s for %s:%s @ %d\n",
1572	dev_name(port->uport_dev), dev_name(&port->dev),
1573	cxl_rr->nr_targets_set - 1, dev_name(ep->dport->dport_dev),
1574	dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1575
1576	return 0;
1577	}
1578
1579	static void cxl_port_reset_targets(struct cxl_port *port,
1580	struct cxl_region *cxlr)
1581	{
1582	struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
1583	struct cxl_decoder *cxld;
1584
1585	/*
1586	* After the last endpoint has been detached the entire cxl_rr may now
1587	* be gone.
1588	*/
1589	if (!cxl_rr)
1590	return;
1591	cxl_rr->nr_targets_set = 0;
1592
1593	cxld = cxl_rr->decoder;
1594	cxld->hpa_range = (struct range) {
1595	.start = 0,
1596	.end = -1,
1597	};
1598	}
1599
1600	static void cxl_region_teardown_targets(struct cxl_region *cxlr)
1601	{
1602	struct cxl_region_params *p = &cxlr->params;
1603	struct cxl_endpoint_decoder *cxled;
1604	struct cxl_dev_state *cxlds;
1605	struct cxl_memdev *cxlmd;
1606	struct cxl_port *iter;
1607	struct cxl_ep *ep;
1608	int i;
1609
1610	/*
1611	* In the auto-discovery case skip automatic teardown since the
1612	* address space is already active
1613	*/
1614	if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags))
1615	return;
1616
1617	for (i = 0; i < p->nr_targets; i++) {
1618	cxled = p->targets[i];
1619	cxlmd = cxled_to_memdev(cxled);
1620	cxlds = cxlmd->cxlds;
1621
1622	if (cxlds->rcd)
1623	continue;
1624
1625	iter = cxled_to_port(cxled);
1626	while (!is_cxl_root(port: to_cxl_port(dev: iter->dev.parent)))
1627	iter = to_cxl_port(dev: iter->dev.parent);
1628
1629	for (ep = cxl_ep_load(port: iter, cxlmd); iter;
1630	iter = ep->next, ep = cxl_ep_load(port: iter, cxlmd))
1631	cxl_port_reset_targets(port: iter, cxlr);
1632	}
1633	}
1634
1635	static int cxl_region_setup_targets(struct cxl_region *cxlr)
1636	{
1637	struct cxl_region_params *p = &cxlr->params;
1638	struct cxl_endpoint_decoder *cxled;
1639	struct cxl_dev_state *cxlds;
1640	int i, rc, rch = 0, vh = 0;
1641	struct cxl_memdev *cxlmd;
1642	struct cxl_port *iter;
1643	struct cxl_ep *ep;
1644
1645	for (i = 0; i < p->nr_targets; i++) {
1646	cxled = p->targets[i];
1647	cxlmd = cxled_to_memdev(cxled);
1648	cxlds = cxlmd->cxlds;
1649
1650	/* validate that all targets agree on topology */
1651	if (!cxlds->rcd) {
1652	vh++;
1653	} else {
1654	rch++;
1655	continue;
1656	}
1657
1658	iter = cxled_to_port(cxled);
1659	while (!is_cxl_root(port: to_cxl_port(dev: iter->dev.parent)))
1660	iter = to_cxl_port(dev: iter->dev.parent);
1661
1662	/*
1663	* Descend the topology tree programming / validating
1664	* targets while looking for conflicts.
1665	*/
1666	for (ep = cxl_ep_load(port: iter, cxlmd); iter;
1667	iter = ep->next, ep = cxl_ep_load(port: iter, cxlmd)) {
1668	rc = cxl_port_setup_targets(port: iter, cxlr, cxled);
1669	if (rc) {
1670	cxl_region_teardown_targets(cxlr);
1671	return rc;
1672	}
1673	}
1674	}
1675
1676	if (rch && vh) {
1677	dev_err(&cxlr->dev, "mismatched CXL topologies detected\n");
1678	cxl_region_teardown_targets(cxlr);
1679	return -ENXIO;
1680	}
1681
1682	return 0;
1683	}
1684
1685	static int cxl_region_validate_position(struct cxl_region *cxlr,
1686	struct cxl_endpoint_decoder *cxled,
1687	int pos)
1688	{
1689	struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1690	struct cxl_region_params *p = &cxlr->params;
1691	int i;
1692
1693	if (pos < 0 || pos >= p->interleave_ways) {
1694	dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
1695	p->interleave_ways);
1696	return -ENXIO;
1697	}
1698
1699	if (p->targets[pos] == cxled)
1700	return 0;
1701
1702	if (p->targets[pos]) {
1703	struct cxl_endpoint_decoder *cxled_target = p->targets[pos];
1704	struct cxl_memdev *cxlmd_target = cxled_to_memdev(cxled: cxled_target);
1705
1706	dev_dbg(&cxlr->dev, "position %d already assigned to %s:%s\n",
1707	pos, dev_name(&cxlmd_target->dev),
1708	dev_name(&cxled_target->cxld.dev));
1709	return -EBUSY;
1710	}
1711
1712	for (i = 0; i < p->interleave_ways; i++) {
1713	struct cxl_endpoint_decoder *cxled_target;
1714	struct cxl_memdev *cxlmd_target;
1715
1716	cxled_target = p->targets[i];
1717	if (!cxled_target)
1718	continue;
1719
1720	cxlmd_target = cxled_to_memdev(cxled: cxled_target);
1721	if (cxlmd_target == cxlmd) {
1722	dev_dbg(&cxlr->dev,
1723	"%s already specified at position %d via: %s\n",
1724	dev_name(&cxlmd->dev), pos,
1725	dev_name(&cxled_target->cxld.dev));
1726	return -EBUSY;
1727	}
1728	}
1729
1730	return 0;
1731	}
1732
1733	static int cxl_region_attach_position(struct cxl_region *cxlr,
1734	struct cxl_root_decoder *cxlrd,
1735	struct cxl_endpoint_decoder *cxled,
1736	const struct cxl_dport *dport, int pos)
1737	{
1738	struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1739	struct cxl_switch_decoder *cxlsd = &cxlrd->cxlsd;
1740	struct cxl_decoder *cxld = &cxlsd->cxld;
1741	int iw = cxld->interleave_ways;
1742	struct cxl_port *iter;
1743	int rc;
1744
1745	if (dport != cxlrd->cxlsd.target[pos % iw]) {
1746	dev_dbg(&cxlr->dev, "%s:%s invalid target position for %s\n",
1747	dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1748	dev_name(&cxlrd->cxlsd.cxld.dev));
1749	return -ENXIO;
1750	}
1751
1752	for (iter = cxled_to_port(cxled); !is_cxl_root(port: iter);
1753	iter = to_cxl_port(dev: iter->dev.parent)) {
1754	rc = cxl_port_attach_region(port: iter, cxlr, cxled, pos);
1755	if (rc)
1756	goto err;
1757	}
1758
1759	return 0;
1760
1761	err:
1762	for (iter = cxled_to_port(cxled); !is_cxl_root(port: iter);
1763	iter = to_cxl_port(dev: iter->dev.parent))
1764	cxl_port_detach_region(port: iter, cxlr, cxled);
1765	return rc;
1766	}
1767
1768	static int cxl_region_attach_auto(struct cxl_region *cxlr,
1769	struct cxl_endpoint_decoder *cxled, int pos)
1770	{
1771	struct cxl_region_params *p = &cxlr->params;
1772
1773	if (cxled->state != CXL_DECODER_STATE_AUTO) {
1774	dev_err(&cxlr->dev,
1775	"%s: unable to add decoder to autodetected region\n",
1776	dev_name(&cxled->cxld.dev));
1777	return -EINVAL;
1778	}
1779
1780	if (pos >= 0) {
1781	dev_dbg(&cxlr->dev, "%s: expected auto position, not %d\n",
1782	dev_name(&cxled->cxld.dev), pos);
1783	return -EINVAL;
1784	}
1785
1786	if (p->nr_targets >= p->interleave_ways) {
1787	dev_err(&cxlr->dev, "%s: no more target slots available\n",
1788	dev_name(&cxled->cxld.dev));
1789	return -ENXIO;
1790	}
1791
1792	/*
1793	* Temporarily record the endpoint decoder into the target array. Yes,
1794	* this means that userspace can view devices in the wrong position
1795	* before the region activates, and must be careful to understand when
1796	* it might be racing region autodiscovery.
1797	*/
1798	pos = p->nr_targets;
1799	p->targets[pos] = cxled;
1800	cxled->pos = pos;
1801	p->nr_targets++;
1802
1803	return 0;
1804	}
1805
1806	static int cmp_interleave_pos(const void *a, const void *b)
1807	{
1808	struct cxl_endpoint_decoder *cxled_a = *(typeof(cxled_a) *)a;
1809	struct cxl_endpoint_decoder *cxled_b = *(typeof(cxled_b) *)b;
1810
1811	return cxled_a->pos - cxled_b->pos;
1812	}
1813
1814	static int match_switch_decoder_by_range(struct device *dev,
1815	const void *data)
1816	{
1817	struct cxl_switch_decoder *cxlsd;
1818	const struct range *r1, *r2 = data;
1819
1820
1821	if (!is_switch_decoder(dev))
1822	return 0;
1823
1824	cxlsd = to_cxl_switch_decoder(dev);
1825	r1 = &cxlsd->cxld.hpa_range;
1826
1827	if (is_root_decoder(dev))
1828	return range_contains(r1, r2);
1829	return (r1->start == r2->start && r1->end == r2->end);
1830	}
1831
1832	static int find_pos_and_ways(struct cxl_port *port, struct range *range,
1833	int *pos, int *ways)
1834	{
1835	struct cxl_switch_decoder *cxlsd;
1836	struct cxl_port *parent;
1837	struct device *dev;
1838	int rc = -ENXIO;
1839
1840	parent = parent_port_of(port);
1841	if (!parent)
1842	return rc;
1843
1844	dev = device_find_child(parent: &parent->dev, data: range,
1845	match: match_switch_decoder_by_range);
1846	if (!dev) {
1847	dev_err(port->uport_dev,
1848	"failed to find decoder mapping %#llx-%#llx\n",
1849	range->start, range->end);
1850	return rc;
1851	}
1852	cxlsd = to_cxl_switch_decoder(dev);
1853	*ways = cxlsd->cxld.interleave_ways;
1854
1855	for (int i = 0; i < *ways; i++) {
1856	if (cxlsd->target[i] == port->parent_dport) {
1857	*pos = i;
1858	rc = 0;
1859	break;
1860	}
1861	}
1862	put_device(dev);
1863
1864	if (rc)
1865	dev_err(port->uport_dev,
1866	"failed to find %s:%s in target list of %s\n",
1867	dev_name(&port->dev),
1868	dev_name(port->parent_dport->dport_dev),
1869	dev_name(&cxlsd->cxld.dev));
1870
1871	return rc;
1872	}
1873
1874	/**
1875	* cxl_calc_interleave_pos() - calculate an endpoint position in a region
1876	* @cxled: endpoint decoder member of given region
1877	*
1878	* The endpoint position is calculated by traversing the topology from
1879	* the endpoint to the root decoder and iteratively applying this
1880	* calculation:
1881	*
1882	* position = position * parent_ways + parent_pos;
1883	*
1884	* ...where @position is inferred from switch and root decoder target lists.
1885	*
1886	* Return: position >= 0 on success
1887	* -ENXIO on failure
1888	*/
1889	static int cxl_calc_interleave_pos(struct cxl_endpoint_decoder *cxled)
1890	{
1891	struct cxl_port *iter, *port = cxled_to_port(cxled);
1892	struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1893	struct range *range = &cxled->cxld.hpa_range;
1894	int parent_ways = 0, parent_pos = 0, pos = 0;
1895	int rc;
1896
1897	/*
1898	* Example: the expected interleave order of the 4-way region shown
1899	* below is: mem0, mem2, mem1, mem3
1900	*
1901	* root_port
1902	* / \
1903	* host_bridge_0 host_bridge_1
1904	* | | | |
1905	* mem0 mem1 mem2 mem3
1906	*
1907	* In the example the calculator will iterate twice. The first iteration
1908	* uses the mem position in the host-bridge and the ways of the host-
1909	* bridge to generate the first, or local, position. The second
1910	* iteration uses the host-bridge position in the root_port and the ways
1911	* of the root_port to refine the position.
1912	*
1913	* A trace of the calculation per endpoint looks like this:
1914	* mem0: pos = 0 * 2 + 0 mem2: pos = 0 * 2 + 0
1915	* pos = 0 * 2 + 0 pos = 0 * 2 + 1
1916	* pos: 0 pos: 1
1917	*
1918	* mem1: pos = 0 * 2 + 1 mem3: pos = 0 * 2 + 1
1919	* pos = 1 * 2 + 0 pos = 1 * 2 + 1
1920	* pos: 2 pos = 3
1921	*
1922	* Note that while this example is simple, the method applies to more
1923	* complex topologies, including those with switches.
1924	*/
1925
1926	/* Iterate from endpoint to root_port refining the position */
1927	for (iter = port; iter; iter = parent_port_of(port: iter)) {
1928	if (is_cxl_root(port: iter))
1929	break;
1930
1931	rc = find_pos_and_ways(port: iter, range, pos: &parent_pos, ways: &parent_ways);
1932	if (rc)
1933	return rc;
1934
1935	pos = pos * parent_ways + parent_pos;
1936	}
1937
1938	dev_dbg(&cxlmd->dev,
1939	"decoder:%s parent:%s port:%s range:%#llx-%#llx pos:%d\n",
1940	dev_name(&cxled->cxld.dev), dev_name(cxlmd->dev.parent),
1941	dev_name(&port->dev), range->start, range->end, pos);
1942
1943	return pos;
1944	}
1945
1946	static int cxl_region_sort_targets(struct cxl_region *cxlr)
1947	{
1948	struct cxl_region_params *p = &cxlr->params;
1949	int i, rc = 0;
1950
1951	for (i = 0; i < p->nr_targets; i++) {
1952	struct cxl_endpoint_decoder *cxled = p->targets[i];
1953
1954	cxled->pos = cxl_calc_interleave_pos(cxled);
1955	/*
1956	* Record that sorting failed, but still continue to calc
1957	* cxled->pos so that follow-on code paths can reliably
1958	* do p->targets[cxled->pos] to self-reference their entry.
1959	*/
1960	if (cxled->pos < 0)
1961	rc = -ENXIO;
1962	}
1963	/* Keep the cxlr target list in interleave position order */
1964	sort(base: p->targets, num: p->nr_targets, size: sizeof(p->targets[0]),
1965	cmp_func: cmp_interleave_pos, NULL);
1966
1967	dev_dbg(&cxlr->dev, "region sort %s\n", rc ? "failed" : "successful");
1968	return rc;
1969	}
1970
1971	static int cxl_region_attach(struct cxl_region *cxlr,
1972	struct cxl_endpoint_decoder *cxled, int pos)
1973	{
1974	struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev: cxlr->dev.parent);
1975	struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1976	struct cxl_dev_state *cxlds = cxlmd->cxlds;
1977	struct cxl_region_params *p = &cxlr->params;
1978	struct cxl_port *ep_port, *root_port;
1979	struct cxl_dport *dport;
1980	int rc = -ENXIO;
1981
1982	rc = check_interleave_cap(cxld: &cxled->cxld, iw: p->interleave_ways,
1983	ig: p->interleave_granularity);
1984	if (rc) {
1985	dev_dbg(&cxlr->dev, "%s iw: %d ig: %d is not supported\n",
1986	dev_name(&cxled->cxld.dev), p->interleave_ways,
1987	p->interleave_granularity);
1988	return rc;
1989	}
1990
1991	if (cxled->part < 0) {
1992	dev_dbg(&cxlr->dev, "%s dead\n", dev_name(&cxled->cxld.dev));
1993	return -ENODEV;
1994	}
1995
1996	if (cxlds->part[cxled->part].mode != cxlr->mode) {
1997	dev_dbg(&cxlr->dev, "%s region mode: %d mismatch\n",
1998	dev_name(&cxled->cxld.dev), cxlr->mode);
1999	return -EINVAL;
2000	}
2001
2002	/* all full of members, or interleave config not established? */
2003	if (p->state > CXL_CONFIG_INTERLEAVE_ACTIVE) {
2004	dev_dbg(&cxlr->dev, "region already active\n");
2005	return -EBUSY;
2006	}
2007
2008	if (p->state < CXL_CONFIG_INTERLEAVE_ACTIVE) {
2009	dev_dbg(&cxlr->dev, "interleave config missing\n");
2010	return -ENXIO;
2011	}
2012
2013	if (p->nr_targets >= p->interleave_ways) {
2014	dev_dbg(&cxlr->dev, "region already has %d endpoints\n",
2015	p->nr_targets);
2016	return -EINVAL;
2017	}
2018
2019	ep_port = cxled_to_port(cxled);
2020	root_port = cxlrd_to_port(cxlrd);
2021	dport = cxl_find_dport_by_dev(port: root_port, dport_dev: ep_port->host_bridge);
2022	if (!dport) {
2023	dev_dbg(&cxlr->dev, "%s:%s invalid target for %s\n",
2024	dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2025	dev_name(cxlr->dev.parent));
2026	return -ENXIO;
2027	}
2028
2029	if (cxled->cxld.target_type != cxlr->type) {
2030	dev_dbg(&cxlr->dev, "%s:%s type mismatch: %d vs %d\n",
2031	dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2032	cxled->cxld.target_type, cxlr->type);
2033	return -ENXIO;
2034	}
2035
2036	if (!cxled->dpa_res) {
2037	dev_dbg(&cxlr->dev, "%s:%s: missing DPA allocation.\n",
2038	dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev));
2039	return -ENXIO;
2040	}
2041
2042	if (resource_size(res: cxled->dpa_res) * p->interleave_ways + p->cache_size !=
2043	resource_size(res: p->res)) {
2044	dev_dbg(&cxlr->dev,
2045	"%s:%s-size-%#llx * ways-%d + cache-%#llx != region-size-%#llx\n",
2046	dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2047	(u64)resource_size(cxled->dpa_res), p->interleave_ways,
2048	(u64)p->cache_size, (u64)resource_size(p->res));
2049	return -EINVAL;
2050	}
2051
2052	cxl_region_perf_data_calculate(cxlr, cxled);
2053
2054	if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
2055	int i;
2056
2057	rc = cxl_region_attach_auto(cxlr, cxled, pos);
2058	if (rc)
2059	return rc;
2060
2061	/* await more targets to arrive... */
2062	if (p->nr_targets < p->interleave_ways)
2063	return 0;
2064
2065	/*
2066	* All targets are here, which implies all PCI enumeration that
2067	* affects this region has been completed. Walk the topology to
2068	* sort the devices into their relative region decode position.
2069	*/
2070	rc = cxl_region_sort_targets(cxlr);
2071	if (rc)
2072	return rc;
2073
2074	for (i = 0; i < p->nr_targets; i++) {
2075	cxled = p->targets[i];
2076	ep_port = cxled_to_port(cxled);
2077	dport = cxl_find_dport_by_dev(port: root_port,
2078	dport_dev: ep_port->host_bridge);
2079	rc = cxl_region_attach_position(cxlr, cxlrd, cxled,
2080	dport, pos: i);
2081	if (rc)
2082	return rc;
2083	}
2084
2085	rc = cxl_region_setup_targets(cxlr);
2086	if (rc)
2087	return rc;
2088
2089	/*
2090	* If target setup succeeds in the autodiscovery case
2091	* then the region is already committed.
2092	*/
2093	p->state = CXL_CONFIG_COMMIT;
2094	cxl_region_shared_upstream_bandwidth_update(cxlr);
2095
2096	return 0;
2097	}
2098
2099	rc = cxl_region_validate_position(cxlr, cxled, pos);
2100	if (rc)
2101	return rc;
2102
2103	rc = cxl_region_attach_position(cxlr, cxlrd, cxled, dport, pos);
2104	if (rc)
2105	return rc;
2106
2107	p->targets[pos] = cxled;
2108	cxled->pos = pos;
2109	p->nr_targets++;
2110
2111	if (p->nr_targets == p->interleave_ways) {
2112	rc = cxl_region_setup_targets(cxlr);
2113	if (rc)
2114	return rc;
2115	p->state = CXL_CONFIG_ACTIVE;
2116	cxl_region_shared_upstream_bandwidth_update(cxlr);
2117	}
2118
2119	cxled->cxld.interleave_ways = p->interleave_ways;
2120	cxled->cxld.interleave_granularity = p->interleave_granularity;
2121	cxled->cxld.hpa_range = (struct range) {
2122	.start = p->res->start,
2123	.end = p->res->end,
2124	};
2125
2126	if (p->nr_targets != p->interleave_ways)
2127	return 0;
2128
2129	/*
2130	* Test the auto-discovery position calculator function
2131	* against this successfully created user-defined region.
2132	* A fail message here means that this interleave config
2133	* will fail when presented as CXL_REGION_F_AUTO.
2134	*/
2135	for (int i = 0; i < p->nr_targets; i++) {
2136	struct cxl_endpoint_decoder *cxled = p->targets[i];
2137	int test_pos;
2138
2139	test_pos = cxl_calc_interleave_pos(cxled);
2140	dev_dbg(&cxled->cxld.dev,
2141	"Test cxl_calc_interleave_pos(): %s test_pos:%d cxled->pos:%d\n",
2142	(test_pos == cxled->pos) ? "success" : "fail",
2143	test_pos, cxled->pos);
2144	}
2145
2146	return 0;
2147	}
2148
2149	static struct cxl_region *
2150	__cxl_decoder_detach(struct cxl_region *cxlr,
2151	struct cxl_endpoint_decoder *cxled, int pos,
2152	enum cxl_detach_mode mode)
2153	{
2154	struct cxl_region_params *p;
2155
2156	lockdep_assert_held_write(&cxl_rwsem.region);
2157
2158	if (!cxled) {
2159	p = &cxlr->params;
2160
2161	if (pos >= p->interleave_ways) {
2162	dev_dbg(&cxlr->dev, "position %d out of range %d\n",
2163	pos, p->interleave_ways);
2164	return NULL;
2165	}
2166
2167	if (!p->targets[pos])
2168	return NULL;
2169	cxled = p->targets[pos];
2170	} else {
2171	cxlr = cxled->cxld.region;
2172	if (!cxlr)
2173	return NULL;
2174	p = &cxlr->params;
2175	}
2176
2177	if (mode == DETACH_INVALIDATE)
2178	cxled->part = -1;
2179
2180	if (p->state > CXL_CONFIG_ACTIVE) {
2181	cxl_region_decode_reset(cxlr, count: p->interleave_ways);
2182	p->state = CXL_CONFIG_ACTIVE;
2183	}
2184
2185	for (struct cxl_port *iter = cxled_to_port(cxled); !is_cxl_root(port: iter);
2186	iter = to_cxl_port(dev: iter->dev.parent))
2187	cxl_port_detach_region(port: iter, cxlr, cxled);
2188
2189	if (cxled->pos < 0 || cxled->pos >= p->interleave_ways ||
2190	p->targets[cxled->pos] != cxled) {
2191	struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2192
2193	dev_WARN_ONCE(&cxlr->dev, 1, "expected %s:%s at position %d\n",
2194	dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2195	cxled->pos);
2196	return NULL;
2197	}
2198
2199	if (p->state == CXL_CONFIG_ACTIVE) {
2200	p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
2201	cxl_region_teardown_targets(cxlr);
2202	}
2203	p->targets[cxled->pos] = NULL;
2204	p->nr_targets--;
2205	cxled->cxld.hpa_range = (struct range) {
2206	.start = 0,
2207	.end = -1,
2208	};
2209
2210	get_device(dev: &cxlr->dev);
2211	return cxlr;
2212	}
2213
2214	/*
2215	* Cleanup a decoder's interest in a region. There are 2 cases to
2216	* handle, removing an unknown @cxled from a known position in a region
2217	* (detach_target()) or removing a known @cxled from an unknown @cxlr
2218	* (cxld_unregister())
2219	*
2220	* When the detachment finds a region release the region driver.
2221	*/
2222	int cxl_decoder_detach(struct cxl_region *cxlr,
2223	struct cxl_endpoint_decoder *cxled, int pos,
2224	enum cxl_detach_mode mode)
2225	{
2226	struct cxl_region *detach;
2227
2228	/* when the decoder is being destroyed lock unconditionally */
2229	if (mode == DETACH_INVALIDATE) {
2230	guard(rwsem_write)(T: &cxl_rwsem.region);
2231	detach = __cxl_decoder_detach(cxlr, cxled, pos, mode);
2232	} else {
2233	int rc;
2234
2235	ACQUIRE(rwsem_write_kill, rwsem)(T: &cxl_rwsem.region);
2236	if ((rc = ACQUIRE_ERR(rwsem_write_kill, &rwsem)))
2237	return rc;
2238	detach = __cxl_decoder_detach(cxlr, cxled, pos, mode);
2239	}
2240
2241	if (detach) {
2242	device_release_driver(dev: &detach->dev);
2243	put_device(dev: &detach->dev);
2244	}
2245	return 0;
2246	}
2247
2248	static int __attach_target(struct cxl_region *cxlr,
2249	struct cxl_endpoint_decoder *cxled, int pos,
2250	unsigned int state)
2251	{
2252	int rc;
2253
2254	if (state == TASK_INTERRUPTIBLE) {
2255	ACQUIRE(rwsem_write_kill, rwsem)(T: &cxl_rwsem.region);
2256	if ((rc = ACQUIRE_ERR(rwsem_write_kill, &rwsem)))
2257	return rc;
2258	guard(rwsem_read)(T: &cxl_rwsem.dpa);
2259	return cxl_region_attach(cxlr, cxled, pos);
2260	}
2261	guard(rwsem_write)(T: &cxl_rwsem.region);
2262	guard(rwsem_read)(T: &cxl_rwsem.dpa);
2263	return cxl_region_attach(cxlr, cxled, pos);
2264	}
2265
2266	static int attach_target(struct cxl_region *cxlr,
2267	struct cxl_endpoint_decoder *cxled, int pos,
2268	unsigned int state)
2269	{
2270	int rc = __attach_target(cxlr, cxled, pos, state);
2271
2272	if (rc == 0)
2273	return 0;
2274
2275	dev_warn(cxled->cxld.dev.parent, "failed to attach %s to %s: %d\n",
2276	dev_name(&cxled->cxld.dev), dev_name(&cxlr->dev), rc);
2277	return rc;
2278	}
2279
2280	static int detach_target(struct cxl_region *cxlr, int pos)
2281	{
2282	return cxl_decoder_detach(cxlr, NULL, pos, mode: DETACH_ONLY);
2283	}
2284
2285	static size_t store_targetN(struct cxl_region *cxlr, const char *buf, int pos,
2286	size_t len)
2287	{
2288	int rc;
2289
2290	if (sysfs_streq(s1: buf, s2: "\n"))
2291	rc = detach_target(cxlr, pos);
2292	else {
2293	struct device *dev;
2294
2295	dev = bus_find_device_by_name(bus: &cxl_bus_type, NULL, name: buf);
2296	if (!dev)
2297	return -ENODEV;
2298
2299	if (!is_endpoint_decoder(dev)) {
2300	rc = -EINVAL;
2301	goto out;
2302	}
2303
2304	rc = attach_target(cxlr, cxled: to_cxl_endpoint_decoder(dev), pos,
2305	TASK_INTERRUPTIBLE);
2306	out:
2307	put_device(dev);
2308	}
2309
2310	if (rc < 0)
2311	return rc;
2312	return len;
2313	}
2314
2315	#define TARGET_ATTR_RW(n) \
2316	static ssize_t target##n##_show( \
2317	struct device *dev, struct device_attribute *attr, char *buf) \
2318	{ \
2319	return show_targetN(to_cxl_region(dev), buf, (n)); \
2320	} \
2321	static ssize_t target##n##_store(struct device *dev, \
2322	struct device_attribute *attr, \
2323	const char *buf, size_t len) \
2324	{ \
2325	return store_targetN(to_cxl_region(dev), buf, (n), len); \
2326	} \
2327	static DEVICE_ATTR_RW(target##n)
2328
2329	TARGET_ATTR_RW(0);
2330	TARGET_ATTR_RW(1);
2331	TARGET_ATTR_RW(2);
2332	TARGET_ATTR_RW(3);
2333	TARGET_ATTR_RW(4);
2334	TARGET_ATTR_RW(5);
2335	TARGET_ATTR_RW(6);
2336	TARGET_ATTR_RW(7);
2337	TARGET_ATTR_RW(8);
2338	TARGET_ATTR_RW(9);
2339	TARGET_ATTR_RW(10);
2340	TARGET_ATTR_RW(11);
2341	TARGET_ATTR_RW(12);
2342	TARGET_ATTR_RW(13);
2343	TARGET_ATTR_RW(14);
2344	TARGET_ATTR_RW(15);
2345
2346	static struct attribute *target_attrs[] = {
2347	&dev_attr_target0.attr,
2348	&dev_attr_target1.attr,
2349	&dev_attr_target2.attr,
2350	&dev_attr_target3.attr,
2351	&dev_attr_target4.attr,
2352	&dev_attr_target5.attr,
2353	&dev_attr_target6.attr,
2354	&dev_attr_target7.attr,
2355	&dev_attr_target8.attr,
2356	&dev_attr_target9.attr,
2357	&dev_attr_target10.attr,
2358	&dev_attr_target11.attr,
2359	&dev_attr_target12.attr,
2360	&dev_attr_target13.attr,
2361	&dev_attr_target14.attr,
2362	&dev_attr_target15.attr,
2363	NULL,
2364	};
2365
2366	static umode_t cxl_region_target_visible(struct kobject *kobj,
2367	struct attribute *a, int n)
2368	{
2369	struct device *dev = kobj_to_dev(kobj);
2370	struct cxl_region *cxlr = to_cxl_region(dev);
2371	struct cxl_region_params *p = &cxlr->params;
2372
2373	if (n < p->interleave_ways)
2374	return a->mode;
2375	return 0;
2376	}
2377
2378	static const struct attribute_group cxl_region_target_group = {
2379	.attrs = target_attrs,
2380	.is_visible = cxl_region_target_visible,
2381	};
2382
2383	static const struct attribute_group *get_cxl_region_target_group(void)
2384	{
2385	return &cxl_region_target_group;
2386	}
2387
2388	static const struct attribute_group *region_groups[] = {
2389	&cxl_base_attribute_group,
2390	&cxl_region_group,
2391	&cxl_region_target_group,
2392	&cxl_region_access0_coordinate_group,
2393	&cxl_region_access1_coordinate_group,
2394	NULL,
2395	};
2396
2397	static void cxl_region_release(struct device *dev)
2398	{
2399	struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev: dev->parent);
2400	struct cxl_region *cxlr = to_cxl_region(dev);
2401	int id = atomic_read(v: &cxlrd->region_id);
2402
2403	/*
2404	* Try to reuse the recently idled id rather than the cached
2405	* next id to prevent the region id space from increasing
2406	* unnecessarily.
2407	*/
2408	if (cxlr->id < id)
2409	if (atomic_try_cmpxchg(v: &cxlrd->region_id, old: &id, new: cxlr->id)) {
2410	memregion_free(id);
2411	goto out;
2412	}
2413
2414	memregion_free(id: cxlr->id);
2415	out:
2416	put_device(dev: dev->parent);
2417	kfree(objp: cxlr);
2418	}
2419
2420	const struct device_type cxl_region_type = {
2421	.name = "cxl_region",
2422	.release = cxl_region_release,
2423	.groups = region_groups
2424	};
2425
2426	bool is_cxl_region(struct device *dev)
2427	{
2428	return dev->type == &cxl_region_type;
2429	}
2430	EXPORT_SYMBOL_NS_GPL(is_cxl_region, "CXL");
2431
2432	static struct cxl_region *to_cxl_region(struct device *dev)
2433	{
2434	if (dev_WARN_ONCE(dev, dev->type != &cxl_region_type,
2435	"not a cxl_region device\n"))
2436	return NULL;
2437
2438	return container_of(dev, struct cxl_region, dev);
2439	}
2440
2441	static void unregister_region(void *_cxlr)
2442	{
2443	struct cxl_region *cxlr = _cxlr;
2444	struct cxl_region_params *p = &cxlr->params;
2445	int i;
2446
2447	device_del(dev: &cxlr->dev);
2448
2449	/*
2450	* Now that region sysfs is shutdown, the parameter block is now
2451	* read-only, so no need to hold the region rwsem to access the
2452	* region parameters.
2453	*/
2454	for (i = 0; i < p->interleave_ways; i++)
2455	detach_target(cxlr, pos: i);
2456
2457	cxl_region_iomem_release(cxlr);
2458	put_device(dev: &cxlr->dev);
2459	}
2460
2461	static struct lock_class_key cxl_region_key;
2462
2463	static struct cxl_region *cxl_region_alloc(struct cxl_root_decoder *cxlrd, int id)
2464	{
2465	struct cxl_region *cxlr;
2466	struct device *dev;
2467
2468	cxlr = kzalloc(sizeof(*cxlr), GFP_KERNEL);
2469	if (!cxlr) {
2470	memregion_free(id);
2471	return ERR_PTR(error: -ENOMEM);
2472	}
2473
2474	dev = &cxlr->dev;
2475	device_initialize(dev);
2476	lockdep_set_class(&dev->mutex, &cxl_region_key);
2477	dev->parent = &cxlrd->cxlsd.cxld.dev;
2478	/*
2479	* Keep root decoder pinned through cxl_region_release to fixup
2480	* region id allocations
2481	*/
2482	get_device(dev: dev->parent);
2483	device_set_pm_not_required(dev);
2484	dev->bus = &cxl_bus_type;
2485	dev->type = &cxl_region_type;
2486	cxlr->id = id;
2487	cxl_region_set_lock(cxlr, cxld: &cxlrd->cxlsd.cxld);
2488
2489	return cxlr;
2490	}
2491
2492	static bool cxl_region_update_coordinates(struct cxl_region *cxlr, int nid)
2493	{
2494	int cset = 0;
2495	int rc;
2496
2497	for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
2498	if (cxlr->coord[i].read_bandwidth) {
2499	node_update_perf_attrs(nid, coord: &cxlr->coord[i], access: i);
2500	cset++;
2501	}
2502	}
2503
2504	if (!cset)
2505	return false;
2506
2507	rc = sysfs_update_group(kobj: &cxlr->dev.kobj, grp: get_cxl_region_access0_group());
2508	if (rc)
2509	dev_dbg(&cxlr->dev, "Failed to update access0 group\n");
2510
2511	rc = sysfs_update_group(kobj: &cxlr->dev.kobj, grp: get_cxl_region_access1_group());
2512	if (rc)
2513	dev_dbg(&cxlr->dev, "Failed to update access1 group\n");
2514
2515	return true;
2516	}
2517
2518	static int cxl_region_perf_attrs_callback(struct notifier_block *nb,
2519	unsigned long action, void *arg)
2520	{
2521	struct cxl_region *cxlr = container_of(nb, struct cxl_region,
2522	node_notifier);
2523	struct node_notify *nn = arg;
2524	int nid = nn->nid;
2525	int region_nid;
2526
2527	if (action != NODE_ADDED_FIRST_MEMORY)
2528	return NOTIFY_DONE;
2529
2530	/*
2531	* No need to hold cxl_rwsem.region; region parameters are stable
2532	* within the cxl_region driver.
2533	*/
2534	region_nid = phys_to_target_node(start: cxlr->params.res->start);
2535	if (nid != region_nid)
2536	return NOTIFY_DONE;
2537
2538	/* No action needed if node bit already set */
2539	if (node_test_and_set(nid, nodemask_region_seen))
2540	return NOTIFY_DONE;
2541
2542	if (!cxl_region_update_coordinates(cxlr, nid))
2543	return NOTIFY_DONE;
2544
2545	return NOTIFY_OK;
2546	}
2547
2548	static int cxl_region_calculate_adistance(struct notifier_block *nb,
2549	unsigned long nid, void *data)
2550	{
2551	struct cxl_region *cxlr = container_of(nb, struct cxl_region,
2552	adist_notifier);
2553	struct access_coordinate *perf;
2554	int *adist = data;
2555	int region_nid;
2556
2557	/*
2558	* No need to hold cxl_rwsem.region; region parameters are stable
2559	* within the cxl_region driver.
2560	*/
2561	region_nid = phys_to_target_node(start: cxlr->params.res->start);
2562	if (nid != region_nid)
2563	return NOTIFY_OK;
2564
2565	perf = &cxlr->coord[ACCESS_COORDINATE_CPU];
2566
2567	if (mt_perf_to_adistance(perf, adist))
2568	return NOTIFY_OK;
2569
2570	return NOTIFY_STOP;
2571	}
2572
2573	/**
2574	* devm_cxl_add_region - Adds a region to a decoder
2575	* @cxlrd: root decoder
2576	* @id: memregion id to create, or memregion_free() on failure
2577	* @mode: mode for the endpoint decoders of this region
2578	* @type: select whether this is an expander or accelerator (type-2 or type-3)
2579	*
2580	* This is the second step of region initialization. Regions exist within an
2581	* address space which is mapped by a @cxlrd.
2582	*
2583	* Return: 0 if the region was added to the @cxlrd, else returns negative error
2584	* code. The region will be named "regionZ" where Z is the unique region number.
2585	*/
2586	static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd,
2587	int id,
2588	enum cxl_partition_mode mode,
2589	enum cxl_decoder_type type)
2590	{
2591	struct cxl_port *port = to_cxl_port(dev: cxlrd->cxlsd.cxld.dev.parent);
2592	struct cxl_region *cxlr;
2593	struct device *dev;
2594	int rc;
2595
2596	cxlr = cxl_region_alloc(cxlrd, id);
2597	if (IS_ERR(ptr: cxlr))
2598	return cxlr;
2599	cxlr->mode = mode;
2600	cxlr->type = type;
2601
2602	dev = &cxlr->dev;
2603	rc = dev_set_name(dev, name: "region%d", id);
2604	if (rc)
2605	goto err;
2606
2607	rc = device_add(dev);
2608	if (rc)
2609	goto err;
2610
2611	rc = devm_add_action_or_reset(port->uport_dev, unregister_region, cxlr);
2612	if (rc)
2613	return ERR_PTR(error: rc);
2614
2615	dev_dbg(port->uport_dev, "%s: created %s\n",
2616	dev_name(&cxlrd->cxlsd.cxld.dev), dev_name(dev));
2617	return cxlr;
2618
2619	err:
2620	put_device(dev);
2621	return ERR_PTR(error: rc);
2622	}
2623
2624	static ssize_t __create_region_show(struct cxl_root_decoder *cxlrd, char *buf)
2625	{
2626	return sysfs_emit(buf, fmt: "region%u\n", atomic_read(v: &cxlrd->region_id));
2627	}
2628
2629	static ssize_t create_pmem_region_show(struct device *dev,
2630	struct device_attribute *attr, char *buf)
2631	{
2632	return __create_region_show(cxlrd: to_cxl_root_decoder(dev), buf);
2633	}
2634
2635	static ssize_t create_ram_region_show(struct device *dev,
2636	struct device_attribute *attr, char *buf)
2637	{
2638	return __create_region_show(cxlrd: to_cxl_root_decoder(dev), buf);
2639	}
2640
2641	static struct cxl_region *__create_region(struct cxl_root_decoder *cxlrd,
2642	enum cxl_partition_mode mode, int id)
2643	{
2644	int rc;
2645
2646	switch (mode) {
2647	case CXL_PARTMODE_RAM:
2648	case CXL_PARTMODE_PMEM:
2649	break;
2650	default:
2651	dev_err(&cxlrd->cxlsd.cxld.dev, "unsupported mode %d\n", mode);
2652	return ERR_PTR(error: -EINVAL);
2653	}
2654
2655	rc = memregion_alloc(GFP_KERNEL);
2656	if (rc < 0)
2657	return ERR_PTR(error: rc);
2658
2659	if (atomic_cmpxchg(v: &cxlrd->region_id, old: id, new: rc) != id) {
2660	memregion_free(id: rc);
2661	return ERR_PTR(error: -EBUSY);
2662	}
2663
2664	return devm_cxl_add_region(cxlrd, id, mode, type: CXL_DECODER_HOSTONLYMEM);
2665	}
2666
2667	static ssize_t create_region_store(struct device *dev, const char *buf,
2668	size_t len, enum cxl_partition_mode mode)
2669	{
2670	struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2671	struct cxl_region *cxlr;
2672	int rc, id;
2673
2674	rc = sscanf(buf, "region%d\n", &id);
2675	if (rc != 1)
2676	return -EINVAL;
2677
2678	cxlr = __create_region(cxlrd, mode, id);
2679	if (IS_ERR(ptr: cxlr))
2680	return PTR_ERR(ptr: cxlr);
2681
2682	return len;
2683	}
2684
2685	static ssize_t create_pmem_region_store(struct device *dev,
2686	struct device_attribute *attr,
2687	const char *buf, size_t len)
2688	{
2689	return create_region_store(dev, buf, len, mode: CXL_PARTMODE_PMEM);
2690	}
2691	DEVICE_ATTR_RW(create_pmem_region);
2692
2693	static ssize_t create_ram_region_store(struct device *dev,
2694	struct device_attribute *attr,
2695	const char *buf, size_t len)
2696	{
2697	return create_region_store(dev, buf, len, mode: CXL_PARTMODE_RAM);
2698	}
2699	DEVICE_ATTR_RW(create_ram_region);
2700
2701	static ssize_t region_show(struct device *dev, struct device_attribute *attr,
2702	char *buf)
2703	{
2704	struct cxl_decoder *cxld = to_cxl_decoder(dev);
2705	ssize_t rc;
2706
2707	ACQUIRE(rwsem_read_intr, rwsem)(T: &cxl_rwsem.region);
2708	if ((rc = ACQUIRE_ERR(rwsem_read_intr, &rwsem)))
2709	return rc;
2710
2711	if (cxld->region)
2712	return sysfs_emit(buf, fmt: "%s\n", dev_name(dev: &cxld->region->dev));
2713	return sysfs_emit(buf, fmt: "\n");
2714	}
2715	DEVICE_ATTR_RO(region);
2716
2717	static struct cxl_region *
2718	cxl_find_region_by_name(struct cxl_root_decoder *cxlrd, const char *name)
2719	{
2720	struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
2721	struct device *region_dev;
2722
2723	region_dev = device_find_child_by_name(parent: &cxld->dev, name);
2724	if (!region_dev)
2725	return ERR_PTR(error: -ENODEV);
2726
2727	return to_cxl_region(dev: region_dev);
2728	}
2729
2730	static ssize_t delete_region_store(struct device *dev,
2731	struct device_attribute *attr,
2732	const char *buf, size_t len)
2733	{
2734	struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2735	struct cxl_port *port = to_cxl_port(dev: dev->parent);
2736	struct cxl_region *cxlr;
2737
2738	cxlr = cxl_find_region_by_name(cxlrd, name: buf);
2739	if (IS_ERR(ptr: cxlr))
2740	return PTR_ERR(ptr: cxlr);
2741
2742	devm_release_action(dev: port->uport_dev, action: unregister_region, data: cxlr);
2743	put_device(dev: &cxlr->dev);
2744
2745	return len;
2746	}
2747	DEVICE_ATTR_WO(delete_region);
2748
2749	static void cxl_pmem_region_release(struct device *dev)
2750	{
2751	struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev);
2752	int i;
2753
2754	for (i = 0; i < cxlr_pmem->nr_mappings; i++) {
2755	struct cxl_memdev *cxlmd = cxlr_pmem->mapping[i].cxlmd;
2756
2757	put_device(dev: &cxlmd->dev);
2758	}
2759
2760	kfree(objp: cxlr_pmem);
2761	}
2762
2763	static const struct attribute_group *cxl_pmem_region_attribute_groups[] = {
2764	&cxl_base_attribute_group,
2765	NULL,
2766	};
2767
2768	const struct device_type cxl_pmem_region_type = {
2769	.name = "cxl_pmem_region",
2770	.release = cxl_pmem_region_release,
2771	.groups = cxl_pmem_region_attribute_groups,
2772	};
2773
2774	bool is_cxl_pmem_region(struct device *dev)
2775	{
2776	return dev->type == &cxl_pmem_region_type;
2777	}
2778	EXPORT_SYMBOL_NS_GPL(is_cxl_pmem_region, "CXL");
2779
2780	struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev)
2781	{
2782	if (dev_WARN_ONCE(dev, !is_cxl_pmem_region(dev),
2783	"not a cxl_pmem_region device\n"))
2784	return NULL;
2785	return container_of(dev, struct cxl_pmem_region, dev);
2786	}
2787	EXPORT_SYMBOL_NS_GPL(to_cxl_pmem_region, "CXL");
2788
2789	struct cxl_poison_context {
2790	struct cxl_port *port;
2791	int part;
2792	u64 offset;
2793	};
2794
2795	static int cxl_get_poison_unmapped(struct cxl_memdev *cxlmd,
2796	struct cxl_poison_context *ctx)
2797	{
2798	struct cxl_dev_state *cxlds = cxlmd->cxlds;
2799	const struct resource *res;
2800	struct resource *p, *last;
2801	u64 offset, length;
2802	int rc = 0;
2803
2804	if (ctx->part < 0)
2805	return 0;
2806
2807	/*
2808	* Collect poison for the remaining unmapped resources after
2809	* poison is collected by committed endpoints decoders.
2810	*/
2811	for (int i = ctx->part; i < cxlds->nr_partitions; i++) {
2812	res = &cxlds->part[i].res;
2813	for (p = res->child, last = NULL; p; p = p->sibling)
2814	last = p;
2815	if (last)
2816	offset = last->end + 1;
2817	else
2818	offset = res->start;
2819	length = res->end - offset + 1;
2820	if (!length)
2821	break;
2822	rc = cxl_mem_get_poison(cxlmd, offset, len: length, NULL);
2823	if (rc == -EFAULT && cxlds->part[i].mode == CXL_PARTMODE_RAM)
2824	continue;
2825	if (rc)
2826	break;
2827	}
2828
2829	return rc;
2830	}
2831
2832	static int poison_by_decoder(struct device *dev, void *arg)
2833	{
2834	struct cxl_poison_context *ctx = arg;
2835	struct cxl_endpoint_decoder *cxled;
2836	enum cxl_partition_mode mode;
2837	struct cxl_dev_state *cxlds;
2838	struct cxl_memdev *cxlmd;
2839	u64 offset, length;
2840	int rc = 0;
2841
2842	if (!is_endpoint_decoder(dev))
2843	return rc;
2844
2845	cxled = to_cxl_endpoint_decoder(dev);
2846	if (!cxled->dpa_res)
2847	return rc;
2848
2849	cxlmd = cxled_to_memdev(cxled);
2850	cxlds = cxlmd->cxlds;
2851	mode = cxlds->part[cxled->part].mode;
2852
2853	if (cxled->skip) {
2854	offset = cxled->dpa_res->start - cxled->skip;
2855	length = cxled->skip;
2856	rc = cxl_mem_get_poison(cxlmd, offset, len: length, NULL);
2857	if (rc == -EFAULT && mode == CXL_PARTMODE_RAM)
2858	rc = 0;
2859	if (rc)
2860	return rc;
2861	}
2862
2863	offset = cxled->dpa_res->start;
2864	length = cxled->dpa_res->end - offset + 1;
2865	rc = cxl_mem_get_poison(cxlmd, offset, len: length, cxlr: cxled->cxld.region);
2866	if (rc == -EFAULT && mode == CXL_PARTMODE_RAM)
2867	rc = 0;
2868	if (rc)
2869	return rc;
2870
2871	/* Iterate until commit_end is reached */
2872	if (cxled->cxld.id == ctx->port->commit_end) {
2873	ctx->offset = cxled->dpa_res->end + 1;
2874	ctx->part = cxled->part;
2875	return 1;
2876	}
2877
2878	return 0;
2879	}
2880
2881	int cxl_get_poison_by_endpoint(struct cxl_port *port)
2882	{
2883	struct cxl_poison_context ctx;
2884	int rc = 0;
2885
2886	ctx = (struct cxl_poison_context) {
2887	.port = port,
2888	.part = -1,
2889	};
2890
2891	rc = device_for_each_child(parent: &port->dev, data: &ctx, fn: poison_by_decoder);
2892	if (rc == 1)
2893	rc = cxl_get_poison_unmapped(cxlmd: to_cxl_memdev(dev: port->uport_dev),
2894	ctx: &ctx);
2895
2896	return rc;
2897	}
2898
2899	struct cxl_dpa_to_region_context {
2900	struct cxl_region *cxlr;
2901	u64 dpa;
2902	};
2903
2904	static int __cxl_dpa_to_region(struct device *dev, void *arg)
2905	{
2906	struct cxl_dpa_to_region_context *ctx = arg;
2907	struct cxl_endpoint_decoder *cxled;
2908	struct cxl_region *cxlr;
2909	u64 dpa = ctx->dpa;
2910
2911	if (!is_endpoint_decoder(dev))
2912	return 0;
2913
2914	cxled = to_cxl_endpoint_decoder(dev);
2915	if (!cxled || !cxled->dpa_res || !resource_size(res: cxled->dpa_res))
2916	return 0;
2917
2918	if (!cxl_resource_contains_addr(res: cxled->dpa_res, addr: dpa))
2919	return 0;
2920
2921	/*
2922	* Stop the region search (return 1) when an endpoint mapping is
2923	* found. The region may not be fully constructed so offering
2924	* the cxlr in the context structure is not guaranteed.
2925	*/
2926	cxlr = cxled->cxld.region;
2927	if (cxlr)
2928	dev_dbg(dev, "dpa:0x%llx mapped in region:%s\n", dpa,
2929	dev_name(&cxlr->dev));
2930	else
2931	dev_dbg(dev, "dpa:0x%llx mapped in endpoint:%s\n", dpa,
2932	dev_name(dev));
2933
2934	ctx->cxlr = cxlr;
2935
2936	return 1;
2937	}
2938
2939	struct cxl_region *cxl_dpa_to_region(const struct cxl_memdev *cxlmd, u64 dpa)
2940	{
2941	struct cxl_dpa_to_region_context ctx;
2942	struct cxl_port *port;
2943
2944	ctx = (struct cxl_dpa_to_region_context) {
2945	.dpa = dpa,
2946	};
2947	port = cxlmd->endpoint;
2948	if (port && is_cxl_endpoint(port) && cxl_num_decoders_committed(port))
2949	device_for_each_child(parent: &port->dev, data: &ctx, fn: __cxl_dpa_to_region);
2950
2951	return ctx.cxlr;
2952	}
2953
2954	static bool cxl_is_hpa_in_chunk(u64 hpa, struct cxl_region *cxlr, int pos)
2955	{
2956	struct cxl_region_params *p = &cxlr->params;
2957	int gran = p->interleave_granularity;
2958	int ways = p->interleave_ways;
2959	u64 offset;
2960
2961	/* Is the hpa in an expected chunk for its pos(-ition) */
2962	offset = hpa - p->res->start;
2963	offset = do_div(offset, gran * ways);
2964	if ((offset >= pos * gran) && (offset < (pos + 1) * gran))
2965	return true;
2966
2967	dev_dbg(&cxlr->dev,
2968	"Addr trans fail: hpa 0x%llx not in expected chunk\n", hpa);
2969
2970	return false;
2971	}
2972
2973	#define CXL_POS_ZERO 0
2974	/**
2975	* cxl_validate_translation_params
2976	* @eiw: encoded interleave ways
2977	* @eig: encoded interleave granularity
2978	* @pos: position in interleave
2979	*
2980	* Callers pass CXL_POS_ZERO when no position parameter needs validating.
2981	*
2982	* Returns: 0 on success, -EINVAL on first invalid parameter
2983	*/
2984	int cxl_validate_translation_params(u8 eiw, u16 eig, int pos)
2985	{
2986	int ways, gran;
2987
2988	if (eiw_to_ways(eiw, ways: &ways)) {
2989	pr_debug("%s: invalid eiw=%u\n", __func__, eiw);
2990	return -EINVAL;
2991	}
2992	if (eig_to_granularity(eig, granularity: &gran)) {
2993	pr_debug("%s: invalid eig=%u\n", __func__, eig);
2994	return -EINVAL;
2995	}
2996	if (pos < 0 || pos >= ways) {
2997	pr_debug("%s: invalid pos=%d for ways=%u\n", __func__, pos,
2998	ways);
2999	return -EINVAL;
3000	}
3001
3002	return 0;
3003	}
3004	EXPORT_SYMBOL_FOR_MODULES(cxl_validate_translation_params, "cxl_translate");
3005
3006	u64 cxl_calculate_dpa_offset(u64 hpa_offset, u8 eiw, u16 eig)
3007	{
3008	u64 dpa_offset, bits_lower, bits_upper, temp;
3009	int ret;
3010
3011	ret = cxl_validate_translation_params(eiw, eig, CXL_POS_ZERO);
3012	if (ret)
3013	return ULLONG_MAX;
3014
3015	/*
3016	* DPA offset: CXL Spec 3.2 Section 8.2.4.20.13
3017	* Lower bits [IG+7:0] pass through unchanged
3018	* (eiw < 8)
3019	* Per spec: DPAOffset[51:IG+8] = (HPAOffset[51:IG+IW+8] >> IW)
3020	* Clear the position bits to isolate upper section, then
3021	* reverse the left shift by eiw that occurred during DPA->HPA
3022	* (eiw >= 8)
3023	* Per spec: DPAOffset[51:IG+8] = HPAOffset[51:IG+IW] / 3
3024	* Extract upper bits from the correct bit range and divide by 3
3025	* to recover the original DPA upper bits
3026	*/
3027	bits_lower = hpa_offset & GENMASK_ULL(eig + 7, 0);
3028	if (eiw < 8) {
3029	temp = hpa_offset &= ~GENMASK_ULL(eig + eiw + 8 - 1, 0);
3030	dpa_offset = temp >> eiw;
3031	} else {
3032	bits_upper = div64_u64(dividend: hpa_offset >> (eig + eiw), divisor: 3);
3033	dpa_offset = bits_upper << (eig + 8);
3034	}
3035	dpa_offset |= bits_lower;
3036
3037	return dpa_offset;
3038	}
3039	EXPORT_SYMBOL_FOR_MODULES(cxl_calculate_dpa_offset, "cxl_translate");
3040
3041	int cxl_calculate_position(u64 hpa_offset, u8 eiw, u16 eig)
3042	{
3043	unsigned int ways = 0;
3044	u64 shifted, rem;
3045	int pos, ret;
3046
3047	ret = cxl_validate_translation_params(eiw, eig, CXL_POS_ZERO);
3048	if (ret)
3049	return ret;
3050
3051	if (!eiw)
3052	/* position is 0 if no interleaving */
3053	return 0;
3054
3055	/*
3056	* Interleave position: CXL Spec 3.2 Section 8.2.4.20.13
3057	* eiw < 8
3058	* Position is in the IW bits at HPA_OFFSET[IG+8+IW-1:IG+8].
3059	* Per spec "remove IW bits starting with bit position IG+8"
3060	* eiw >= 8
3061	* Position is not explicitly stored in HPA_OFFSET bits. It is
3062	* derived from the modulo operation of the upper bits using
3063	* the total number of interleave ways.
3064	*/
3065	if (eiw < 8) {
3066	pos = (hpa_offset >> (eig + 8)) & GENMASK(eiw - 1, 0);
3067	} else {
3068	shifted = hpa_offset >> (eig + 8);
3069	eiw_to_ways(eiw, ways: &ways);
3070	div64_u64_rem(dividend: shifted, divisor: ways, remainder: &rem);
3071	pos = rem;
3072	}
3073
3074	return pos;
3075	}
3076	EXPORT_SYMBOL_FOR_MODULES(cxl_calculate_position, "cxl_translate");
3077
3078	u64 cxl_calculate_hpa_offset(u64 dpa_offset, int pos, u8 eiw, u16 eig)
3079	{
3080	u64 mask_upper, hpa_offset, bits_upper;
3081	int ret;
3082
3083	ret = cxl_validate_translation_params(eiw, eig, pos);
3084	if (ret)
3085	return ULLONG_MAX;
3086
3087	/*
3088	* The device position in the region interleave set was removed
3089	* from the offset at HPA->DPA translation. To reconstruct the
3090	* HPA, place the 'pos' in the offset.
3091	*
3092	* The placement of 'pos' in the HPA is determined by interleave
3093	* ways and granularity and is defined in the CXL Spec 3.0 Section
3094	* 8.2.4.19.13 Implementation Note: Device Decode Logic
3095	*/
3096
3097	mask_upper = GENMASK_ULL(51, eig + 8);
3098
3099	if (eiw < 8) {
3100	hpa_offset = (dpa_offset & mask_upper) << eiw;
3101	hpa_offset |= pos << (eig + 8);
3102	} else {
3103	bits_upper = (dpa_offset & mask_upper) >> (eig + 8);
3104	bits_upper = bits_upper * 3;
3105	hpa_offset = ((bits_upper << (eiw - 8)) + pos) << (eig + 8);
3106	}
3107
3108	/* The lower bits remain unchanged */
3109	hpa_offset |= dpa_offset & GENMASK_ULL(eig + 7, 0);
3110
3111	return hpa_offset;
3112	}
3113	EXPORT_SYMBOL_FOR_MODULES(cxl_calculate_hpa_offset, "cxl_translate");
3114
3115	u64 cxl_dpa_to_hpa(struct cxl_region *cxlr, const struct cxl_memdev *cxlmd,
3116	u64 dpa)
3117	{
3118	struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev: cxlr->dev.parent);
3119	struct cxl_region_params *p = &cxlr->params;
3120	struct cxl_endpoint_decoder *cxled = NULL;
3121	u64 base, dpa_offset, hpa_offset, hpa;
3122	u16 eig = 0;
3123	u8 eiw = 0;
3124	int pos;
3125
3126	for (int i = 0; i < p->nr_targets; i++) {
3127	if (cxlmd == cxled_to_memdev(cxled: p->targets[i])) {
3128	cxled = p->targets[i];
3129	break;
3130	}
3131	}
3132	if (!cxled)
3133	return ULLONG_MAX;
3134
3135	pos = cxled->pos;
3136	ways_to_eiw(ways: p->interleave_ways, eiw: &eiw);
3137	granularity_to_eig(granularity: p->interleave_granularity, eig: &eig);
3138
3139	base = cxl_dpa_resource_start(cxled);
3140	if (base == RESOURCE_SIZE_MAX)
3141	return ULLONG_MAX;
3142
3143	dpa_offset = dpa - base;
3144	hpa_offset = cxl_calculate_hpa_offset(dpa_offset, pos, eiw, eig);
3145	if (hpa_offset == ULLONG_MAX)
3146	return ULLONG_MAX;
3147
3148	/* Apply the hpa_offset to the region base address */
3149	hpa = hpa_offset + p->res->start + p->cache_size;
3150
3151	/* Root decoder translation overrides typical modulo decode */
3152	if (cxlrd->ops.hpa_to_spa)
3153	hpa = cxlrd->ops.hpa_to_spa(cxlrd, hpa);
3154
3155	if (hpa == ULLONG_MAX)
3156	return ULLONG_MAX;
3157
3158	if (!cxl_resource_contains_addr(res: p->res, addr: hpa)) {
3159	dev_dbg(&cxlr->dev,
3160	"Addr trans fail: hpa 0x%llx not in region\n", hpa);
3161	return ULLONG_MAX;
3162	}
3163
3164	/* Simple chunk check, by pos & gran, only applies to modulo decodes */
3165	if (!cxlrd->ops.hpa_to_spa && !cxl_is_hpa_in_chunk(hpa, cxlr, pos))
3166	return ULLONG_MAX;
3167
3168	return hpa;
3169	}
3170
3171	struct dpa_result {
3172	struct cxl_memdev *cxlmd;
3173	u64 dpa;
3174	};
3175
3176	static int region_offset_to_dpa_result(struct cxl_region *cxlr, u64 offset,
3177	struct dpa_result *result)
3178	{
3179	struct cxl_region_params *p = &cxlr->params;
3180	struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev: cxlr->dev.parent);
3181	struct cxl_endpoint_decoder *cxled;
3182	u64 hpa_offset = offset;
3183	u64 dpa, dpa_offset;
3184	u16 eig = 0;
3185	u8 eiw = 0;
3186	int pos;
3187
3188	lockdep_assert_held(&cxl_rwsem.region);
3189	lockdep_assert_held(&cxl_rwsem.dpa);
3190
3191	/* Input validation ensures valid ways and gran */
3192	granularity_to_eig(granularity: p->interleave_granularity, eig: &eig);
3193	ways_to_eiw(ways: p->interleave_ways, eiw: &eiw);
3194
3195	/*
3196	* If the root decoder has SPA to CXL HPA callback, use it. Otherwise
3197	* CXL HPA is assumed to equal SPA.
3198	*/
3199	if (cxlrd->ops.spa_to_hpa) {
3200	hpa_offset = cxlrd->ops.spa_to_hpa(cxlrd, p->res->start + offset);
3201	if (hpa_offset == ULLONG_MAX) {
3202	dev_dbg(&cxlr->dev, "HPA not found for %pr offset %#llx\n",
3203	p->res, offset);
3204	return -ENXIO;
3205	}
3206	hpa_offset -= p->res->start;
3207	}
3208
3209	pos = cxl_calculate_position(hpa_offset, eiw, eig);
3210	if (pos < 0 || pos >= p->nr_targets) {
3211	dev_dbg(&cxlr->dev, "Invalid position %d for %d targets\n",
3212	pos, p->nr_targets);
3213	return -ENXIO;
3214	}
3215
3216	dpa_offset = cxl_calculate_dpa_offset(hpa_offset, eiw, eig);
3217
3218	/* Look-up and return the result: a memdev and a DPA */
3219	for (int i = 0; i < p->nr_targets; i++) {
3220	cxled = p->targets[i];
3221	if (cxled->pos != pos)
3222	continue;
3223
3224	dpa = cxl_dpa_resource_start(cxled);
3225	if (dpa != RESOURCE_SIZE_MAX)
3226	dpa += dpa_offset;
3227
3228	result->cxlmd = cxled_to_memdev(cxled);
3229	result->dpa = dpa;
3230
3231	return 0;
3232	}
3233	dev_err(&cxlr->dev, "No device found for position %d\n", pos);
3234
3235	return -ENXIO;
3236	}
3237
3238	static struct lock_class_key cxl_pmem_region_key;
3239
3240	static int cxl_pmem_region_alloc(struct cxl_region *cxlr)
3241	{
3242	struct cxl_region_params *p = &cxlr->params;
3243	struct cxl_nvdimm_bridge *cxl_nvb;
3244	struct device *dev;
3245	int i;
3246
3247	guard(rwsem_read)(T: &cxl_rwsem.region);
3248	if (p->state != CXL_CONFIG_COMMIT)
3249	return -ENXIO;
3250
3251	struct cxl_pmem_region *cxlr_pmem __free(kfree) =
3252	kzalloc(struct_size(cxlr_pmem, mapping, p->nr_targets), GFP_KERNEL);
3253	if (!cxlr_pmem)
3254	return -ENOMEM;
3255
3256	cxlr_pmem->hpa_range.start = p->res->start;
3257	cxlr_pmem->hpa_range.end = p->res->end;
3258
3259	/* Snapshot the region configuration underneath the cxl_rwsem.region */
3260	cxlr_pmem->nr_mappings = p->nr_targets;
3261	for (i = 0; i < p->nr_targets; i++) {
3262	struct cxl_endpoint_decoder *cxled = p->targets[i];
3263	struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
3264	struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];
3265
3266	/*
3267	* Regions never span CXL root devices, so by definition the
3268	* bridge for one device is the same for all.
3269	*/
3270	if (i == 0) {
3271	cxl_nvb = cxl_find_nvdimm_bridge(port: cxlmd->endpoint);
3272	if (!cxl_nvb)
3273	return -ENODEV;
3274	cxlr->cxl_nvb = cxl_nvb;
3275	}
3276	m->cxlmd = cxlmd;
3277	get_device(dev: &cxlmd->dev);
3278	m->start = cxled->dpa_res->start;
3279	m->size = resource_size(res: cxled->dpa_res);
3280	m->position = i;
3281	}
3282
3283	dev = &cxlr_pmem->dev;
3284	device_initialize(dev);
3285	lockdep_set_class(&dev->mutex, &cxl_pmem_region_key);
3286	device_set_pm_not_required(dev);
3287	dev->parent = &cxlr->dev;
3288	dev->bus = &cxl_bus_type;
3289	dev->type = &cxl_pmem_region_type;
3290	cxlr_pmem->cxlr = cxlr;
3291	cxlr->cxlr_pmem = no_free_ptr(cxlr_pmem);
3292
3293	return 0;
3294	}
3295
3296	static void cxl_dax_region_release(struct device *dev)
3297	{
3298	struct cxl_dax_region *cxlr_dax = to_cxl_dax_region(dev);
3299
3300	kfree(objp: cxlr_dax);
3301	}
3302
3303	static const struct attribute_group *cxl_dax_region_attribute_groups[] = {
3304	&cxl_base_attribute_group,
3305	NULL,
3306	};
3307
3308	const struct device_type cxl_dax_region_type = {
3309	.name = "cxl_dax_region",
3310	.release = cxl_dax_region_release,
3311	.groups = cxl_dax_region_attribute_groups,
3312	};
3313
3314	static bool is_cxl_dax_region(struct device *dev)
3315	{
3316	return dev->type == &cxl_dax_region_type;
3317	}
3318
3319	struct cxl_dax_region *to_cxl_dax_region(struct device *dev)
3320	{
3321	if (dev_WARN_ONCE(dev, !is_cxl_dax_region(dev),
3322	"not a cxl_dax_region device\n"))
3323	return NULL;
3324	return container_of(dev, struct cxl_dax_region, dev);
3325	}
3326	EXPORT_SYMBOL_NS_GPL(to_cxl_dax_region, "CXL");
3327
3328	static struct lock_class_key cxl_dax_region_key;
3329
3330	static struct cxl_dax_region *cxl_dax_region_alloc(struct cxl_region *cxlr)
3331	{
3332	struct cxl_region_params *p = &cxlr->params;
3333	struct cxl_dax_region *cxlr_dax;
3334	struct device *dev;
3335
3336	guard(rwsem_read)(T: &cxl_rwsem.region);
3337	if (p->state != CXL_CONFIG_COMMIT)
3338	return ERR_PTR(error: -ENXIO);
3339
3340	cxlr_dax = kzalloc(sizeof(*cxlr_dax), GFP_KERNEL);
3341	if (!cxlr_dax)
3342	return ERR_PTR(error: -ENOMEM);
3343
3344	cxlr_dax->hpa_range.start = p->res->start;
3345	cxlr_dax->hpa_range.end = p->res->end;
3346
3347	dev = &cxlr_dax->dev;
3348	cxlr_dax->cxlr = cxlr;
3349	device_initialize(dev);
3350	lockdep_set_class(&dev->mutex, &cxl_dax_region_key);
3351	device_set_pm_not_required(dev);
3352	dev->parent = &cxlr->dev;
3353	dev->bus = &cxl_bus_type;
3354	dev->type = &cxl_dax_region_type;
3355
3356	return cxlr_dax;
3357	}
3358
3359	static void cxlr_pmem_unregister(void *_cxlr_pmem)
3360	{
3361	struct cxl_pmem_region *cxlr_pmem = _cxlr_pmem;
3362	struct cxl_region *cxlr = cxlr_pmem->cxlr;
3363	struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
3364
3365	/*
3366	* Either the bridge is in ->remove() context under the device_lock(),
3367	* or cxlr_release_nvdimm() is cancelling the bridge's release action
3368	* for @cxlr_pmem and doing it itself (while manually holding the bridge
3369	* lock).
3370	*/
3371	device_lock_assert(dev: &cxl_nvb->dev);
3372	cxlr->cxlr_pmem = NULL;
3373	cxlr_pmem->cxlr = NULL;
3374	device_unregister(dev: &cxlr_pmem->dev);
3375	}
3376
3377	static void cxlr_release_nvdimm(void *_cxlr)
3378	{
3379	struct cxl_region *cxlr = _cxlr;
3380	struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
3381
3382	scoped_guard(device, &cxl_nvb->dev) {
3383	if (cxlr->cxlr_pmem)
3384	devm_release_action(dev: &cxl_nvb->dev, action: cxlr_pmem_unregister,
3385	data: cxlr->cxlr_pmem);
3386	}
3387	cxlr->cxl_nvb = NULL;
3388	put_device(dev: &cxl_nvb->dev);
3389	}
3390
3391	/**
3392	* devm_cxl_add_pmem_region() - add a cxl_region-to-nd_region bridge
3393	* @cxlr: parent CXL region for this pmem region bridge device
3394	*
3395	* Return: 0 on success negative error code on failure.
3396	*/
3397	static int devm_cxl_add_pmem_region(struct cxl_region *cxlr)
3398	{
3399	struct cxl_pmem_region *cxlr_pmem;
3400	struct cxl_nvdimm_bridge *cxl_nvb;
3401	struct device *dev;
3402	int rc;
3403
3404	rc = cxl_pmem_region_alloc(cxlr);
3405	if (rc)
3406	return rc;
3407	cxlr_pmem = cxlr->cxlr_pmem;
3408	cxl_nvb = cxlr->cxl_nvb;
3409
3410	dev = &cxlr_pmem->dev;
3411	rc = dev_set_name(dev, name: "pmem_region%d", cxlr->id);
3412	if (rc)
3413	goto err;
3414
3415	rc = device_add(dev);
3416	if (rc)
3417	goto err;
3418
3419	dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
3420	dev_name(dev));
3421
3422	scoped_guard(device, &cxl_nvb->dev) {
3423	if (cxl_nvb->dev.driver)
3424	rc = devm_add_action_or_reset(&cxl_nvb->dev,
3425	cxlr_pmem_unregister,
3426	cxlr_pmem);
3427	else
3428	rc = -ENXIO;
3429	}
3430
3431	if (rc)
3432	goto err_bridge;
3433
3434	/* @cxlr carries a reference on @cxl_nvb until cxlr_release_nvdimm */
3435	return devm_add_action_or_reset(&cxlr->dev, cxlr_release_nvdimm, cxlr);
3436
3437	err:
3438	put_device(dev);
3439	err_bridge:
3440	put_device(dev: &cxl_nvb->dev);
3441	cxlr->cxl_nvb = NULL;
3442	return rc;
3443	}
3444
3445	static void cxlr_dax_unregister(void *_cxlr_dax)
3446	{
3447	struct cxl_dax_region *cxlr_dax = _cxlr_dax;
3448
3449	device_unregister(dev: &cxlr_dax->dev);
3450	}
3451
3452	static int devm_cxl_add_dax_region(struct cxl_region *cxlr)
3453	{
3454	struct cxl_dax_region *cxlr_dax;
3455	struct device *dev;
3456	int rc;
3457
3458	cxlr_dax = cxl_dax_region_alloc(cxlr);
3459	if (IS_ERR(ptr: cxlr_dax))
3460	return PTR_ERR(ptr: cxlr_dax);
3461
3462	dev = &cxlr_dax->dev;
3463	rc = dev_set_name(dev, name: "dax_region%d", cxlr->id);
3464	if (rc)
3465	goto err;
3466
3467	rc = device_add(dev);
3468	if (rc)
3469	goto err;
3470
3471	dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
3472	dev_name(dev));
3473
3474	return devm_add_action_or_reset(&cxlr->dev, cxlr_dax_unregister,
3475	cxlr_dax);
3476	err:
3477	put_device(dev);
3478	return rc;
3479	}
3480
3481	static int match_decoder_by_range(struct device *dev, const void *data)
3482	{
3483	const struct range *r1, *r2 = data;
3484	struct cxl_decoder *cxld;
3485
3486	if (!is_switch_decoder(dev))
3487	return 0;
3488
3489	cxld = to_cxl_decoder(dev);
3490	r1 = &cxld->hpa_range;
3491	return range_contains(r1, r2);
3492	}
3493
3494	static struct cxl_decoder *
3495	cxl_port_find_switch_decoder(struct cxl_port *port, struct range *hpa)
3496	{
3497	struct device *cxld_dev = device_find_child(parent: &port->dev, data: hpa,
3498	match: match_decoder_by_range);
3499
3500	return cxld_dev ? to_cxl_decoder(dev: cxld_dev) : NULL;
3501	}
3502
3503	static struct cxl_root_decoder *
3504	cxl_find_root_decoder(struct cxl_endpoint_decoder *cxled)
3505	{
3506	struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
3507	struct cxl_port *port = cxled_to_port(cxled);
3508	struct cxl_root *cxl_root __free(put_cxl_root) = find_cxl_root(port);
3509	struct cxl_decoder *root, *cxld = &cxled->cxld;
3510	struct range *hpa = &cxld->hpa_range;
3511
3512	root = cxl_port_find_switch_decoder(port: &cxl_root->port, hpa);
3513	if (!root) {
3514	dev_err(cxlmd->dev.parent,
3515	"%s:%s no CXL window for range %#llx:%#llx\n",
3516	dev_name(&cxlmd->dev), dev_name(&cxld->dev),
3517	cxld->hpa_range.start, cxld->hpa_range.end);
3518	return NULL;
3519	}
3520
3521	return to_cxl_root_decoder(dev: &root->dev);
3522	}
3523
3524	static int match_region_by_range(struct device *dev, const void *data)
3525	{
3526	struct cxl_region_params *p;
3527	struct cxl_region *cxlr;
3528	const struct range *r = data;
3529
3530	if (!is_cxl_region(dev))
3531	return 0;
3532
3533	cxlr = to_cxl_region(dev);
3534	p = &cxlr->params;
3535
3536	guard(rwsem_read)(T: &cxl_rwsem.region);
3537	return spa_maps_hpa(p, range: r);
3538	}
3539
3540	static int cxl_extended_linear_cache_resize(struct cxl_region *cxlr,
3541	struct resource *res)
3542	{
3543	struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev: cxlr->dev.parent);
3544	struct cxl_region_params *p = &cxlr->params;
3545	resource_size_t size = resource_size(res);
3546	resource_size_t cache_size, start;
3547
3548	cache_size = cxlrd->cache_size;
3549	if (!cache_size)
3550	return 0;
3551
3552	if (size != cache_size) {
3553	dev_warn(&cxlr->dev,
3554	"Extended Linear Cache size %pa != CXL size %pa. No Support!",
3555	&cache_size, &size);
3556	return -ENXIO;
3557	}
3558
3559	/*
3560	* Move the start of the range to where the cache range starts. The
3561	* implementation assumes that the cache range is in front of the
3562	* CXL range. This is not dictated by the HMAT spec but is how the
3563	* current known implementation is configured.
3564	*
3565	* The cache range is expected to be within the CFMWS. The adjusted
3566	* res->start should not be less than cxlrd->res->start.
3567	*/
3568	start = res->start - cache_size;
3569	if (start < cxlrd->res->start)
3570	return -ENXIO;
3571
3572	res->start = start;
3573	p->cache_size = cache_size;
3574
3575	return 0;
3576	}
3577
3578	static int __construct_region(struct cxl_region *cxlr,
3579	struct cxl_root_decoder *cxlrd,
3580	struct cxl_endpoint_decoder *cxled)
3581	{
3582	struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
3583	struct range *hpa = &cxled->cxld.hpa_range;
3584	struct cxl_region_params *p;
3585	struct resource *res;
3586	int rc;
3587
3588	guard(rwsem_write)(T: &cxl_rwsem.region);
3589	p = &cxlr->params;
3590	if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
3591	dev_err(cxlmd->dev.parent,
3592	"%s:%s: %s autodiscovery interrupted\n",
3593	dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
3594	__func__);
3595	return -EBUSY;
3596	}
3597
3598	set_bit(CXL_REGION_F_AUTO, addr: &cxlr->flags);
3599
3600	res = kmalloc(sizeof(*res), GFP_KERNEL);
3601	if (!res)
3602	return -ENOMEM;
3603
3604	*res = DEFINE_RES_MEM_NAMED(hpa->start, range_len(hpa),
3605	dev_name(&cxlr->dev));
3606
3607	rc = cxl_extended_linear_cache_resize(cxlr, res);
3608	if (rc && rc != -EOPNOTSUPP) {
3609	/*
3610	* Failing to support extended linear cache region resize does not
3611	* prevent the region from functioning. Only causes cxl list showing
3612	* incorrect region size.
3613	*/
3614	dev_warn(cxlmd->dev.parent,
3615	"Extended linear cache calculation failed rc:%d\n", rc);
3616	}
3617
3618	rc = sysfs_update_group(kobj: &cxlr->dev.kobj, grp: &cxl_region_group);
3619	if (rc)
3620	return rc;
3621
3622	rc = insert_resource(parent: cxlrd->res, new: res);
3623	if (rc) {
3624	/*
3625	* Platform-firmware may not have split resources like "System
3626	* RAM" on CXL window boundaries see cxl_region_iomem_release()
3627	*/
3628	dev_warn(cxlmd->dev.parent,
3629	"%s:%s: %s %s cannot insert resource\n",
3630	dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
3631	__func__, dev_name(&cxlr->dev));
3632	}
3633
3634	p->res = res;
3635	p->interleave_ways = cxled->cxld.interleave_ways;
3636	p->interleave_granularity = cxled->cxld.interleave_granularity;
3637	p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
3638
3639	rc = sysfs_update_group(kobj: &cxlr->dev.kobj, grp: get_cxl_region_target_group());
3640	if (rc)
3641	return rc;
3642
3643	dev_dbg(cxlmd->dev.parent, "%s:%s: %s %s res: %pr iw: %d ig: %d\n",
3644	dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), __func__,
3645	dev_name(&cxlr->dev), p->res, p->interleave_ways,
3646	p->interleave_granularity);
3647
3648	/* ...to match put_device() in cxl_add_to_region() */
3649	get_device(dev: &cxlr->dev);
3650
3651	return 0;
3652	}
3653
3654	/* Establish an empty region covering the given HPA range */
3655	static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
3656	struct cxl_endpoint_decoder *cxled)
3657	{
3658	struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
3659	struct cxl_port *port = cxlrd_to_port(cxlrd);
3660	struct cxl_dev_state *cxlds = cxlmd->cxlds;
3661	int rc, part = READ_ONCE(cxled->part);
3662	struct cxl_region *cxlr;
3663
3664	do {
3665	cxlr = __create_region(cxlrd, mode: cxlds->part[part].mode,
3666	id: atomic_read(v: &cxlrd->region_id));
3667	} while (IS_ERR(ptr: cxlr) && PTR_ERR(ptr: cxlr) == -EBUSY);
3668
3669	if (IS_ERR(ptr: cxlr)) {
3670	dev_err(cxlmd->dev.parent,
3671	"%s:%s: %s failed assign region: %ld\n",
3672	dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
3673	__func__, PTR_ERR(cxlr));
3674	return cxlr;
3675	}
3676
3677	rc = __construct_region(cxlr, cxlrd, cxled);
3678	if (rc) {
3679	devm_release_action(dev: port->uport_dev, action: unregister_region, data: cxlr);
3680	return ERR_PTR(error: rc);
3681	}
3682
3683	return cxlr;
3684	}
3685
3686	static struct cxl_region *
3687	cxl_find_region_by_range(struct cxl_root_decoder *cxlrd, struct range *hpa)
3688	{
3689	struct device *region_dev;
3690
3691	region_dev = device_find_child(parent: &cxlrd->cxlsd.cxld.dev, data: hpa,
3692	match: match_region_by_range);
3693	if (!region_dev)
3694	return NULL;
3695
3696	return to_cxl_region(dev: region_dev);
3697	}
3698
3699	int cxl_add_to_region(struct cxl_endpoint_decoder *cxled)
3700	{
3701	struct range *hpa = &cxled->cxld.hpa_range;
3702	struct cxl_region_params *p;
3703	bool attach = false;
3704	int rc;
3705
3706	struct cxl_root_decoder *cxlrd __free(put_cxl_root_decoder) =
3707	cxl_find_root_decoder(cxled);
3708	if (!cxlrd)
3709	return -ENXIO;
3710
3711	/*
3712	* Ensure that if multiple threads race to construct_region() for @hpa
3713	* one does the construction and the others add to that.
3714	*/
3715	mutex_lock(&cxlrd->range_lock);
3716	struct cxl_region *cxlr __free(put_cxl_region) =
3717	cxl_find_region_by_range(cxlrd, hpa);
3718	if (!cxlr)
3719	cxlr = construct_region(cxlrd, cxled);
3720	mutex_unlock(lock: &cxlrd->range_lock);
3721
3722	rc = PTR_ERR_OR_ZERO(ptr: cxlr);
3723	if (rc)
3724	return rc;
3725
3726	attach_target(cxlr, cxled, pos: -1, TASK_UNINTERRUPTIBLE);
3727
3728	scoped_guard(rwsem_read, &cxl_rwsem.region) {
3729	p = &cxlr->params;
3730	attach = p->state == CXL_CONFIG_COMMIT;
3731	}
3732
3733	if (attach) {
3734	/*
3735	* If device_attach() fails the range may still be active via
3736	* the platform-firmware memory map, otherwise the driver for
3737	* regions is local to this file, so driver matching can't fail.
3738	*/
3739	if (device_attach(dev: &cxlr->dev) < 0)
3740	dev_err(&cxlr->dev, "failed to enable, range: %pr\n",
3741	p->res);
3742	}
3743
3744	return rc;
3745	}
3746	EXPORT_SYMBOL_NS_GPL(cxl_add_to_region, "CXL");
3747
3748	u64 cxl_port_get_spa_cache_alias(struct cxl_port *endpoint, u64 spa)
3749	{
3750	struct cxl_region_ref *iter;
3751	unsigned long index;
3752
3753	if (!endpoint)
3754	return ~0ULL;
3755
3756	guard(rwsem_write)(T: &cxl_rwsem.region);
3757
3758	xa_for_each(&endpoint->regions, index, iter) {
3759	struct cxl_region_params *p = &iter->region->params;
3760
3761	if (cxl_resource_contains_addr(res: p->res, addr: spa)) {
3762	if (!p->cache_size)
3763	return ~0ULL;
3764
3765	if (spa >= p->res->start + p->cache_size)
3766	return spa - p->cache_size;
3767
3768	return spa + p->cache_size;
3769	}
3770	}
3771
3772	return ~0ULL;
3773	}
3774	EXPORT_SYMBOL_NS_GPL(cxl_port_get_spa_cache_alias, "CXL");
3775
3776	static int is_system_ram(struct resource *res, void *arg)
3777	{
3778	struct cxl_region *cxlr = arg;
3779	struct cxl_region_params *p = &cxlr->params;
3780
3781	dev_dbg(&cxlr->dev, "%pr has System RAM: %pr\n", p->res, res);
3782	return 1;
3783	}
3784
3785	static void shutdown_notifiers(void *_cxlr)
3786	{
3787	struct cxl_region *cxlr = _cxlr;
3788
3789	unregister_node_notifier(nb: &cxlr->node_notifier);
3790	unregister_mt_adistance_algorithm(nb: &cxlr->adist_notifier);
3791	}
3792
3793	static void remove_debugfs(void *dentry)
3794	{
3795	debugfs_remove_recursive(dentry);
3796	}
3797
3798	static int validate_region_offset(struct cxl_region *cxlr, u64 offset)
3799	{
3800	struct cxl_region_params *p = &cxlr->params;
3801	resource_size_t region_size;
3802	u64 hpa;
3803
3804	if (offset < p->cache_size) {
3805	dev_err(&cxlr->dev,
3806	"Offset %#llx is within extended linear cache %pa\n",
3807	offset, &p->cache_size);
3808	return -EINVAL;
3809	}
3810
3811	region_size = resource_size(res: p->res);
3812	if (offset >= region_size) {
3813	dev_err(&cxlr->dev, "Offset %#llx exceeds region size %pa\n",
3814	offset, &region_size);
3815	return -EINVAL;
3816	}
3817
3818	hpa = p->res->start + offset;
3819	if (hpa < p->res->start || hpa > p->res->end) {
3820	dev_err(&cxlr->dev, "HPA %#llx not in region %pr\n", hpa,
3821	p->res);
3822	return -EINVAL;
3823	}
3824
3825	return 0;
3826	}
3827
3828	static int cxl_region_debugfs_poison_inject(void *data, u64 offset)
3829	{
3830	struct dpa_result result = { .dpa = ULLONG_MAX, .cxlmd = NULL };
3831	struct cxl_region *cxlr = data;
3832	int rc;
3833
3834	ACQUIRE(rwsem_read_intr, region_rwsem)(T: &cxl_rwsem.region);
3835	if ((rc = ACQUIRE_ERR(rwsem_read_intr, &region_rwsem)))
3836	return rc;
3837
3838	ACQUIRE(rwsem_read_intr, dpa_rwsem)(T: &cxl_rwsem.dpa);
3839	if ((rc = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem)))
3840	return rc;
3841
3842	if (validate_region_offset(cxlr, offset))
3843	return -EINVAL;
3844
3845	offset -= cxlr->params.cache_size;
3846	rc = region_offset_to_dpa_result(cxlr, offset, result: &result);
3847	if (rc || !result.cxlmd || result.dpa == ULLONG_MAX) {
3848	dev_dbg(&cxlr->dev,
3849	"Failed to resolve DPA for region offset %#llx rc %d\n",
3850	offset, rc);
3851
3852	return rc ? rc : -EINVAL;
3853	}
3854
3855	return cxl_inject_poison_locked(cxlmd: result.cxlmd, dpa: result.dpa);
3856	}
3857
3858	DEFINE_DEBUGFS_ATTRIBUTE(cxl_poison_inject_fops, NULL,
3859	cxl_region_debugfs_poison_inject, "%llx\n");
3860
3861	static int cxl_region_debugfs_poison_clear(void *data, u64 offset)
3862	{
3863	struct dpa_result result = { .dpa = ULLONG_MAX, .cxlmd = NULL };
3864	struct cxl_region *cxlr = data;
3865	int rc;
3866
3867	ACQUIRE(rwsem_read_intr, region_rwsem)(T: &cxl_rwsem.region);
3868	if ((rc = ACQUIRE_ERR(rwsem_read_intr, &region_rwsem)))
3869	return rc;
3870
3871	ACQUIRE(rwsem_read_intr, dpa_rwsem)(T: &cxl_rwsem.dpa);
3872	if ((rc = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem)))
3873	return rc;
3874
3875	if (validate_region_offset(cxlr, offset))
3876	return -EINVAL;
3877
3878	offset -= cxlr->params.cache_size;
3879	rc = region_offset_to_dpa_result(cxlr, offset, result: &result);
3880	if (rc || !result.cxlmd || result.dpa == ULLONG_MAX) {
3881	dev_dbg(&cxlr->dev,
3882	"Failed to resolve DPA for region offset %#llx rc %d\n",
3883	offset, rc);
3884
3885	return rc ? rc : -EINVAL;
3886	}
3887
3888	return cxl_clear_poison_locked(cxlmd: result.cxlmd, dpa: result.dpa);
3889	}
3890
3891	DEFINE_DEBUGFS_ATTRIBUTE(cxl_poison_clear_fops, NULL,
3892	cxl_region_debugfs_poison_clear, "%llx\n");
3893
3894	static int cxl_region_can_probe(struct cxl_region *cxlr)
3895	{
3896	struct cxl_region_params *p = &cxlr->params;
3897	int rc;
3898
3899	ACQUIRE(rwsem_read_intr, rwsem)(T: &cxl_rwsem.region);
3900	if ((rc = ACQUIRE_ERR(rwsem_read_intr, &rwsem))) {
3901	dev_dbg(&cxlr->dev, "probe interrupted\n");
3902	return rc;
3903	}
3904
3905	if (p->state < CXL_CONFIG_COMMIT) {
3906	dev_dbg(&cxlr->dev, "config state: %d\n", p->state);
3907	return -ENXIO;
3908	}
3909
3910	if (test_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags)) {
3911	dev_err(&cxlr->dev,
3912	"failed to activate, re-commit region and retry\n");
3913	return -ENXIO;
3914	}
3915
3916	return 0;
3917	}
3918
3919	static int cxl_region_probe(struct device *dev)
3920	{
3921	struct cxl_region *cxlr = to_cxl_region(dev);
3922	struct cxl_region_params *p = &cxlr->params;
3923	bool poison_supported = true;
3924	int rc;
3925
3926	rc = cxl_region_can_probe(cxlr);
3927	if (rc)
3928	return rc;
3929
3930	/*
3931	* From this point on any path that changes the region's state away from
3932	* CXL_CONFIG_COMMIT is also responsible for releasing the driver.
3933	*/
3934
3935	cxlr->node_notifier.notifier_call = cxl_region_perf_attrs_callback;
3936	cxlr->node_notifier.priority = CXL_CALLBACK_PRI;
3937	register_node_notifier(nb: &cxlr->node_notifier);
3938
3939	cxlr->adist_notifier.notifier_call = cxl_region_calculate_adistance;
3940	cxlr->adist_notifier.priority = 100;
3941	register_mt_adistance_algorithm(nb: &cxlr->adist_notifier);
3942
3943	rc = devm_add_action_or_reset(&cxlr->dev, shutdown_notifiers, cxlr);
3944	if (rc)
3945	return rc;
3946
3947	/* Create poison attributes if all memdevs support the capabilities */
3948	for (int i = 0; i < p->nr_targets; i++) {
3949	struct cxl_endpoint_decoder *cxled = p->targets[i];
3950	struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
3951
3952	if (!cxl_memdev_has_poison_cmd(cxlmd, cmd: CXL_POISON_ENABLED_INJECT) ||
3953	!cxl_memdev_has_poison_cmd(cxlmd, cmd: CXL_POISON_ENABLED_CLEAR)) {
3954	poison_supported = false;
3955	break;
3956	}
3957	}
3958
3959	if (poison_supported) {
3960	struct dentry *dentry;
3961
3962	dentry = cxl_debugfs_create_dir(dir: dev_name(dev));
3963	debugfs_create_file("inject_poison", 0200, dentry, cxlr,
3964	&cxl_poison_inject_fops);
3965	debugfs_create_file("clear_poison", 0200, dentry, cxlr,
3966	&cxl_poison_clear_fops);
3967	rc = devm_add_action_or_reset(dev, remove_debugfs, dentry);
3968	if (rc)
3969	return rc;
3970	}
3971
3972	switch (cxlr->mode) {
3973	case CXL_PARTMODE_PMEM:
3974	rc = devm_cxl_region_edac_register(cxlr);
3975	if (rc)
3976	dev_dbg(&cxlr->dev, "CXL EDAC registration for region_id=%d failed\n",
3977	cxlr->id);
3978
3979	return devm_cxl_add_pmem_region(cxlr);
3980	case CXL_PARTMODE_RAM:
3981	rc = devm_cxl_region_edac_register(cxlr);
3982	if (rc)
3983	dev_dbg(&cxlr->dev, "CXL EDAC registration for region_id=%d failed\n",
3984	cxlr->id);
3985
3986	/*
3987	* The region can not be manged by CXL if any portion of
3988	* it is already online as 'System RAM'
3989	*/
3990	if (walk_iomem_res_desc(desc: IORES_DESC_NONE,
3991	IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY,
3992	start: p->res->start, end: p->res->end, arg: cxlr,
3993	func: is_system_ram) > 0)
3994	return 0;
3995	return devm_cxl_add_dax_region(cxlr);
3996	default:
3997	dev_dbg(&cxlr->dev, "unsupported region mode: %d\n",
3998	cxlr->mode);
3999	return -ENXIO;
4000	}
4001	}
4002
4003	static struct cxl_driver cxl_region_driver = {
4004	.name = "cxl_region",
4005	.probe = cxl_region_probe,
4006	.id = CXL_DEVICE_REGION,
4007	};
4008
4009	int cxl_region_init(void)
4010	{
4011	return cxl_driver_register(&cxl_region_driver);
4012	}
4013
4014	void cxl_region_exit(void)
4015	{
4016	cxl_driver_unregister(cxl_drv: &cxl_region_driver);
4017	}
4018
4019	MODULE_IMPORT_NS("CXL");
4020	MODULE_IMPORT_NS("DEVMEM");
4021	MODULE_ALIAS_CXL(CXL_DEVICE_REGION);
4022