1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (c) 2019, Intel Corporation.
4 *
5 * Heterogeneous Memory Attributes Table (HMAT) representation
6 *
7 * This program parses and reports the platform's HMAT tables, and registers
8 * the applicable attributes with the node's interfaces.
9 */
10
11#define pr_fmt(fmt) "acpi/hmat: " fmt
12
13#include <linux/acpi.h>
14#include <linux/bitops.h>
15#include <linux/device.h>
16#include <linux/init.h>
17#include <linux/list.h>
18#include <linux/mm.h>
19#include <linux/platform_device.h>
20#include <linux/list_sort.h>
21#include <linux/memregion.h>
22#include <linux/memory.h>
23#include <linux/mutex.h>
24#include <linux/node.h>
25#include <linux/sysfs.h>
26#include <linux/dax.h>
27#include <linux/memory-tiers.h>
28
29static u8 hmat_revision;
30static int hmat_disable __initdata;
31
32void __init disable_hmat(void)
33{
34 hmat_disable = 1;
35}
36
37static LIST_HEAD(targets);
38static LIST_HEAD(initiators);
39static LIST_HEAD(localities);
40
41static DEFINE_MUTEX(target_lock);
42
43/*
44 * The defined enum order is used to prioritize attributes to break ties when
45 * selecting the best performing node.
46 */
47enum locality_types {
48 WRITE_LATENCY,
49 READ_LATENCY,
50 WRITE_BANDWIDTH,
51 READ_BANDWIDTH,
52};
53
54static struct memory_locality *localities_types[4];
55
56struct target_cache {
57 struct list_head node;
58 struct node_cache_attrs cache_attrs;
59};
60
61enum {
62 NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL = ACCESS_COORDINATE_MAX,
63 NODE_ACCESS_CLASS_GENPORT_SINK_CPU,
64 NODE_ACCESS_CLASS_MAX,
65};
66
67struct memory_target {
68 struct list_head node;
69 unsigned int memory_pxm;
70 unsigned int processor_pxm;
71 struct resource memregions;
72 struct access_coordinate coord[NODE_ACCESS_CLASS_MAX];
73 struct list_head caches;
74 struct node_cache_attrs cache_attrs;
75 u8 gen_port_device_handle[ACPI_SRAT_DEVICE_HANDLE_SIZE];
76 bool registered;
77};
78
79struct memory_initiator {
80 struct list_head node;
81 unsigned int processor_pxm;
82 bool has_cpu;
83};
84
85struct memory_locality {
86 struct list_head node;
87 struct acpi_hmat_locality *hmat_loc;
88};
89
90static struct memory_initiator *find_mem_initiator(unsigned int cpu_pxm)
91{
92 struct memory_initiator *initiator;
93
94 list_for_each_entry(initiator, &initiators, node)
95 if (initiator->processor_pxm == cpu_pxm)
96 return initiator;
97 return NULL;
98}
99
100static struct memory_target *find_mem_target(unsigned int mem_pxm)
101{
102 struct memory_target *target;
103
104 list_for_each_entry(target, &targets, node)
105 if (target->memory_pxm == mem_pxm)
106 return target;
107 return NULL;
108}
109
110/**
111 * hmat_get_extended_linear_cache_size - Retrieve the extended linear cache size
112 * @backing_res: resource from the backing media
113 * @nid: node id for the memory region
114 * @cache_size: (Output) size of extended linear cache.
115 *
116 * Return: 0 on success. Errno on failure.
117 *
118 */
119int hmat_get_extended_linear_cache_size(struct resource *backing_res, int nid,
120 resource_size_t *cache_size)
121{
122 unsigned int pxm = node_to_pxm(nid);
123 struct memory_target *target;
124 struct target_cache *tcache;
125 struct resource *res;
126
127 target = find_mem_target(mem_pxm: pxm);
128 if (!target)
129 return -ENOENT;
130
131 list_for_each_entry(tcache, &target->caches, node) {
132 if (tcache->cache_attrs.address_mode !=
133 NODE_CACHE_ADDR_MODE_EXTENDED_LINEAR)
134 continue;
135
136 res = &target->memregions;
137 if (!resource_contains(r1: res, r2: backing_res))
138 continue;
139
140 *cache_size = tcache->cache_attrs.size;
141 return 0;
142 }
143
144 *cache_size = 0;
145 return 0;
146}
147EXPORT_SYMBOL_NS_GPL(hmat_get_extended_linear_cache_size, "CXL");
148
149static struct memory_target *acpi_find_genport_target(u32 uid)
150{
151 struct memory_target *target;
152 u32 target_uid;
153 u8 *uid_ptr;
154
155 list_for_each_entry(target, &targets, node) {
156 uid_ptr = target->gen_port_device_handle + 8;
157 target_uid = *(u32 *)uid_ptr;
158 if (uid == target_uid)
159 return target;
160 }
161
162 return NULL;
163}
164
165/**
166 * acpi_get_genport_coordinates - Retrieve the access coordinates for a generic port
167 * @uid: ACPI unique id
168 * @coord: The access coordinates written back out for the generic port.
169 * Expect 2 levels array.
170 *
171 * Return: 0 on success. Errno on failure.
172 *
173 * Only supports device handles that are ACPI. Assume ACPI0016 HID for CXL.
174 */
175int acpi_get_genport_coordinates(u32 uid,
176 struct access_coordinate *coord)
177{
178 struct memory_target *target;
179
180 guard(mutex)(T: &target_lock);
181 target = acpi_find_genport_target(uid);
182 if (!target)
183 return -ENOENT;
184
185 coord[ACCESS_COORDINATE_LOCAL] =
186 target->coord[NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL];
187 coord[ACCESS_COORDINATE_CPU] =
188 target->coord[NODE_ACCESS_CLASS_GENPORT_SINK_CPU];
189
190 return 0;
191}
192EXPORT_SYMBOL_NS_GPL(acpi_get_genport_coordinates, "CXL");
193
194static __init void alloc_memory_initiator(unsigned int cpu_pxm)
195{
196 struct memory_initiator *initiator;
197
198 if (pxm_to_node(cpu_pxm) == NUMA_NO_NODE)
199 return;
200
201 initiator = find_mem_initiator(cpu_pxm);
202 if (initiator)
203 return;
204
205 initiator = kzalloc(sizeof(*initiator), GFP_KERNEL);
206 if (!initiator)
207 return;
208
209 initiator->processor_pxm = cpu_pxm;
210 initiator->has_cpu = node_state(node: pxm_to_node(cpu_pxm), state: N_CPU);
211 list_add_tail(new: &initiator->node, head: &initiators);
212}
213
214static __init struct memory_target *alloc_target(unsigned int mem_pxm)
215{
216 struct memory_target *target;
217
218 target = find_mem_target(mem_pxm);
219 if (!target) {
220 target = kzalloc(sizeof(*target), GFP_KERNEL);
221 if (!target)
222 return NULL;
223 target->memory_pxm = mem_pxm;
224 target->processor_pxm = PXM_INVAL;
225 target->memregions = (struct resource) {
226 .name = "ACPI mem",
227 .start = 0,
228 .end = -1,
229 .flags = IORESOURCE_MEM,
230 };
231 list_add_tail(new: &target->node, head: &targets);
232 INIT_LIST_HEAD(list: &target->caches);
233 }
234
235 return target;
236}
237
238static __init void alloc_memory_target(unsigned int mem_pxm,
239 resource_size_t start,
240 resource_size_t len)
241{
242 struct memory_target *target;
243
244 target = alloc_target(mem_pxm);
245 if (!target)
246 return;
247
248 /*
249 * There are potentially multiple ranges per PXM, so record each
250 * in the per-target memregions resource tree.
251 */
252 if (!__request_region(&target->memregions, start, n: len, name: "memory target",
253 IORESOURCE_MEM))
254 pr_warn("failed to reserve %#llx - %#llx in pxm: %d\n",
255 start, start + len, mem_pxm);
256}
257
258static __init void alloc_genport_target(unsigned int mem_pxm, u8 *handle)
259{
260 struct memory_target *target;
261
262 target = alloc_target(mem_pxm);
263 if (!target)
264 return;
265
266 memcpy(target->gen_port_device_handle, handle,
267 ACPI_SRAT_DEVICE_HANDLE_SIZE);
268}
269
270static __init const char *hmat_data_type(u8 type)
271{
272 switch (type) {
273 case ACPI_HMAT_ACCESS_LATENCY:
274 return "Access Latency";
275 case ACPI_HMAT_READ_LATENCY:
276 return "Read Latency";
277 case ACPI_HMAT_WRITE_LATENCY:
278 return "Write Latency";
279 case ACPI_HMAT_ACCESS_BANDWIDTH:
280 return "Access Bandwidth";
281 case ACPI_HMAT_READ_BANDWIDTH:
282 return "Read Bandwidth";
283 case ACPI_HMAT_WRITE_BANDWIDTH:
284 return "Write Bandwidth";
285 default:
286 return "Reserved";
287 }
288}
289
290static __init const char *hmat_data_type_suffix(u8 type)
291{
292 switch (type) {
293 case ACPI_HMAT_ACCESS_LATENCY:
294 case ACPI_HMAT_READ_LATENCY:
295 case ACPI_HMAT_WRITE_LATENCY:
296 return " nsec";
297 case ACPI_HMAT_ACCESS_BANDWIDTH:
298 case ACPI_HMAT_READ_BANDWIDTH:
299 case ACPI_HMAT_WRITE_BANDWIDTH:
300 return " MB/s";
301 default:
302 return "";
303 }
304}
305
306static u32 hmat_normalize(u16 entry, u64 base, u8 type)
307{
308 u32 value;
309
310 /*
311 * Check for invalid and overflow values
312 */
313 if (entry == 0xffff || !entry)
314 return 0;
315 else if (base > (UINT_MAX / (entry)))
316 return 0;
317
318 /*
319 * Divide by the base unit for version 1, convert latency from
320 * picosenonds to nanoseconds if revision 2.
321 */
322 value = entry * base;
323 if (hmat_revision == 1) {
324 if (value < 10)
325 return 0;
326 value = DIV_ROUND_UP(value, 10);
327 } else if (hmat_revision == 2) {
328 switch (type) {
329 case ACPI_HMAT_ACCESS_LATENCY:
330 case ACPI_HMAT_READ_LATENCY:
331 case ACPI_HMAT_WRITE_LATENCY:
332 value = DIV_ROUND_UP(value, 1000);
333 break;
334 default:
335 break;
336 }
337 }
338 return value;
339}
340
341static void hmat_update_target_access(struct memory_target *target,
342 u8 type, u32 value, int access)
343{
344 switch (type) {
345 case ACPI_HMAT_ACCESS_LATENCY:
346 target->coord[access].read_latency = value;
347 target->coord[access].write_latency = value;
348 break;
349 case ACPI_HMAT_READ_LATENCY:
350 target->coord[access].read_latency = value;
351 break;
352 case ACPI_HMAT_WRITE_LATENCY:
353 target->coord[access].write_latency = value;
354 break;
355 case ACPI_HMAT_ACCESS_BANDWIDTH:
356 target->coord[access].read_bandwidth = value;
357 target->coord[access].write_bandwidth = value;
358 break;
359 case ACPI_HMAT_READ_BANDWIDTH:
360 target->coord[access].read_bandwidth = value;
361 break;
362 case ACPI_HMAT_WRITE_BANDWIDTH:
363 target->coord[access].write_bandwidth = value;
364 break;
365 default:
366 break;
367 }
368}
369
370static __init void hmat_add_locality(struct acpi_hmat_locality *hmat_loc)
371{
372 struct memory_locality *loc;
373
374 loc = kzalloc(sizeof(*loc), GFP_KERNEL);
375 if (!loc) {
376 pr_notice_once("Failed to allocate HMAT locality\n");
377 return;
378 }
379
380 loc->hmat_loc = hmat_loc;
381 list_add_tail(new: &loc->node, head: &localities);
382
383 switch (hmat_loc->data_type) {
384 case ACPI_HMAT_ACCESS_LATENCY:
385 localities_types[READ_LATENCY] = loc;
386 localities_types[WRITE_LATENCY] = loc;
387 break;
388 case ACPI_HMAT_READ_LATENCY:
389 localities_types[READ_LATENCY] = loc;
390 break;
391 case ACPI_HMAT_WRITE_LATENCY:
392 localities_types[WRITE_LATENCY] = loc;
393 break;
394 case ACPI_HMAT_ACCESS_BANDWIDTH:
395 localities_types[READ_BANDWIDTH] = loc;
396 localities_types[WRITE_BANDWIDTH] = loc;
397 break;
398 case ACPI_HMAT_READ_BANDWIDTH:
399 localities_types[READ_BANDWIDTH] = loc;
400 break;
401 case ACPI_HMAT_WRITE_BANDWIDTH:
402 localities_types[WRITE_BANDWIDTH] = loc;
403 break;
404 default:
405 break;
406 }
407}
408
409static __init void hmat_update_target(unsigned int tgt_pxm, unsigned int init_pxm,
410 u8 mem_hier, u8 type, u32 value)
411{
412 struct memory_target *target = find_mem_target(mem_pxm: tgt_pxm);
413
414 if (mem_hier != ACPI_HMAT_MEMORY)
415 return;
416
417 if (target && target->processor_pxm == init_pxm) {
418 hmat_update_target_access(target, type, value,
419 access: ACCESS_COORDINATE_LOCAL);
420 /* If the node has a CPU, update access ACCESS_COORDINATE_CPU */
421 if (node_state(node: pxm_to_node(init_pxm), state: N_CPU))
422 hmat_update_target_access(target, type, value,
423 access: ACCESS_COORDINATE_CPU);
424 }
425}
426
427static __init int hmat_parse_locality(union acpi_subtable_headers *header,
428 const unsigned long end)
429{
430 struct acpi_hmat_locality *hmat_loc = (void *)header;
431 unsigned int init, targ, total_size, ipds, tpds;
432 u32 *inits, *targs, value;
433 u16 *entries;
434 u8 type, mem_hier;
435
436 if (hmat_loc->header.length < sizeof(*hmat_loc)) {
437 pr_notice("Unexpected locality header length: %u\n",
438 hmat_loc->header.length);
439 return -EINVAL;
440 }
441
442 type = hmat_loc->data_type;
443 mem_hier = hmat_loc->flags & ACPI_HMAT_MEMORY_HIERARCHY;
444 ipds = hmat_loc->number_of_initiator_Pds;
445 tpds = hmat_loc->number_of_target_Pds;
446 total_size = sizeof(*hmat_loc) + sizeof(*entries) * ipds * tpds +
447 sizeof(*inits) * ipds + sizeof(*targs) * tpds;
448 if (hmat_loc->header.length < total_size) {
449 pr_notice("Unexpected locality header length:%u, minimum required:%u\n",
450 hmat_loc->header.length, total_size);
451 return -EINVAL;
452 }
453
454 pr_debug("Locality: Flags:%02x Type:%s Initiator Domains:%u Target Domains:%u Base:%lld\n",
455 hmat_loc->flags, hmat_data_type(type), ipds, tpds,
456 hmat_loc->entry_base_unit);
457
458 inits = (u32 *)(hmat_loc + 1);
459 targs = inits + ipds;
460 entries = (u16 *)(targs + tpds);
461 for (init = 0; init < ipds; init++) {
462 alloc_memory_initiator(cpu_pxm: inits[init]);
463 for (targ = 0; targ < tpds; targ++) {
464 value = hmat_normalize(entry: entries[init * tpds + targ],
465 base: hmat_loc->entry_base_unit,
466 type);
467 pr_debug(" Initiator-Target[%u-%u]:%u%s\n",
468 inits[init], targs[targ], value,
469 hmat_data_type_suffix(type));
470
471 hmat_update_target(tgt_pxm: targs[targ], init_pxm: inits[init],
472 mem_hier, type, value);
473 }
474 }
475
476 if (mem_hier == ACPI_HMAT_MEMORY)
477 hmat_add_locality(hmat_loc);
478
479 return 0;
480}
481
482static __init int hmat_parse_cache(union acpi_subtable_headers *header,
483 const unsigned long end)
484{
485 struct acpi_hmat_cache *cache = (void *)header;
486 struct memory_target *target;
487 struct target_cache *tcache;
488 u32 attrs;
489
490 if (cache->header.length < sizeof(*cache)) {
491 pr_notice("Unexpected cache header length: %u\n",
492 cache->header.length);
493 return -EINVAL;
494 }
495
496 attrs = cache->cache_attributes;
497 pr_debug("Cache: Domain:%u Size:%llu Attrs:%08x SMBIOS Handles:%d\n",
498 cache->memory_PD, cache->cache_size, attrs,
499 cache->number_of_SMBIOShandles);
500
501 target = find_mem_target(mem_pxm: cache->memory_PD);
502 if (!target)
503 return 0;
504
505 tcache = kzalloc(sizeof(*tcache), GFP_KERNEL);
506 if (!tcache) {
507 pr_notice_once("Failed to allocate HMAT cache info\n");
508 return 0;
509 }
510
511 tcache->cache_attrs.size = cache->cache_size;
512 tcache->cache_attrs.level = (attrs & ACPI_HMAT_CACHE_LEVEL) >> 4;
513 tcache->cache_attrs.line_size = (attrs & ACPI_HMAT_CACHE_LINE_SIZE) >> 16;
514
515 switch ((attrs & ACPI_HMAT_CACHE_ASSOCIATIVITY) >> 8) {
516 case ACPI_HMAT_CA_DIRECT_MAPPED:
517 tcache->cache_attrs.indexing = NODE_CACHE_DIRECT_MAP;
518 /* Extended Linear mode is only valid if cache is direct mapped */
519 if (cache->address_mode == ACPI_HMAT_CACHE_MODE_EXTENDED_LINEAR) {
520 tcache->cache_attrs.address_mode =
521 NODE_CACHE_ADDR_MODE_EXTENDED_LINEAR;
522 }
523 break;
524 case ACPI_HMAT_CA_COMPLEX_CACHE_INDEXING:
525 tcache->cache_attrs.indexing = NODE_CACHE_INDEXED;
526 break;
527 case ACPI_HMAT_CA_NONE:
528 default:
529 tcache->cache_attrs.indexing = NODE_CACHE_OTHER;
530 break;
531 }
532
533 switch ((attrs & ACPI_HMAT_WRITE_POLICY) >> 12) {
534 case ACPI_HMAT_CP_WB:
535 tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_BACK;
536 break;
537 case ACPI_HMAT_CP_WT:
538 tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_THROUGH;
539 break;
540 case ACPI_HMAT_CP_NONE:
541 default:
542 tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_OTHER;
543 break;
544 }
545 list_add_tail(new: &tcache->node, head: &target->caches);
546
547 return 0;
548}
549
550static int __init hmat_parse_proximity_domain(union acpi_subtable_headers *header,
551 const unsigned long end)
552{
553 struct acpi_hmat_proximity_domain *p = (void *)header;
554 struct memory_target *target = NULL;
555
556 if (p->header.length != sizeof(*p)) {
557 pr_notice("Unexpected address range header length: %u\n",
558 p->header.length);
559 return -EINVAL;
560 }
561
562 if (hmat_revision == 1)
563 pr_debug("Memory (%#llx length %#llx) Flags:%04x Processor Domain:%u Memory Domain:%u\n",
564 p->reserved3, p->reserved4, p->flags, p->processor_PD,
565 p->memory_PD);
566 else
567 pr_info("Memory Flags:%04x Processor Domain:%u Memory Domain:%u\n",
568 p->flags, p->processor_PD, p->memory_PD);
569
570 if ((hmat_revision == 1 && p->flags & ACPI_HMAT_MEMORY_PD_VALID) ||
571 hmat_revision > 1) {
572 target = find_mem_target(mem_pxm: p->memory_PD);
573 if (!target) {
574 pr_debug("Memory Domain missing from SRAT\n");
575 return -EINVAL;
576 }
577 }
578 if (target && p->flags & ACPI_HMAT_PROCESSOR_PD_VALID) {
579 int p_node = pxm_to_node(p->processor_PD);
580
581 if (p_node == NUMA_NO_NODE) {
582 pr_debug("Invalid Processor Domain\n");
583 return -EINVAL;
584 }
585 target->processor_pxm = p->processor_PD;
586 }
587
588 return 0;
589}
590
591static int __init hmat_parse_subtable(union acpi_subtable_headers *header,
592 const unsigned long end)
593{
594 struct acpi_hmat_structure *hdr = (void *)header;
595
596 if (!hdr)
597 return -EINVAL;
598
599 switch (hdr->type) {
600 case ACPI_HMAT_TYPE_PROXIMITY:
601 return hmat_parse_proximity_domain(header, end);
602 case ACPI_HMAT_TYPE_LOCALITY:
603 return hmat_parse_locality(header, end);
604 case ACPI_HMAT_TYPE_CACHE:
605 return hmat_parse_cache(header, end);
606 default:
607 return -EINVAL;
608 }
609}
610
611static __init int srat_parse_mem_affinity(union acpi_subtable_headers *header,
612 const unsigned long end)
613{
614 struct acpi_srat_mem_affinity *ma = (void *)header;
615
616 if (!ma)
617 return -EINVAL;
618 if (!(ma->flags & ACPI_SRAT_MEM_ENABLED))
619 return 0;
620 alloc_memory_target(mem_pxm: ma->proximity_domain, start: ma->base_address, len: ma->length);
621 return 0;
622}
623
624static __init int srat_parse_genport_affinity(union acpi_subtable_headers *header,
625 const unsigned long end)
626{
627 struct acpi_srat_generic_affinity *ga = (void *)header;
628
629 if (!ga)
630 return -EINVAL;
631
632 if (!(ga->flags & ACPI_SRAT_GENERIC_AFFINITY_ENABLED))
633 return 0;
634
635 /* Skip PCI device_handle for now */
636 if (ga->device_handle_type != 0)
637 return 0;
638
639 alloc_genport_target(mem_pxm: ga->proximity_domain,
640 handle: (u8 *)ga->device_handle);
641
642 return 0;
643}
644
645static u32 hmat_initiator_perf(struct memory_target *target,
646 struct memory_initiator *initiator,
647 struct acpi_hmat_locality *hmat_loc)
648{
649 unsigned int ipds, tpds, i, idx = 0, tdx = 0;
650 u32 *inits, *targs;
651 u16 *entries;
652
653 ipds = hmat_loc->number_of_initiator_Pds;
654 tpds = hmat_loc->number_of_target_Pds;
655 inits = (u32 *)(hmat_loc + 1);
656 targs = inits + ipds;
657 entries = (u16 *)(targs + tpds);
658
659 for (i = 0; i < ipds; i++) {
660 if (inits[i] == initiator->processor_pxm) {
661 idx = i;
662 break;
663 }
664 }
665
666 if (i == ipds)
667 return 0;
668
669 for (i = 0; i < tpds; i++) {
670 if (targs[i] == target->memory_pxm) {
671 tdx = i;
672 break;
673 }
674 }
675 if (i == tpds)
676 return 0;
677
678 return hmat_normalize(entry: entries[idx * tpds + tdx],
679 base: hmat_loc->entry_base_unit,
680 type: hmat_loc->data_type);
681}
682
683static bool hmat_update_best(u8 type, u32 value, u32 *best)
684{
685 bool updated = false;
686
687 if (!value)
688 return false;
689
690 switch (type) {
691 case ACPI_HMAT_ACCESS_LATENCY:
692 case ACPI_HMAT_READ_LATENCY:
693 case ACPI_HMAT_WRITE_LATENCY:
694 if (!*best || *best > value) {
695 *best = value;
696 updated = true;
697 }
698 break;
699 case ACPI_HMAT_ACCESS_BANDWIDTH:
700 case ACPI_HMAT_READ_BANDWIDTH:
701 case ACPI_HMAT_WRITE_BANDWIDTH:
702 if (!*best || *best < value) {
703 *best = value;
704 updated = true;
705 }
706 break;
707 }
708
709 return updated;
710}
711
712static int initiator_cmp(void *priv, const struct list_head *a,
713 const struct list_head *b)
714{
715 struct memory_initiator *ia;
716 struct memory_initiator *ib;
717
718 ia = list_entry(a, struct memory_initiator, node);
719 ib = list_entry(b, struct memory_initiator, node);
720
721 return ia->processor_pxm - ib->processor_pxm;
722}
723
724static int initiators_to_nodemask(unsigned long *p_nodes)
725{
726 struct memory_initiator *initiator;
727
728 if (list_empty(head: &initiators))
729 return -ENXIO;
730
731 list_for_each_entry(initiator, &initiators, node)
732 set_bit(nr: initiator->processor_pxm, addr: p_nodes);
733
734 return 0;
735}
736
737static void hmat_update_target_attrs(struct memory_target *target,
738 unsigned long *p_nodes, int access)
739{
740 struct memory_initiator *initiator;
741 unsigned int cpu_nid;
742 struct memory_locality *loc = NULL;
743 u32 best = 0;
744 int i;
745
746 /* Don't update for generic port if there's no device handle */
747 if ((access == NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL ||
748 access == NODE_ACCESS_CLASS_GENPORT_SINK_CPU) &&
749 !(*(u16 *)target->gen_port_device_handle))
750 return;
751
752 bitmap_zero(dst: p_nodes, MAX_NUMNODES);
753 /*
754 * If the Address Range Structure provides a local processor pxm, set
755 * only that one. Otherwise, find the best performance attributes and
756 * collect all initiators that match.
757 */
758 if (target->processor_pxm != PXM_INVAL) {
759 cpu_nid = pxm_to_node(target->processor_pxm);
760 if (access == ACCESS_COORDINATE_LOCAL ||
761 node_state(node: cpu_nid, state: N_CPU)) {
762 set_bit(nr: target->processor_pxm, addr: p_nodes);
763 return;
764 }
765 }
766
767 if (list_empty(head: &localities))
768 return;
769
770 /*
771 * We need the initiator list sorted so we can use bitmap_clear for
772 * previously set initiators when we find a better memory accessor.
773 * We'll also use the sorting to prime the candidate nodes with known
774 * initiators.
775 */
776 list_sort(NULL, head: &initiators, cmp: initiator_cmp);
777 if (initiators_to_nodemask(p_nodes) < 0)
778 return;
779
780 for (i = WRITE_LATENCY; i <= READ_BANDWIDTH; i++) {
781 loc = localities_types[i];
782 if (!loc)
783 continue;
784
785 best = 0;
786 list_for_each_entry(initiator, &initiators, node) {
787 u32 value;
788
789 if ((access == ACCESS_COORDINATE_CPU ||
790 access == NODE_ACCESS_CLASS_GENPORT_SINK_CPU) &&
791 !initiator->has_cpu) {
792 clear_bit(nr: initiator->processor_pxm, addr: p_nodes);
793 continue;
794 }
795 if (!test_bit(initiator->processor_pxm, p_nodes))
796 continue;
797
798 value = hmat_initiator_perf(target, initiator, hmat_loc: loc->hmat_loc);
799 if (hmat_update_best(type: loc->hmat_loc->data_type, value, best: &best))
800 bitmap_clear(map: p_nodes, start: 0, nbits: initiator->processor_pxm);
801 if (value != best)
802 clear_bit(nr: initiator->processor_pxm, addr: p_nodes);
803 }
804 if (best)
805 hmat_update_target_access(target, type: loc->hmat_loc->data_type, value: best, access);
806 }
807}
808
809static void __hmat_register_target_initiators(struct memory_target *target,
810 unsigned long *p_nodes,
811 int access)
812{
813 unsigned int mem_nid, cpu_nid;
814 int i;
815
816 mem_nid = pxm_to_node(target->memory_pxm);
817 hmat_update_target_attrs(target, p_nodes, access);
818 for_each_set_bit(i, p_nodes, MAX_NUMNODES) {
819 cpu_nid = pxm_to_node(i);
820 register_memory_node_under_compute_node(mem_nid, cpu_nid, access);
821 }
822}
823
824static void hmat_update_generic_target(struct memory_target *target)
825{
826 static DECLARE_BITMAP(p_nodes, MAX_NUMNODES);
827
828 hmat_update_target_attrs(target, p_nodes,
829 access: NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL);
830 hmat_update_target_attrs(target, p_nodes,
831 access: NODE_ACCESS_CLASS_GENPORT_SINK_CPU);
832}
833
834static void hmat_register_target_initiators(struct memory_target *target)
835{
836 static DECLARE_BITMAP(p_nodes, MAX_NUMNODES);
837
838 __hmat_register_target_initiators(target, p_nodes,
839 access: ACCESS_COORDINATE_LOCAL);
840 __hmat_register_target_initiators(target, p_nodes,
841 access: ACCESS_COORDINATE_CPU);
842}
843
844static void hmat_register_target_cache(struct memory_target *target)
845{
846 unsigned mem_nid = pxm_to_node(target->memory_pxm);
847 struct target_cache *tcache;
848
849 list_for_each_entry(tcache, &target->caches, node)
850 node_add_cache(nid: mem_nid, cache_attrs: &tcache->cache_attrs);
851}
852
853static void hmat_register_target_perf(struct memory_target *target, int access)
854{
855 unsigned mem_nid = pxm_to_node(target->memory_pxm);
856 node_set_perf_attrs(nid: mem_nid, coord: &target->coord[access], access);
857}
858
859static void hmat_register_target_devices(struct memory_target *target)
860{
861 struct resource *res;
862
863 /*
864 * Do not bother creating devices if no driver is available to
865 * consume them.
866 */
867 if (!IS_ENABLED(CONFIG_DEV_DAX_HMEM))
868 return;
869
870 for (res = target->memregions.child; res; res = res->sibling) {
871 int target_nid = pxm_to_node(target->memory_pxm);
872
873 hmem_register_resource(target_nid, r: res);
874 }
875}
876
877static void hmat_hotplug_target(struct memory_target *target)
878{
879 int nid = pxm_to_node(target->memory_pxm);
880
881 /*
882 * Skip offline nodes. This can happen when memory marked EFI_MEMORY_SP,
883 * "specific purpose", is applied to all the memory in a proximity
884 * domain leading to * the node being marked offline / unplugged, or if
885 * memory-only "hotplug" node is offline.
886 */
887 if (nid == NUMA_NO_NODE || !node_online(nid))
888 return;
889
890 guard(mutex)(T: &target_lock);
891 if (target->registered)
892 return;
893
894 hmat_register_target_initiators(target);
895 hmat_register_target_cache(target);
896 hmat_register_target_perf(target, access: ACCESS_COORDINATE_LOCAL);
897 hmat_register_target_perf(target, access: ACCESS_COORDINATE_CPU);
898 target->registered = true;
899}
900
901static void hmat_register_target(struct memory_target *target)
902{
903 /*
904 * Devices may belong to either an offline or online
905 * node, so unconditionally add them.
906 */
907 hmat_register_target_devices(target);
908
909 /*
910 * Register generic port perf numbers. The nid may not be
911 * initialized and is still NUMA_NO_NODE.
912 */
913 scoped_guard(mutex, &target_lock) {
914 if (*(u16 *)target->gen_port_device_handle) {
915 hmat_update_generic_target(target);
916 target->registered = true;
917 return;
918 }
919 }
920
921 hmat_hotplug_target(target);
922}
923
924static void hmat_register_targets(void)
925{
926 struct memory_target *target;
927
928 list_for_each_entry(target, &targets, node)
929 hmat_register_target(target);
930}
931
932static int hmat_callback(struct notifier_block *self,
933 unsigned long action, void *arg)
934{
935 struct memory_target *target;
936 struct node_notify *nn = arg;
937 int pxm, nid = nn->nid;
938
939 if (action != NODE_ADDED_FIRST_MEMORY)
940 return NOTIFY_OK;
941
942 pxm = node_to_pxm(nid);
943 target = find_mem_target(mem_pxm: pxm);
944 if (!target)
945 return NOTIFY_OK;
946
947 hmat_hotplug_target(target);
948 return NOTIFY_OK;
949}
950
951static int __init hmat_set_default_dram_perf(void)
952{
953 int rc;
954 int nid, pxm;
955 struct memory_target *target;
956 struct access_coordinate *attrs;
957
958 for_each_node_mask(nid, default_dram_nodes) {
959 pxm = node_to_pxm(nid);
960 target = find_mem_target(mem_pxm: pxm);
961 if (!target)
962 continue;
963 attrs = &target->coord[ACCESS_COORDINATE_CPU];
964 rc = mt_set_default_dram_perf(nid, perf: attrs, source: "ACPI HMAT");
965 if (rc)
966 return rc;
967 }
968
969 return 0;
970}
971
972static int hmat_calculate_adistance(struct notifier_block *self,
973 unsigned long nid, void *data)
974{
975 static DECLARE_BITMAP(p_nodes, MAX_NUMNODES);
976 struct memory_target *target;
977 struct access_coordinate *perf;
978 int *adist = data;
979 int pxm;
980
981 pxm = node_to_pxm(nid);
982 target = find_mem_target(mem_pxm: pxm);
983 if (!target)
984 return NOTIFY_OK;
985
986 mutex_lock(&target_lock);
987 hmat_update_target_attrs(target, p_nodes, access: ACCESS_COORDINATE_CPU);
988 mutex_unlock(lock: &target_lock);
989
990 perf = &target->coord[ACCESS_COORDINATE_CPU];
991
992 if (mt_perf_to_adistance(perf, adist))
993 return NOTIFY_OK;
994
995 return NOTIFY_STOP;
996}
997
998static struct notifier_block hmat_adist_nb __meminitdata = {
999 .notifier_call = hmat_calculate_adistance,
1000 .priority = 100,
1001};
1002
1003static __init void hmat_free_structures(void)
1004{
1005 struct memory_target *target, *tnext;
1006 struct memory_locality *loc, *lnext;
1007 struct memory_initiator *initiator, *inext;
1008 struct target_cache *tcache, *cnext;
1009
1010 list_for_each_entry_safe(target, tnext, &targets, node) {
1011 struct resource *res, *res_next;
1012
1013 list_for_each_entry_safe(tcache, cnext, &target->caches, node) {
1014 list_del(entry: &tcache->node);
1015 kfree(objp: tcache);
1016 }
1017
1018 list_del(entry: &target->node);
1019 res = target->memregions.child;
1020 while (res) {
1021 res_next = res->sibling;
1022 __release_region(&target->memregions, res->start,
1023 resource_size(res));
1024 res = res_next;
1025 }
1026 kfree(objp: target);
1027 }
1028
1029 list_for_each_entry_safe(initiator, inext, &initiators, node) {
1030 list_del(entry: &initiator->node);
1031 kfree(objp: initiator);
1032 }
1033
1034 list_for_each_entry_safe(loc, lnext, &localities, node) {
1035 list_del(entry: &loc->node);
1036 kfree(objp: loc);
1037 }
1038}
1039
1040static __init int hmat_init(void)
1041{
1042 struct acpi_table_header *tbl;
1043 enum acpi_hmat_type i;
1044 acpi_status status;
1045
1046 if (srat_disabled() || hmat_disable)
1047 return 0;
1048
1049 status = acpi_get_table(ACPI_SIG_SRAT, instance: 0, out_table: &tbl);
1050 if (ACPI_FAILURE(status))
1051 return 0;
1052
1053 if (acpi_table_parse_entries(ACPI_SIG_SRAT,
1054 table_size: sizeof(struct acpi_table_srat),
1055 entry_id: ACPI_SRAT_TYPE_MEMORY_AFFINITY,
1056 handler: srat_parse_mem_affinity, max_entries: 0) < 0)
1057 goto out_put;
1058
1059 if (acpi_table_parse_entries(ACPI_SIG_SRAT,
1060 table_size: sizeof(struct acpi_table_srat),
1061 entry_id: ACPI_SRAT_TYPE_GENERIC_PORT_AFFINITY,
1062 handler: srat_parse_genport_affinity, max_entries: 0) < 0)
1063 goto out_put;
1064
1065 acpi_put_table(table: tbl);
1066
1067 status = acpi_get_table(ACPI_SIG_HMAT, instance: 0, out_table: &tbl);
1068 if (ACPI_FAILURE(status))
1069 goto out_put;
1070
1071 hmat_revision = tbl->revision;
1072 switch (hmat_revision) {
1073 case 1:
1074 case 2:
1075 break;
1076 default:
1077 pr_notice("Ignoring: Unknown revision:%d\n", hmat_revision);
1078 goto out_put;
1079 }
1080
1081 for (i = ACPI_HMAT_TYPE_PROXIMITY; i < ACPI_HMAT_TYPE_RESERVED; i++) {
1082 if (acpi_table_parse_entries(ACPI_SIG_HMAT,
1083 table_size: sizeof(struct acpi_table_hmat), entry_id: i,
1084 handler: hmat_parse_subtable, max_entries: 0) < 0) {
1085 pr_notice("Ignoring: Invalid table");
1086 goto out_put;
1087 }
1088 }
1089 hmat_register_targets();
1090
1091 /* Keep the table and structures if the notifier may use them */
1092 if (hotplug_node_notifier(hmat_callback, HMAT_CALLBACK_PRI))
1093 goto out_put;
1094
1095 if (!hmat_set_default_dram_perf())
1096 register_mt_adistance_algorithm(nb: &hmat_adist_nb);
1097
1098 return 0;
1099out_put:
1100 hmat_free_structures();
1101 acpi_put_table(table: tbl);
1102 return 0;
1103}
1104subsys_initcall(hmat_init);
1105

source code of linux/drivers/acpi/numa/hmat.c