1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright (c) 2019, Intel Corporation. |
4 | * |
5 | * Heterogeneous Memory Attributes Table (HMAT) representation |
6 | * |
7 | * This program parses and reports the platform's HMAT tables, and registers |
8 | * the applicable attributes with the node's interfaces. |
9 | */ |
10 | |
11 | #define pr_fmt(fmt) "acpi/hmat: " fmt |
12 | |
13 | #include <linux/acpi.h> |
14 | #include <linux/bitops.h> |
15 | #include <linux/device.h> |
16 | #include <linux/init.h> |
17 | #include <linux/list.h> |
18 | #include <linux/mm.h> |
19 | #include <linux/platform_device.h> |
20 | #include <linux/list_sort.h> |
21 | #include <linux/memregion.h> |
22 | #include <linux/memory.h> |
23 | #include <linux/mutex.h> |
24 | #include <linux/node.h> |
25 | #include <linux/sysfs.h> |
26 | #include <linux/dax.h> |
27 | #include <linux/memory-tiers.h> |
28 | |
29 | static u8 hmat_revision; |
30 | static int hmat_disable __initdata; |
31 | |
32 | void __init disable_hmat(void) |
33 | { |
34 | hmat_disable = 1; |
35 | } |
36 | |
37 | static LIST_HEAD(targets); |
38 | static LIST_HEAD(initiators); |
39 | static LIST_HEAD(localities); |
40 | |
41 | static DEFINE_MUTEX(target_lock); |
42 | |
43 | /* |
44 | * The defined enum order is used to prioritize attributes to break ties when |
45 | * selecting the best performing node. |
46 | */ |
47 | enum locality_types { |
48 | WRITE_LATENCY, |
49 | READ_LATENCY, |
50 | WRITE_BANDWIDTH, |
51 | READ_BANDWIDTH, |
52 | }; |
53 | |
54 | static struct memory_locality *localities_types[4]; |
55 | |
56 | struct target_cache { |
57 | struct list_head node; |
58 | struct node_cache_attrs cache_attrs; |
59 | }; |
60 | |
61 | struct memory_target { |
62 | struct list_head node; |
63 | unsigned int memory_pxm; |
64 | unsigned int processor_pxm; |
65 | struct resource memregions; |
66 | struct node_hmem_attrs hmem_attrs[2]; |
67 | struct list_head caches; |
68 | struct node_cache_attrs cache_attrs; |
69 | bool registered; |
70 | }; |
71 | |
72 | struct memory_initiator { |
73 | struct list_head node; |
74 | unsigned int processor_pxm; |
75 | bool has_cpu; |
76 | }; |
77 | |
78 | struct memory_locality { |
79 | struct list_head node; |
80 | struct acpi_hmat_locality *hmat_loc; |
81 | }; |
82 | |
83 | static struct memory_initiator *find_mem_initiator(unsigned int cpu_pxm) |
84 | { |
85 | struct memory_initiator *initiator; |
86 | |
87 | list_for_each_entry(initiator, &initiators, node) |
88 | if (initiator->processor_pxm == cpu_pxm) |
89 | return initiator; |
90 | return NULL; |
91 | } |
92 | |
93 | static struct memory_target *find_mem_target(unsigned int mem_pxm) |
94 | { |
95 | struct memory_target *target; |
96 | |
97 | list_for_each_entry(target, &targets, node) |
98 | if (target->memory_pxm == mem_pxm) |
99 | return target; |
100 | return NULL; |
101 | } |
102 | |
103 | static __init void alloc_memory_initiator(unsigned int cpu_pxm) |
104 | { |
105 | struct memory_initiator *initiator; |
106 | |
107 | if (pxm_to_node(cpu_pxm) == NUMA_NO_NODE) |
108 | return; |
109 | |
110 | initiator = find_mem_initiator(cpu_pxm); |
111 | if (initiator) |
112 | return; |
113 | |
114 | initiator = kzalloc(size: sizeof(*initiator), GFP_KERNEL); |
115 | if (!initiator) |
116 | return; |
117 | |
118 | initiator->processor_pxm = cpu_pxm; |
119 | initiator->has_cpu = node_state(node: pxm_to_node(cpu_pxm), state: N_CPU); |
120 | list_add_tail(new: &initiator->node, head: &initiators); |
121 | } |
122 | |
123 | static __init void alloc_memory_target(unsigned int mem_pxm, |
124 | resource_size_t start, resource_size_t len) |
125 | { |
126 | struct memory_target *target; |
127 | |
128 | target = find_mem_target(mem_pxm); |
129 | if (!target) { |
130 | target = kzalloc(size: sizeof(*target), GFP_KERNEL); |
131 | if (!target) |
132 | return; |
133 | target->memory_pxm = mem_pxm; |
134 | target->processor_pxm = PXM_INVAL; |
135 | target->memregions = (struct resource) { |
136 | .name = "ACPI mem" , |
137 | .start = 0, |
138 | .end = -1, |
139 | .flags = IORESOURCE_MEM, |
140 | }; |
141 | list_add_tail(new: &target->node, head: &targets); |
142 | INIT_LIST_HEAD(list: &target->caches); |
143 | } |
144 | |
145 | /* |
146 | * There are potentially multiple ranges per PXM, so record each |
147 | * in the per-target memregions resource tree. |
148 | */ |
149 | if (!__request_region(&target->memregions, start, n: len, name: "memory target" , |
150 | IORESOURCE_MEM)) |
151 | pr_warn("failed to reserve %#llx - %#llx in pxm: %d\n" , |
152 | start, start + len, mem_pxm); |
153 | } |
154 | |
155 | static __init const char *hmat_data_type(u8 type) |
156 | { |
157 | switch (type) { |
158 | case ACPI_HMAT_ACCESS_LATENCY: |
159 | return "Access Latency" ; |
160 | case ACPI_HMAT_READ_LATENCY: |
161 | return "Read Latency" ; |
162 | case ACPI_HMAT_WRITE_LATENCY: |
163 | return "Write Latency" ; |
164 | case ACPI_HMAT_ACCESS_BANDWIDTH: |
165 | return "Access Bandwidth" ; |
166 | case ACPI_HMAT_READ_BANDWIDTH: |
167 | return "Read Bandwidth" ; |
168 | case ACPI_HMAT_WRITE_BANDWIDTH: |
169 | return "Write Bandwidth" ; |
170 | default: |
171 | return "Reserved" ; |
172 | } |
173 | } |
174 | |
175 | static __init const char *hmat_data_type_suffix(u8 type) |
176 | { |
177 | switch (type) { |
178 | case ACPI_HMAT_ACCESS_LATENCY: |
179 | case ACPI_HMAT_READ_LATENCY: |
180 | case ACPI_HMAT_WRITE_LATENCY: |
181 | return " nsec" ; |
182 | case ACPI_HMAT_ACCESS_BANDWIDTH: |
183 | case ACPI_HMAT_READ_BANDWIDTH: |
184 | case ACPI_HMAT_WRITE_BANDWIDTH: |
185 | return " MB/s" ; |
186 | default: |
187 | return "" ; |
188 | } |
189 | } |
190 | |
191 | static u32 hmat_normalize(u16 entry, u64 base, u8 type) |
192 | { |
193 | u32 value; |
194 | |
195 | /* |
196 | * Check for invalid and overflow values |
197 | */ |
198 | if (entry == 0xffff || !entry) |
199 | return 0; |
200 | else if (base > (UINT_MAX / (entry))) |
201 | return 0; |
202 | |
203 | /* |
204 | * Divide by the base unit for version 1, convert latency from |
205 | * picosenonds to nanoseconds if revision 2. |
206 | */ |
207 | value = entry * base; |
208 | if (hmat_revision == 1) { |
209 | if (value < 10) |
210 | return 0; |
211 | value = DIV_ROUND_UP(value, 10); |
212 | } else if (hmat_revision == 2) { |
213 | switch (type) { |
214 | case ACPI_HMAT_ACCESS_LATENCY: |
215 | case ACPI_HMAT_READ_LATENCY: |
216 | case ACPI_HMAT_WRITE_LATENCY: |
217 | value = DIV_ROUND_UP(value, 1000); |
218 | break; |
219 | default: |
220 | break; |
221 | } |
222 | } |
223 | return value; |
224 | } |
225 | |
226 | static void hmat_update_target_access(struct memory_target *target, |
227 | u8 type, u32 value, int access) |
228 | { |
229 | switch (type) { |
230 | case ACPI_HMAT_ACCESS_LATENCY: |
231 | target->hmem_attrs[access].read_latency = value; |
232 | target->hmem_attrs[access].write_latency = value; |
233 | break; |
234 | case ACPI_HMAT_READ_LATENCY: |
235 | target->hmem_attrs[access].read_latency = value; |
236 | break; |
237 | case ACPI_HMAT_WRITE_LATENCY: |
238 | target->hmem_attrs[access].write_latency = value; |
239 | break; |
240 | case ACPI_HMAT_ACCESS_BANDWIDTH: |
241 | target->hmem_attrs[access].read_bandwidth = value; |
242 | target->hmem_attrs[access].write_bandwidth = value; |
243 | break; |
244 | case ACPI_HMAT_READ_BANDWIDTH: |
245 | target->hmem_attrs[access].read_bandwidth = value; |
246 | break; |
247 | case ACPI_HMAT_WRITE_BANDWIDTH: |
248 | target->hmem_attrs[access].write_bandwidth = value; |
249 | break; |
250 | default: |
251 | break; |
252 | } |
253 | } |
254 | |
255 | static __init void hmat_add_locality(struct acpi_hmat_locality *hmat_loc) |
256 | { |
257 | struct memory_locality *loc; |
258 | |
259 | loc = kzalloc(size: sizeof(*loc), GFP_KERNEL); |
260 | if (!loc) { |
261 | pr_notice_once("Failed to allocate HMAT locality\n" ); |
262 | return; |
263 | } |
264 | |
265 | loc->hmat_loc = hmat_loc; |
266 | list_add_tail(new: &loc->node, head: &localities); |
267 | |
268 | switch (hmat_loc->data_type) { |
269 | case ACPI_HMAT_ACCESS_LATENCY: |
270 | localities_types[READ_LATENCY] = loc; |
271 | localities_types[WRITE_LATENCY] = loc; |
272 | break; |
273 | case ACPI_HMAT_READ_LATENCY: |
274 | localities_types[READ_LATENCY] = loc; |
275 | break; |
276 | case ACPI_HMAT_WRITE_LATENCY: |
277 | localities_types[WRITE_LATENCY] = loc; |
278 | break; |
279 | case ACPI_HMAT_ACCESS_BANDWIDTH: |
280 | localities_types[READ_BANDWIDTH] = loc; |
281 | localities_types[WRITE_BANDWIDTH] = loc; |
282 | break; |
283 | case ACPI_HMAT_READ_BANDWIDTH: |
284 | localities_types[READ_BANDWIDTH] = loc; |
285 | break; |
286 | case ACPI_HMAT_WRITE_BANDWIDTH: |
287 | localities_types[WRITE_BANDWIDTH] = loc; |
288 | break; |
289 | default: |
290 | break; |
291 | } |
292 | } |
293 | |
294 | static __init int hmat_parse_locality(union acpi_subtable_headers *, |
295 | const unsigned long end) |
296 | { |
297 | struct acpi_hmat_locality *hmat_loc = (void *)header; |
298 | struct memory_target *target; |
299 | unsigned int init, targ, total_size, ipds, tpds; |
300 | u32 *inits, *targs, value; |
301 | u16 *entries; |
302 | u8 type, mem_hier; |
303 | |
304 | if (hmat_loc->header.length < sizeof(*hmat_loc)) { |
305 | pr_notice("Unexpected locality header length: %u\n" , |
306 | hmat_loc->header.length); |
307 | return -EINVAL; |
308 | } |
309 | |
310 | type = hmat_loc->data_type; |
311 | mem_hier = hmat_loc->flags & ACPI_HMAT_MEMORY_HIERARCHY; |
312 | ipds = hmat_loc->number_of_initiator_Pds; |
313 | tpds = hmat_loc->number_of_target_Pds; |
314 | total_size = sizeof(*hmat_loc) + sizeof(*entries) * ipds * tpds + |
315 | sizeof(*inits) * ipds + sizeof(*targs) * tpds; |
316 | if (hmat_loc->header.length < total_size) { |
317 | pr_notice("Unexpected locality header length:%u, minimum required:%u\n" , |
318 | hmat_loc->header.length, total_size); |
319 | return -EINVAL; |
320 | } |
321 | |
322 | pr_info("Locality: Flags:%02x Type:%s Initiator Domains:%u Target Domains:%u Base:%lld\n" , |
323 | hmat_loc->flags, hmat_data_type(type), ipds, tpds, |
324 | hmat_loc->entry_base_unit); |
325 | |
326 | inits = (u32 *)(hmat_loc + 1); |
327 | targs = inits + ipds; |
328 | entries = (u16 *)(targs + tpds); |
329 | for (init = 0; init < ipds; init++) { |
330 | alloc_memory_initiator(cpu_pxm: inits[init]); |
331 | for (targ = 0; targ < tpds; targ++) { |
332 | value = hmat_normalize(entry: entries[init * tpds + targ], |
333 | base: hmat_loc->entry_base_unit, |
334 | type); |
335 | pr_info(" Initiator-Target[%u-%u]:%u%s\n" , |
336 | inits[init], targs[targ], value, |
337 | hmat_data_type_suffix(type)); |
338 | |
339 | if (mem_hier == ACPI_HMAT_MEMORY) { |
340 | target = find_mem_target(mem_pxm: targs[targ]); |
341 | if (target && target->processor_pxm == inits[init]) { |
342 | hmat_update_target_access(target, type, value, access: 0); |
343 | /* If the node has a CPU, update access 1 */ |
344 | if (node_state(node: pxm_to_node(inits[init]), state: N_CPU)) |
345 | hmat_update_target_access(target, type, value, access: 1); |
346 | } |
347 | } |
348 | } |
349 | } |
350 | |
351 | if (mem_hier == ACPI_HMAT_MEMORY) |
352 | hmat_add_locality(hmat_loc); |
353 | |
354 | return 0; |
355 | } |
356 | |
357 | static __init int hmat_parse_cache(union acpi_subtable_headers *, |
358 | const unsigned long end) |
359 | { |
360 | struct acpi_hmat_cache *cache = (void *)header; |
361 | struct memory_target *target; |
362 | struct target_cache *tcache; |
363 | u32 attrs; |
364 | |
365 | if (cache->header.length < sizeof(*cache)) { |
366 | pr_notice("Unexpected cache header length: %u\n" , |
367 | cache->header.length); |
368 | return -EINVAL; |
369 | } |
370 | |
371 | attrs = cache->cache_attributes; |
372 | pr_info("Cache: Domain:%u Size:%llu Attrs:%08x SMBIOS Handles:%d\n" , |
373 | cache->memory_PD, cache->cache_size, attrs, |
374 | cache->number_of_SMBIOShandles); |
375 | |
376 | target = find_mem_target(mem_pxm: cache->memory_PD); |
377 | if (!target) |
378 | return 0; |
379 | |
380 | tcache = kzalloc(size: sizeof(*tcache), GFP_KERNEL); |
381 | if (!tcache) { |
382 | pr_notice_once("Failed to allocate HMAT cache info\n" ); |
383 | return 0; |
384 | } |
385 | |
386 | tcache->cache_attrs.size = cache->cache_size; |
387 | tcache->cache_attrs.level = (attrs & ACPI_HMAT_CACHE_LEVEL) >> 4; |
388 | tcache->cache_attrs.line_size = (attrs & ACPI_HMAT_CACHE_LINE_SIZE) >> 16; |
389 | |
390 | switch ((attrs & ACPI_HMAT_CACHE_ASSOCIATIVITY) >> 8) { |
391 | case ACPI_HMAT_CA_DIRECT_MAPPED: |
392 | tcache->cache_attrs.indexing = NODE_CACHE_DIRECT_MAP; |
393 | break; |
394 | case ACPI_HMAT_CA_COMPLEX_CACHE_INDEXING: |
395 | tcache->cache_attrs.indexing = NODE_CACHE_INDEXED; |
396 | break; |
397 | case ACPI_HMAT_CA_NONE: |
398 | default: |
399 | tcache->cache_attrs.indexing = NODE_CACHE_OTHER; |
400 | break; |
401 | } |
402 | |
403 | switch ((attrs & ACPI_HMAT_WRITE_POLICY) >> 12) { |
404 | case ACPI_HMAT_CP_WB: |
405 | tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_BACK; |
406 | break; |
407 | case ACPI_HMAT_CP_WT: |
408 | tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_THROUGH; |
409 | break; |
410 | case ACPI_HMAT_CP_NONE: |
411 | default: |
412 | tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_OTHER; |
413 | break; |
414 | } |
415 | list_add_tail(new: &tcache->node, head: &target->caches); |
416 | |
417 | return 0; |
418 | } |
419 | |
420 | static int __init hmat_parse_proximity_domain(union acpi_subtable_headers *, |
421 | const unsigned long end) |
422 | { |
423 | struct acpi_hmat_proximity_domain *p = (void *)header; |
424 | struct memory_target *target = NULL; |
425 | |
426 | if (p->header.length != sizeof(*p)) { |
427 | pr_notice("Unexpected address range header length: %u\n" , |
428 | p->header.length); |
429 | return -EINVAL; |
430 | } |
431 | |
432 | if (hmat_revision == 1) |
433 | pr_info("Memory (%#llx length %#llx) Flags:%04x Processor Domain:%u Memory Domain:%u\n" , |
434 | p->reserved3, p->reserved4, p->flags, p->processor_PD, |
435 | p->memory_PD); |
436 | else |
437 | pr_info("Memory Flags:%04x Processor Domain:%u Memory Domain:%u\n" , |
438 | p->flags, p->processor_PD, p->memory_PD); |
439 | |
440 | if ((hmat_revision == 1 && p->flags & ACPI_HMAT_MEMORY_PD_VALID) || |
441 | hmat_revision > 1) { |
442 | target = find_mem_target(mem_pxm: p->memory_PD); |
443 | if (!target) { |
444 | pr_debug("Memory Domain missing from SRAT\n" ); |
445 | return -EINVAL; |
446 | } |
447 | } |
448 | if (target && p->flags & ACPI_HMAT_PROCESSOR_PD_VALID) { |
449 | int p_node = pxm_to_node(p->processor_PD); |
450 | |
451 | if (p_node == NUMA_NO_NODE) { |
452 | pr_debug("Invalid Processor Domain\n" ); |
453 | return -EINVAL; |
454 | } |
455 | target->processor_pxm = p->processor_PD; |
456 | } |
457 | |
458 | return 0; |
459 | } |
460 | |
461 | static int __init hmat_parse_subtable(union acpi_subtable_headers *, |
462 | const unsigned long end) |
463 | { |
464 | struct acpi_hmat_structure *hdr = (void *)header; |
465 | |
466 | if (!hdr) |
467 | return -EINVAL; |
468 | |
469 | switch (hdr->type) { |
470 | case ACPI_HMAT_TYPE_PROXIMITY: |
471 | return hmat_parse_proximity_domain(header, end); |
472 | case ACPI_HMAT_TYPE_LOCALITY: |
473 | return hmat_parse_locality(header, end); |
474 | case ACPI_HMAT_TYPE_CACHE: |
475 | return hmat_parse_cache(header, end); |
476 | default: |
477 | return -EINVAL; |
478 | } |
479 | } |
480 | |
481 | static __init int srat_parse_mem_affinity(union acpi_subtable_headers *, |
482 | const unsigned long end) |
483 | { |
484 | struct acpi_srat_mem_affinity *ma = (void *)header; |
485 | |
486 | if (!ma) |
487 | return -EINVAL; |
488 | if (!(ma->flags & ACPI_SRAT_MEM_ENABLED)) |
489 | return 0; |
490 | alloc_memory_target(mem_pxm: ma->proximity_domain, start: ma->base_address, len: ma->length); |
491 | return 0; |
492 | } |
493 | |
494 | static u32 hmat_initiator_perf(struct memory_target *target, |
495 | struct memory_initiator *initiator, |
496 | struct acpi_hmat_locality *hmat_loc) |
497 | { |
498 | unsigned int ipds, tpds, i, idx = 0, tdx = 0; |
499 | u32 *inits, *targs; |
500 | u16 *entries; |
501 | |
502 | ipds = hmat_loc->number_of_initiator_Pds; |
503 | tpds = hmat_loc->number_of_target_Pds; |
504 | inits = (u32 *)(hmat_loc + 1); |
505 | targs = inits + ipds; |
506 | entries = (u16 *)(targs + tpds); |
507 | |
508 | for (i = 0; i < ipds; i++) { |
509 | if (inits[i] == initiator->processor_pxm) { |
510 | idx = i; |
511 | break; |
512 | } |
513 | } |
514 | |
515 | if (i == ipds) |
516 | return 0; |
517 | |
518 | for (i = 0; i < tpds; i++) { |
519 | if (targs[i] == target->memory_pxm) { |
520 | tdx = i; |
521 | break; |
522 | } |
523 | } |
524 | if (i == tpds) |
525 | return 0; |
526 | |
527 | return hmat_normalize(entry: entries[idx * tpds + tdx], |
528 | base: hmat_loc->entry_base_unit, |
529 | type: hmat_loc->data_type); |
530 | } |
531 | |
532 | static bool hmat_update_best(u8 type, u32 value, u32 *best) |
533 | { |
534 | bool updated = false; |
535 | |
536 | if (!value) |
537 | return false; |
538 | |
539 | switch (type) { |
540 | case ACPI_HMAT_ACCESS_LATENCY: |
541 | case ACPI_HMAT_READ_LATENCY: |
542 | case ACPI_HMAT_WRITE_LATENCY: |
543 | if (!*best || *best > value) { |
544 | *best = value; |
545 | updated = true; |
546 | } |
547 | break; |
548 | case ACPI_HMAT_ACCESS_BANDWIDTH: |
549 | case ACPI_HMAT_READ_BANDWIDTH: |
550 | case ACPI_HMAT_WRITE_BANDWIDTH: |
551 | if (!*best || *best < value) { |
552 | *best = value; |
553 | updated = true; |
554 | } |
555 | break; |
556 | } |
557 | |
558 | return updated; |
559 | } |
560 | |
561 | static int initiator_cmp(void *priv, const struct list_head *a, |
562 | const struct list_head *b) |
563 | { |
564 | struct memory_initiator *ia; |
565 | struct memory_initiator *ib; |
566 | |
567 | ia = list_entry(a, struct memory_initiator, node); |
568 | ib = list_entry(b, struct memory_initiator, node); |
569 | |
570 | return ia->processor_pxm - ib->processor_pxm; |
571 | } |
572 | |
573 | static int initiators_to_nodemask(unsigned long *p_nodes) |
574 | { |
575 | struct memory_initiator *initiator; |
576 | |
577 | if (list_empty(head: &initiators)) |
578 | return -ENXIO; |
579 | |
580 | list_for_each_entry(initiator, &initiators, node) |
581 | set_bit(nr: initiator->processor_pxm, addr: p_nodes); |
582 | |
583 | return 0; |
584 | } |
585 | |
586 | static void hmat_update_target_attrs(struct memory_target *target, |
587 | unsigned long *p_nodes, int access) |
588 | { |
589 | struct memory_initiator *initiator; |
590 | unsigned int cpu_nid; |
591 | struct memory_locality *loc = NULL; |
592 | u32 best = 0; |
593 | int i; |
594 | |
595 | bitmap_zero(dst: p_nodes, MAX_NUMNODES); |
596 | /* |
597 | * If the Address Range Structure provides a local processor pxm, set |
598 | * only that one. Otherwise, find the best performance attributes and |
599 | * collect all initiators that match. |
600 | */ |
601 | if (target->processor_pxm != PXM_INVAL) { |
602 | cpu_nid = pxm_to_node(target->processor_pxm); |
603 | if (access == 0 || node_state(node: cpu_nid, state: N_CPU)) { |
604 | set_bit(nr: target->processor_pxm, addr: p_nodes); |
605 | return; |
606 | } |
607 | } |
608 | |
609 | if (list_empty(head: &localities)) |
610 | return; |
611 | |
612 | /* |
613 | * We need the initiator list sorted so we can use bitmap_clear for |
614 | * previously set initiators when we find a better memory accessor. |
615 | * We'll also use the sorting to prime the candidate nodes with known |
616 | * initiators. |
617 | */ |
618 | list_sort(NULL, head: &initiators, cmp: initiator_cmp); |
619 | if (initiators_to_nodemask(p_nodes) < 0) |
620 | return; |
621 | |
622 | for (i = WRITE_LATENCY; i <= READ_BANDWIDTH; i++) { |
623 | loc = localities_types[i]; |
624 | if (!loc) |
625 | continue; |
626 | |
627 | best = 0; |
628 | list_for_each_entry(initiator, &initiators, node) { |
629 | u32 value; |
630 | |
631 | if (access == 1 && !initiator->has_cpu) { |
632 | clear_bit(nr: initiator->processor_pxm, addr: p_nodes); |
633 | continue; |
634 | } |
635 | if (!test_bit(initiator->processor_pxm, p_nodes)) |
636 | continue; |
637 | |
638 | value = hmat_initiator_perf(target, initiator, hmat_loc: loc->hmat_loc); |
639 | if (hmat_update_best(type: loc->hmat_loc->data_type, value, best: &best)) |
640 | bitmap_clear(map: p_nodes, start: 0, nbits: initiator->processor_pxm); |
641 | if (value != best) |
642 | clear_bit(nr: initiator->processor_pxm, addr: p_nodes); |
643 | } |
644 | if (best) |
645 | hmat_update_target_access(target, type: loc->hmat_loc->data_type, value: best, access); |
646 | } |
647 | } |
648 | |
649 | static void __hmat_register_target_initiators(struct memory_target *target, |
650 | unsigned long *p_nodes, |
651 | int access) |
652 | { |
653 | unsigned int mem_nid, cpu_nid; |
654 | int i; |
655 | |
656 | mem_nid = pxm_to_node(target->memory_pxm); |
657 | hmat_update_target_attrs(target, p_nodes, access); |
658 | for_each_set_bit(i, p_nodes, MAX_NUMNODES) { |
659 | cpu_nid = pxm_to_node(i); |
660 | register_memory_node_under_compute_node(mem_nid, cpu_nid, access); |
661 | } |
662 | } |
663 | |
664 | static void hmat_register_target_initiators(struct memory_target *target) |
665 | { |
666 | static DECLARE_BITMAP(p_nodes, MAX_NUMNODES); |
667 | |
668 | __hmat_register_target_initiators(target, p_nodes, access: 0); |
669 | __hmat_register_target_initiators(target, p_nodes, access: 1); |
670 | } |
671 | |
672 | static void hmat_register_target_cache(struct memory_target *target) |
673 | { |
674 | unsigned mem_nid = pxm_to_node(target->memory_pxm); |
675 | struct target_cache *tcache; |
676 | |
677 | list_for_each_entry(tcache, &target->caches, node) |
678 | node_add_cache(nid: mem_nid, cache_attrs: &tcache->cache_attrs); |
679 | } |
680 | |
681 | static void hmat_register_target_perf(struct memory_target *target, int access) |
682 | { |
683 | unsigned mem_nid = pxm_to_node(target->memory_pxm); |
684 | node_set_perf_attrs(nid: mem_nid, hmem_attrs: &target->hmem_attrs[access], access); |
685 | } |
686 | |
687 | static void hmat_register_target_devices(struct memory_target *target) |
688 | { |
689 | struct resource *res; |
690 | |
691 | /* |
692 | * Do not bother creating devices if no driver is available to |
693 | * consume them. |
694 | */ |
695 | if (!IS_ENABLED(CONFIG_DEV_DAX_HMEM)) |
696 | return; |
697 | |
698 | for (res = target->memregions.child; res; res = res->sibling) { |
699 | int target_nid = pxm_to_node(target->memory_pxm); |
700 | |
701 | hmem_register_resource(target_nid, r: res); |
702 | } |
703 | } |
704 | |
705 | static void hmat_register_target(struct memory_target *target) |
706 | { |
707 | int nid = pxm_to_node(target->memory_pxm); |
708 | |
709 | /* |
710 | * Devices may belong to either an offline or online |
711 | * node, so unconditionally add them. |
712 | */ |
713 | hmat_register_target_devices(target); |
714 | |
715 | /* |
716 | * Skip offline nodes. This can happen when memory |
717 | * marked EFI_MEMORY_SP, "specific purpose", is applied |
718 | * to all the memory in a proximity domain leading to |
719 | * the node being marked offline / unplugged, or if |
720 | * memory-only "hotplug" node is offline. |
721 | */ |
722 | if (nid == NUMA_NO_NODE || !node_online(nid)) |
723 | return; |
724 | |
725 | mutex_lock(&target_lock); |
726 | if (!target->registered) { |
727 | hmat_register_target_initiators(target); |
728 | hmat_register_target_cache(target); |
729 | hmat_register_target_perf(target, access: 0); |
730 | hmat_register_target_perf(target, access: 1); |
731 | target->registered = true; |
732 | } |
733 | mutex_unlock(lock: &target_lock); |
734 | } |
735 | |
736 | static void hmat_register_targets(void) |
737 | { |
738 | struct memory_target *target; |
739 | |
740 | list_for_each_entry(target, &targets, node) |
741 | hmat_register_target(target); |
742 | } |
743 | |
744 | static int hmat_callback(struct notifier_block *self, |
745 | unsigned long action, void *arg) |
746 | { |
747 | struct memory_target *target; |
748 | struct memory_notify *mnb = arg; |
749 | int pxm, nid = mnb->status_change_nid; |
750 | |
751 | if (nid == NUMA_NO_NODE || action != MEM_ONLINE) |
752 | return NOTIFY_OK; |
753 | |
754 | pxm = node_to_pxm(nid); |
755 | target = find_mem_target(mem_pxm: pxm); |
756 | if (!target) |
757 | return NOTIFY_OK; |
758 | |
759 | hmat_register_target(target); |
760 | return NOTIFY_OK; |
761 | } |
762 | |
763 | static int hmat_set_default_dram_perf(void) |
764 | { |
765 | int rc; |
766 | int nid, pxm; |
767 | struct memory_target *target; |
768 | struct node_hmem_attrs *attrs; |
769 | |
770 | if (!default_dram_type) |
771 | return -EIO; |
772 | |
773 | for_each_node_mask(nid, default_dram_type->nodes) { |
774 | pxm = node_to_pxm(nid); |
775 | target = find_mem_target(mem_pxm: pxm); |
776 | if (!target) |
777 | continue; |
778 | attrs = &target->hmem_attrs[1]; |
779 | rc = mt_set_default_dram_perf(nid, perf: attrs, source: "ACPI HMAT" ); |
780 | if (rc) |
781 | return rc; |
782 | } |
783 | |
784 | return 0; |
785 | } |
786 | |
787 | static int hmat_calculate_adistance(struct notifier_block *self, |
788 | unsigned long nid, void *data) |
789 | { |
790 | static DECLARE_BITMAP(p_nodes, MAX_NUMNODES); |
791 | struct memory_target *target; |
792 | struct node_hmem_attrs *perf; |
793 | int *adist = data; |
794 | int pxm; |
795 | |
796 | pxm = node_to_pxm(nid); |
797 | target = find_mem_target(mem_pxm: pxm); |
798 | if (!target) |
799 | return NOTIFY_OK; |
800 | |
801 | mutex_lock(&target_lock); |
802 | hmat_update_target_attrs(target, p_nodes, access: 1); |
803 | mutex_unlock(lock: &target_lock); |
804 | |
805 | perf = &target->hmem_attrs[1]; |
806 | |
807 | if (mt_perf_to_adistance(perf, adist)) |
808 | return NOTIFY_OK; |
809 | |
810 | return NOTIFY_STOP; |
811 | } |
812 | |
813 | static struct notifier_block hmat_adist_nb __meminitdata = { |
814 | .notifier_call = hmat_calculate_adistance, |
815 | .priority = 100, |
816 | }; |
817 | |
818 | static __init void hmat_free_structures(void) |
819 | { |
820 | struct memory_target *target, *tnext; |
821 | struct memory_locality *loc, *lnext; |
822 | struct memory_initiator *initiator, *inext; |
823 | struct target_cache *tcache, *cnext; |
824 | |
825 | list_for_each_entry_safe(target, tnext, &targets, node) { |
826 | struct resource *res, *res_next; |
827 | |
828 | list_for_each_entry_safe(tcache, cnext, &target->caches, node) { |
829 | list_del(entry: &tcache->node); |
830 | kfree(objp: tcache); |
831 | } |
832 | |
833 | list_del(entry: &target->node); |
834 | res = target->memregions.child; |
835 | while (res) { |
836 | res_next = res->sibling; |
837 | __release_region(&target->memregions, res->start, |
838 | resource_size(res)); |
839 | res = res_next; |
840 | } |
841 | kfree(objp: target); |
842 | } |
843 | |
844 | list_for_each_entry_safe(initiator, inext, &initiators, node) { |
845 | list_del(entry: &initiator->node); |
846 | kfree(objp: initiator); |
847 | } |
848 | |
849 | list_for_each_entry_safe(loc, lnext, &localities, node) { |
850 | list_del(entry: &loc->node); |
851 | kfree(objp: loc); |
852 | } |
853 | } |
854 | |
855 | static __init int hmat_init(void) |
856 | { |
857 | struct acpi_table_header *tbl; |
858 | enum acpi_hmat_type i; |
859 | acpi_status status; |
860 | |
861 | if (srat_disabled() || hmat_disable) |
862 | return 0; |
863 | |
864 | status = acpi_get_table(ACPI_SIG_SRAT, instance: 0, out_table: &tbl); |
865 | if (ACPI_FAILURE(status)) |
866 | return 0; |
867 | |
868 | if (acpi_table_parse_entries(ACPI_SIG_SRAT, |
869 | table_size: sizeof(struct acpi_table_srat), |
870 | entry_id: ACPI_SRAT_TYPE_MEMORY_AFFINITY, |
871 | handler: srat_parse_mem_affinity, max_entries: 0) < 0) |
872 | goto out_put; |
873 | acpi_put_table(table: tbl); |
874 | |
875 | status = acpi_get_table(ACPI_SIG_HMAT, instance: 0, out_table: &tbl); |
876 | if (ACPI_FAILURE(status)) |
877 | goto out_put; |
878 | |
879 | hmat_revision = tbl->revision; |
880 | switch (hmat_revision) { |
881 | case 1: |
882 | case 2: |
883 | break; |
884 | default: |
885 | pr_notice("Ignoring: Unknown revision:%d\n" , hmat_revision); |
886 | goto out_put; |
887 | } |
888 | |
889 | for (i = ACPI_HMAT_TYPE_PROXIMITY; i < ACPI_HMAT_TYPE_RESERVED; i++) { |
890 | if (acpi_table_parse_entries(ACPI_SIG_HMAT, |
891 | table_size: sizeof(struct acpi_table_hmat), entry_id: i, |
892 | handler: hmat_parse_subtable, max_entries: 0) < 0) { |
893 | pr_notice("Ignoring: Invalid table" ); |
894 | goto out_put; |
895 | } |
896 | } |
897 | hmat_register_targets(); |
898 | |
899 | /* Keep the table and structures if the notifier may use them */ |
900 | if (hotplug_memory_notifier(hmat_callback, HMAT_CALLBACK_PRI)) |
901 | goto out_put; |
902 | |
903 | if (!hmat_set_default_dram_perf()) |
904 | register_mt_adistance_algorithm(nb: &hmat_adist_nb); |
905 | |
906 | return 0; |
907 | out_put: |
908 | hmat_free_structures(); |
909 | acpi_put_table(table: tbl); |
910 | return 0; |
911 | } |
912 | subsys_initcall(hmat_init); |
913 | |