1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Low level x86 E820 memory map handling functions.
4 *
5 * The firmware and bootloader passes us the "E820 table", which is the primary
6 * physical memory layout description available about x86 systems.
7 *
8 * The kernel takes the E820 memory layout and optionally modifies it with
9 * quirks and other tweaks, and feeds that into the generic Linux memory
10 * allocation code routines via a platform independent interface (memblock, etc.).
11 */
12#include <linux/crash_dump.h>
13#include <linux/memblock.h>
14#include <linux/suspend.h>
15#include <linux/acpi.h>
16#include <linux/firmware-map.h>
17#include <linux/sort.h>
18#include <linux/memory_hotplug.h>
19#include <linux/kvm_types.h>
20
21#include <asm/e820/api.h>
22#include <asm/setup.h>
23
24/*
25 * We organize the E820 table into three main data structures:
26 *
27 * - 'e820_table_firmware': the original firmware version passed to us by the
28 * bootloader - not modified by the kernel. It is composed of two parts:
29 * the first 128 E820 memory entries in boot_params.e820_table and the remaining
30 * (if any) entries of the SETUP_E820_EXT nodes. We use this to:
31 *
32 * - the hibernation code uses it to generate a kernel-independent CRC32
33 * checksum of the physical memory layout of a system.
34 *
35 * - 'e820_table_kexec': a slightly modified (by the kernel) firmware version
36 * passed to us by the bootloader - the major difference between
37 * e820_table_firmware[] and this one is that e820_table_kexec[]
38 * might be modified by the kexec itself to fake an mptable.
39 * We use this to:
40 *
41 * - kexec, which is a bootloader in disguise, uses the original E820
42 * layout to pass to the kexec-ed kernel. This way the original kernel
43 * can have a restricted E820 map while the kexec()-ed kexec-kernel
44 * can have access to full memory - etc.
45 *
46 * Export the memory layout via /sys/firmware/memmap. kexec-tools uses
47 * the entries to create an E820 table for the kexec kernel.
48 *
49 * kexec_file_load in-kernel code uses the table for the kexec kernel.
50 *
51 * - 'e820_table': this is the main E820 table that is massaged by the
52 * low level x86 platform code, or modified by boot parameters, before
53 * passed on to higher level MM layers.
54 *
55 * Once the E820 map has been converted to the standard Linux memory layout
56 * information its role stops - modifying it has no effect and does not get
57 * re-propagated. So its main role is a temporary bootstrap storage of firmware
58 * specific memory layout data during early bootup.
59 */
60static struct e820_table e820_table_init __initdata;
61static struct e820_table e820_table_kexec_init __initdata;
62static struct e820_table e820_table_firmware_init __initdata;
63
64struct e820_table *e820_table __refdata = &e820_table_init;
65struct e820_table *e820_table_kexec __refdata = &e820_table_kexec_init;
66struct e820_table *e820_table_firmware __refdata = &e820_table_firmware_init;
67
68/* For PCI or other memory-mapped resources */
69unsigned long pci_mem_start = 0xaeedbabe;
70#ifdef CONFIG_PCI
71EXPORT_SYMBOL(pci_mem_start);
72#endif
73
74/*
75 * This function checks if any part of the range <start,end> is mapped
76 * with type.
77 */
78static bool _e820__mapped_any(struct e820_table *table,
79 u64 start, u64 end, enum e820_type type)
80{
81 int i;
82
83 for (i = 0; i < table->nr_entries; i++) {
84 struct e820_entry *entry = &table->entries[i];
85
86 if (type && entry->type != type)
87 continue;
88 if (entry->addr >= end || entry->addr + entry->size <= start)
89 continue;
90 return true;
91 }
92 return false;
93}
94
95bool e820__mapped_raw_any(u64 start, u64 end, enum e820_type type)
96{
97 return _e820__mapped_any(table: e820_table_firmware, start, end, type);
98}
99EXPORT_SYMBOL_FOR_KVM(e820__mapped_raw_any);
100
101bool e820__mapped_any(u64 start, u64 end, enum e820_type type)
102{
103 return _e820__mapped_any(table: e820_table, start, end, type);
104}
105EXPORT_SYMBOL_GPL(e820__mapped_any);
106
107/*
108 * This function checks if the entire <start,end> range is mapped with 'type'.
109 *
110 * Note: this function only works correctly once the E820 table is sorted and
111 * not-overlapping (at least for the range specified), which is the case normally.
112 */
113static struct e820_entry *__e820__mapped_all(u64 start, u64 end,
114 enum e820_type type)
115{
116 int i;
117
118 for (i = 0; i < e820_table->nr_entries; i++) {
119 struct e820_entry *entry = &e820_table->entries[i];
120
121 if (type && entry->type != type)
122 continue;
123
124 /* Is the region (part) in overlap with the current region? */
125 if (entry->addr >= end || entry->addr + entry->size <= start)
126 continue;
127
128 /*
129 * If the region is at the beginning of <start,end> we move
130 * 'start' to the end of the region since it's ok until there
131 */
132 if (entry->addr <= start)
133 start = entry->addr + entry->size;
134
135 /*
136 * If 'start' is now at or beyond 'end', we're done, full
137 * coverage of the desired range exists:
138 */
139 if (start >= end)
140 return entry;
141 }
142
143 return NULL;
144}
145
146/*
147 * This function checks if the entire range <start,end> is mapped with type.
148 */
149bool __init e820__mapped_all(u64 start, u64 end, enum e820_type type)
150{
151 return __e820__mapped_all(start, end, type);
152}
153
154/*
155 * This function returns the type associated with the range <start,end>.
156 */
157int e820__get_entry_type(u64 start, u64 end)
158{
159 struct e820_entry *entry = __e820__mapped_all(start, end, type: 0);
160
161 return entry ? entry->type : -EINVAL;
162}
163
164/*
165 * Add a memory region to the kernel E820 map.
166 */
167static void __init __e820__range_add(struct e820_table *table, u64 start, u64 size, enum e820_type type)
168{
169 int x = table->nr_entries;
170
171 if (x >= ARRAY_SIZE(table->entries)) {
172 pr_err("too many entries; ignoring [mem %#010llx-%#010llx]\n",
173 start, start + size - 1);
174 return;
175 }
176
177 table->entries[x].addr = start;
178 table->entries[x].size = size;
179 table->entries[x].type = type;
180 table->nr_entries++;
181}
182
183void __init e820__range_add(u64 start, u64 size, enum e820_type type)
184{
185 __e820__range_add(table: e820_table, start, size, type);
186}
187
188static void __init e820_print_type(enum e820_type type)
189{
190 switch (type) {
191 case E820_TYPE_RAM: pr_cont("usable"); break;
192 case E820_TYPE_RESERVED: pr_cont("reserved"); break;
193 case E820_TYPE_SOFT_RESERVED: pr_cont("soft reserved"); break;
194 case E820_TYPE_ACPI: pr_cont("ACPI data"); break;
195 case E820_TYPE_NVS: pr_cont("ACPI NVS"); break;
196 case E820_TYPE_UNUSABLE: pr_cont("unusable"); break;
197 case E820_TYPE_PMEM: /* Fall through: */
198 case E820_TYPE_PRAM: pr_cont("persistent (type %u)", type); break;
199 default: pr_cont("type %u", type); break;
200 }
201}
202
203void __init e820__print_table(char *who)
204{
205 int i;
206
207 for (i = 0; i < e820_table->nr_entries; i++) {
208 pr_info("%s: [mem %#018Lx-%#018Lx] ",
209 who,
210 e820_table->entries[i].addr,
211 e820_table->entries[i].addr + e820_table->entries[i].size - 1);
212
213 e820_print_type(type: e820_table->entries[i].type);
214 pr_cont("\n");
215 }
216}
217
218/*
219 * Sanitize an E820 map.
220 *
221 * Some E820 layouts include overlapping entries. The following
222 * replaces the original E820 map with a new one, removing overlaps,
223 * and resolving conflicting memory types in favor of highest
224 * numbered type.
225 *
226 * The input parameter 'entries' points to an array of 'struct
227 * e820_entry' which on entry has elements in the range [0, *nr_entries)
228 * valid, and which has space for up to max_nr_entries entries.
229 * On return, the resulting sanitized E820 map entries will be in
230 * overwritten in the same location, starting at 'entries'.
231 *
232 * The integer pointed to by nr_entries must be valid on entry (the
233 * current number of valid entries located at 'entries'). If the
234 * sanitizing succeeds the *nr_entries will be updated with the new
235 * number of valid entries (something no more than max_nr_entries).
236 *
237 * The return value from e820__update_table() is zero if it
238 * successfully 'sanitized' the map entries passed in, and is -1
239 * if it did nothing, which can happen if either of (1) it was
240 * only passed one map entry, or (2) any of the input map entries
241 * were invalid (start + size < start, meaning that the size was
242 * so big the described memory range wrapped around through zero.)
243 *
244 * Visually we're performing the following
245 * (1,2,3,4 = memory types)...
246 *
247 * Sample memory map (w/overlaps):
248 * ____22__________________
249 * ______________________4_
250 * ____1111________________
251 * _44_____________________
252 * 11111111________________
253 * ____________________33__
254 * ___________44___________
255 * __________33333_________
256 * ______________22________
257 * ___________________2222_
258 * _________111111111______
259 * _____________________11_
260 * _________________4______
261 *
262 * Sanitized equivalent (no overlap):
263 * 1_______________________
264 * _44_____________________
265 * ___1____________________
266 * ____22__________________
267 * ______11________________
268 * _________1______________
269 * __________3_____________
270 * ___________44___________
271 * _____________33_________
272 * _______________2________
273 * ________________1_______
274 * _________________4______
275 * ___________________2____
276 * ____________________33__
277 * ______________________4_
278 */
279struct change_member {
280 /* Pointer to the original entry: */
281 struct e820_entry *entry;
282 /* Address for this change point: */
283 unsigned long long addr;
284};
285
286static struct change_member change_point_list[2*E820_MAX_ENTRIES] __initdata;
287static struct change_member *change_point[2*E820_MAX_ENTRIES] __initdata;
288static struct e820_entry *overlap_list[E820_MAX_ENTRIES] __initdata;
289static struct e820_entry new_entries[E820_MAX_ENTRIES] __initdata;
290
291static int __init cpcompare(const void *a, const void *b)
292{
293 struct change_member * const *app = a, * const *bpp = b;
294 const struct change_member *ap = *app, *bp = *bpp;
295
296 /*
297 * Inputs are pointers to two elements of change_point[]. If their
298 * addresses are not equal, their difference dominates. If the addresses
299 * are equal, then consider one that represents the end of its region
300 * to be greater than one that does not.
301 */
302 if (ap->addr != bp->addr)
303 return ap->addr > bp->addr ? 1 : -1;
304
305 return (ap->addr != ap->entry->addr) - (bp->addr != bp->entry->addr);
306}
307
308static bool e820_nomerge(enum e820_type type)
309{
310 /*
311 * These types may indicate distinct platform ranges aligned to
312 * numa node, protection domain, performance domain, or other
313 * boundaries. Do not merge them.
314 */
315 if (type == E820_TYPE_PRAM)
316 return true;
317 if (type == E820_TYPE_SOFT_RESERVED)
318 return true;
319 return false;
320}
321
322int __init e820__update_table(struct e820_table *table)
323{
324 struct e820_entry *entries = table->entries;
325 u32 max_nr_entries = ARRAY_SIZE(table->entries);
326 enum e820_type current_type, last_type;
327 unsigned long long last_addr;
328 u32 new_nr_entries, overlap_entries;
329 u32 i, chg_idx, chg_nr;
330
331 /* If there's only one memory region, don't bother: */
332 if (table->nr_entries < 2)
333 return -1;
334
335 BUG_ON(table->nr_entries > max_nr_entries);
336
337 /* Bail out if we find any unreasonable addresses in the map: */
338 for (i = 0; i < table->nr_entries; i++) {
339 if (entries[i].addr + entries[i].size < entries[i].addr)
340 return -1;
341 }
342
343 /* Create pointers for initial change-point information (for sorting): */
344 for (i = 0; i < 2 * table->nr_entries; i++)
345 change_point[i] = &change_point_list[i];
346
347 /*
348 * Record all known change-points (starting and ending addresses),
349 * omitting empty memory regions:
350 */
351 chg_idx = 0;
352 for (i = 0; i < table->nr_entries; i++) {
353 if (entries[i].size != 0) {
354 change_point[chg_idx]->addr = entries[i].addr;
355 change_point[chg_idx++]->entry = &entries[i];
356 change_point[chg_idx]->addr = entries[i].addr + entries[i].size;
357 change_point[chg_idx++]->entry = &entries[i];
358 }
359 }
360 chg_nr = chg_idx;
361
362 /* Sort change-point list by memory addresses (low -> high): */
363 sort(base: change_point, num: chg_nr, size: sizeof(*change_point), cmp_func: cpcompare, NULL);
364
365 /* Create a new memory map, removing overlaps: */
366 overlap_entries = 0; /* Number of entries in the overlap table */
367 new_nr_entries = 0; /* Index for creating new map entries */
368 last_type = 0; /* Start with undefined memory type */
369 last_addr = 0; /* Start with 0 as last starting address */
370
371 /* Loop through change-points, determining effect on the new map: */
372 for (chg_idx = 0; chg_idx < chg_nr; chg_idx++) {
373 /* Keep track of all overlapping entries */
374 if (change_point[chg_idx]->addr == change_point[chg_idx]->entry->addr) {
375 /* Add map entry to overlap list (> 1 entry implies an overlap) */
376 overlap_list[overlap_entries++] = change_point[chg_idx]->entry;
377 } else {
378 /* Remove entry from list (order independent, so swap with last): */
379 for (i = 0; i < overlap_entries; i++) {
380 if (overlap_list[i] == change_point[chg_idx]->entry)
381 overlap_list[i] = overlap_list[overlap_entries-1];
382 }
383 overlap_entries--;
384 }
385 /*
386 * If there are overlapping entries, decide which
387 * "type" to use (larger value takes precedence --
388 * 1=usable, 2,3,4,4+=unusable)
389 */
390 current_type = 0;
391 for (i = 0; i < overlap_entries; i++) {
392 if (overlap_list[i]->type > current_type)
393 current_type = overlap_list[i]->type;
394 }
395
396 /* Continue building up new map based on this information: */
397 if (current_type != last_type || e820_nomerge(type: current_type)) {
398 if (last_type) {
399 new_entries[new_nr_entries].size = change_point[chg_idx]->addr - last_addr;
400 /* Move forward only if the new size was non-zero: */
401 if (new_entries[new_nr_entries].size != 0)
402 /* No more space left for new entries? */
403 if (++new_nr_entries >= max_nr_entries)
404 break;
405 }
406 if (current_type) {
407 new_entries[new_nr_entries].addr = change_point[chg_idx]->addr;
408 new_entries[new_nr_entries].type = current_type;
409 last_addr = change_point[chg_idx]->addr;
410 }
411 last_type = current_type;
412 }
413 }
414
415 /* Copy the new entries into the original location: */
416 memcpy(entries, new_entries, new_nr_entries*sizeof(*entries));
417 table->nr_entries = new_nr_entries;
418
419 return 0;
420}
421
422static int __init __append_e820_table(struct boot_e820_entry *entries, u32 nr_entries)
423{
424 struct boot_e820_entry *entry = entries;
425
426 while (nr_entries) {
427 u64 start = entry->addr;
428 u64 size = entry->size;
429 u64 end = start + size - 1;
430 u32 type = entry->type;
431
432 /* Ignore the entry on 64-bit overflow: */
433 if (start > end && likely(size))
434 return -1;
435
436 e820__range_add(start, size, type);
437
438 entry++;
439 nr_entries--;
440 }
441 return 0;
442}
443
444/*
445 * Copy the BIOS E820 map into a safe place.
446 *
447 * Sanity-check it while we're at it..
448 *
449 * If we're lucky and live on a modern system, the setup code
450 * will have given us a memory map that we can use to properly
451 * set up memory. If we aren't, we'll fake a memory map.
452 */
453static int __init append_e820_table(struct boot_e820_entry *entries, u32 nr_entries)
454{
455 /* Only one memory region (or negative)? Ignore it */
456 if (nr_entries < 2)
457 return -1;
458
459 return __append_e820_table(entries, nr_entries);
460}
461
462static u64 __init
463__e820__range_update(struct e820_table *table, u64 start, u64 size, enum e820_type old_type, enum e820_type new_type)
464{
465 u64 end;
466 unsigned int i;
467 u64 real_updated_size = 0;
468
469 BUG_ON(old_type == new_type);
470
471 if (size > (ULLONG_MAX - start))
472 size = ULLONG_MAX - start;
473
474 end = start + size;
475 printk(KERN_DEBUG "e820: update [mem %#010Lx-%#010Lx] ", start, end - 1);
476 e820_print_type(type: old_type);
477 pr_cont(" ==> ");
478 e820_print_type(type: new_type);
479 pr_cont("\n");
480
481 for (i = 0; i < table->nr_entries; i++) {
482 struct e820_entry *entry = &table->entries[i];
483 u64 final_start, final_end;
484 u64 entry_end;
485
486 if (entry->type != old_type)
487 continue;
488
489 entry_end = entry->addr + entry->size;
490
491 /* Completely covered by new range? */
492 if (entry->addr >= start && entry_end <= end) {
493 entry->type = new_type;
494 real_updated_size += entry->size;
495 continue;
496 }
497
498 /* New range is completely covered? */
499 if (entry->addr < start && entry_end > end) {
500 __e820__range_add(table, start, size, type: new_type);
501 __e820__range_add(table, start: end, size: entry_end - end, type: entry->type);
502 entry->size = start - entry->addr;
503 real_updated_size += size;
504 continue;
505 }
506
507 /* Partially covered: */
508 final_start = max(start, entry->addr);
509 final_end = min(end, entry_end);
510 if (final_start >= final_end)
511 continue;
512
513 __e820__range_add(table, start: final_start, size: final_end - final_start, type: new_type);
514
515 real_updated_size += final_end - final_start;
516
517 /*
518 * Left range could be head or tail, so need to update
519 * its size first:
520 */
521 entry->size -= final_end - final_start;
522 if (entry->addr < final_start)
523 continue;
524
525 entry->addr = final_end;
526 }
527 return real_updated_size;
528}
529
530u64 __init e820__range_update(u64 start, u64 size, enum e820_type old_type, enum e820_type new_type)
531{
532 return __e820__range_update(table: e820_table, start, size, old_type, new_type);
533}
534
535u64 __init e820__range_update_table(struct e820_table *t, u64 start, u64 size,
536 enum e820_type old_type, enum e820_type new_type)
537{
538 return __e820__range_update(table: t, start, size, old_type, new_type);
539}
540
541/* Remove a range of memory from the E820 table: */
542u64 __init e820__range_remove(u64 start, u64 size, enum e820_type old_type, bool check_type)
543{
544 int i;
545 u64 end;
546 u64 real_removed_size = 0;
547
548 if (size > (ULLONG_MAX - start))
549 size = ULLONG_MAX - start;
550
551 end = start + size;
552 printk(KERN_DEBUG "e820: remove [mem %#010Lx-%#010Lx] ", start, end - 1);
553 if (check_type)
554 e820_print_type(type: old_type);
555 pr_cont("\n");
556
557 for (i = 0; i < e820_table->nr_entries; i++) {
558 struct e820_entry *entry = &e820_table->entries[i];
559 u64 final_start, final_end;
560 u64 entry_end;
561
562 if (check_type && entry->type != old_type)
563 continue;
564
565 entry_end = entry->addr + entry->size;
566
567 /* Completely covered? */
568 if (entry->addr >= start && entry_end <= end) {
569 real_removed_size += entry->size;
570 memset(entry, 0, sizeof(*entry));
571 continue;
572 }
573
574 /* Is the new range completely covered? */
575 if (entry->addr < start && entry_end > end) {
576 e820__range_add(start: end, size: entry_end - end, type: entry->type);
577 entry->size = start - entry->addr;
578 real_removed_size += size;
579 continue;
580 }
581
582 /* Partially covered: */
583 final_start = max(start, entry->addr);
584 final_end = min(end, entry_end);
585 if (final_start >= final_end)
586 continue;
587
588 real_removed_size += final_end - final_start;
589
590 /*
591 * Left range could be head or tail, so need to update
592 * the size first:
593 */
594 entry->size -= final_end - final_start;
595 if (entry->addr < final_start)
596 continue;
597
598 entry->addr = final_end;
599 }
600 return real_removed_size;
601}
602
603void __init e820__update_table_print(void)
604{
605 if (e820__update_table(table: e820_table))
606 return;
607
608 pr_info("modified physical RAM map:\n");
609 e820__print_table(who: "modified");
610}
611
612static void __init e820__update_table_kexec(void)
613{
614 e820__update_table(table: e820_table_kexec);
615}
616
617#define MAX_GAP_END 0x100000000ull
618
619/*
620 * Search for a gap in the E820 memory space from 0 to MAX_GAP_END (4GB).
621 */
622static int __init e820_search_gap(unsigned long *gapstart, unsigned long *gapsize)
623{
624 unsigned long long last = MAX_GAP_END;
625 int i = e820_table->nr_entries;
626 int found = 0;
627
628 while (--i >= 0) {
629 unsigned long long start = e820_table->entries[i].addr;
630 unsigned long long end = start + e820_table->entries[i].size;
631
632 /*
633 * Since "last" is at most 4GB, we know we'll
634 * fit in 32 bits if this condition is true:
635 */
636 if (last > end) {
637 unsigned long gap = last - end;
638
639 if (gap >= *gapsize) {
640 *gapsize = gap;
641 *gapstart = end;
642 found = 1;
643 }
644 }
645 if (start < last)
646 last = start;
647 }
648 return found;
649}
650
651/*
652 * Search for the biggest gap in the low 32 bits of the E820
653 * memory space. We pass this space to the PCI subsystem, so
654 * that it can assign MMIO resources for hotplug or
655 * unconfigured devices in.
656 *
657 * Hopefully the BIOS let enough space left.
658 */
659__init void e820__setup_pci_gap(void)
660{
661 unsigned long gapstart, gapsize;
662 int found;
663
664 gapsize = 0x400000;
665 found = e820_search_gap(gapstart: &gapstart, gapsize: &gapsize);
666
667 if (!found) {
668#ifdef CONFIG_X86_64
669 gapstart = (max_pfn << PAGE_SHIFT) + 1024*1024;
670 pr_err("Cannot find an available gap in the 32-bit address range\n");
671 pr_err("PCI devices with unassigned 32-bit BARs may not work!\n");
672#else
673 gapstart = 0x10000000;
674#endif
675 }
676
677 /*
678 * e820__reserve_resources_late() protects stolen RAM already:
679 */
680 pci_mem_start = gapstart;
681
682 pr_info("[mem %#010lx-%#010lx] available for PCI devices\n",
683 gapstart, gapstart + gapsize - 1);
684}
685
686/*
687 * Called late during init, in free_initmem().
688 *
689 * Initial e820_table and e820_table_kexec are largish __initdata arrays.
690 *
691 * Copy them to a (usually much smaller) dynamically allocated area that is
692 * sized precisely after the number of e820 entries.
693 *
694 * This is done after we've performed all the fixes and tweaks to the tables.
695 * All functions which modify them are __init functions, which won't exist
696 * after free_initmem().
697 */
698__init void e820__reallocate_tables(void)
699{
700 struct e820_table *n;
701 int size;
702
703 size = offsetof(struct e820_table, entries) + sizeof(struct e820_entry)*e820_table->nr_entries;
704 n = kmemdup(e820_table, size, GFP_KERNEL);
705 BUG_ON(!n);
706 e820_table = n;
707
708 size = offsetof(struct e820_table, entries) + sizeof(struct e820_entry)*e820_table_kexec->nr_entries;
709 n = kmemdup(e820_table_kexec, size, GFP_KERNEL);
710 BUG_ON(!n);
711 e820_table_kexec = n;
712
713 size = offsetof(struct e820_table, entries) + sizeof(struct e820_entry)*e820_table_firmware->nr_entries;
714 n = kmemdup(e820_table_firmware, size, GFP_KERNEL);
715 BUG_ON(!n);
716 e820_table_firmware = n;
717}
718
719/*
720 * Because of the small fixed size of struct boot_params, only the first
721 * 128 E820 memory entries are passed to the kernel via boot_params.e820_table,
722 * the remaining (if any) entries are passed via the SETUP_E820_EXT node of
723 * struct setup_data, which is parsed here.
724 */
725void __init e820__memory_setup_extended(u64 phys_addr, u32 data_len)
726{
727 int entries;
728 struct boot_e820_entry *extmap;
729 struct setup_data *sdata;
730
731 sdata = early_memremap(phys_addr, size: data_len);
732 entries = sdata->len / sizeof(*extmap);
733 extmap = (struct boot_e820_entry *)(sdata->data);
734
735 __append_e820_table(entries: extmap, nr_entries: entries);
736 e820__update_table(table: e820_table);
737
738 memcpy(e820_table_kexec, e820_table, sizeof(*e820_table_kexec));
739 memcpy(e820_table_firmware, e820_table, sizeof(*e820_table_firmware));
740
741 early_memunmap(addr: sdata, size: data_len);
742 pr_info("extended physical RAM map:\n");
743 e820__print_table(who: "extended");
744}
745
746/*
747 * Find the ranges of physical addresses that do not correspond to
748 * E820 RAM areas and register the corresponding pages as 'nosave' for
749 * hibernation (32-bit) or software suspend and suspend to RAM (64-bit).
750 *
751 * This function requires the E820 map to be sorted and without any
752 * overlapping entries.
753 */
754void __init e820__register_nosave_regions(unsigned long limit_pfn)
755{
756 int i;
757 u64 last_addr = 0;
758
759 for (i = 0; i < e820_table->nr_entries; i++) {
760 struct e820_entry *entry = &e820_table->entries[i];
761
762 if (entry->type != E820_TYPE_RAM)
763 continue;
764
765 if (last_addr < entry->addr)
766 register_nosave_region(PFN_DOWN(last_addr), PFN_UP(entry->addr));
767
768 last_addr = entry->addr + entry->size;
769 }
770
771 register_nosave_region(PFN_DOWN(last_addr), e: limit_pfn);
772}
773
774#ifdef CONFIG_ACPI
775/*
776 * Register ACPI NVS memory regions, so that we can save/restore them during
777 * hibernation and the subsequent resume:
778 */
779static int __init e820__register_nvs_regions(void)
780{
781 int i;
782
783 for (i = 0; i < e820_table->nr_entries; i++) {
784 struct e820_entry *entry = &e820_table->entries[i];
785
786 if (entry->type == E820_TYPE_NVS)
787 acpi_nvs_register(start: entry->addr, size: entry->size);
788 }
789
790 return 0;
791}
792core_initcall(e820__register_nvs_regions);
793#endif
794
795/*
796 * Allocate the requested number of bytes with the requested alignment
797 * and return (the physical address) to the caller. Also register this
798 * range in the 'kexec' E820 table as a reserved range.
799 *
800 * This allows kexec to fake a new mptable, as if it came from the real
801 * system.
802 */
803u64 __init e820__memblock_alloc_reserved(u64 size, u64 align)
804{
805 u64 addr;
806
807 addr = memblock_phys_alloc(size, align);
808 if (addr) {
809 e820__range_update_table(t: e820_table_kexec, start: addr, size, old_type: E820_TYPE_RAM, new_type: E820_TYPE_RESERVED);
810 pr_info("update e820_table_kexec for e820__memblock_alloc_reserved()\n");
811 e820__update_table_kexec();
812 }
813
814 return addr;
815}
816
817#ifdef CONFIG_X86_32
818# ifdef CONFIG_X86_PAE
819# define MAX_ARCH_PFN (1ULL<<(36-PAGE_SHIFT))
820# else
821# define MAX_ARCH_PFN (1ULL<<(32-PAGE_SHIFT))
822# endif
823#else /* CONFIG_X86_32 */
824# define MAX_ARCH_PFN MAXMEM>>PAGE_SHIFT
825#endif
826
827/*
828 * Find the highest page frame number we have available
829 */
830static unsigned long __init e820__end_ram_pfn(unsigned long limit_pfn)
831{
832 int i;
833 unsigned long last_pfn = 0;
834 unsigned long max_arch_pfn = MAX_ARCH_PFN;
835
836 for (i = 0; i < e820_table->nr_entries; i++) {
837 struct e820_entry *entry = &e820_table->entries[i];
838 unsigned long start_pfn;
839 unsigned long end_pfn;
840
841 if (entry->type != E820_TYPE_RAM &&
842 entry->type != E820_TYPE_ACPI)
843 continue;
844
845 start_pfn = entry->addr >> PAGE_SHIFT;
846 end_pfn = (entry->addr + entry->size) >> PAGE_SHIFT;
847
848 if (start_pfn >= limit_pfn)
849 continue;
850 if (end_pfn > limit_pfn) {
851 last_pfn = limit_pfn;
852 break;
853 }
854 if (end_pfn > last_pfn)
855 last_pfn = end_pfn;
856 }
857
858 if (last_pfn > max_arch_pfn)
859 last_pfn = max_arch_pfn;
860
861 pr_info("last_pfn = %#lx max_arch_pfn = %#lx\n",
862 last_pfn, max_arch_pfn);
863 return last_pfn;
864}
865
866unsigned long __init e820__end_of_ram_pfn(void)
867{
868 return e820__end_ram_pfn(MAX_ARCH_PFN);
869}
870
871unsigned long __init e820__end_of_low_ram_pfn(void)
872{
873 return e820__end_ram_pfn(limit_pfn: 1UL << (32 - PAGE_SHIFT));
874}
875
876static void __init early_panic(char *msg)
877{
878 early_printk(fmt: msg);
879 panic(fmt: msg);
880}
881
882static int userdef __initdata;
883
884/* The "mem=nopentium" boot option disables 4MB page tables on 32-bit kernels: */
885static int __init parse_memopt(char *p)
886{
887 u64 mem_size;
888
889 if (!p)
890 return -EINVAL;
891
892 if (!strcmp(p, "nopentium")) {
893#ifdef CONFIG_X86_32
894 setup_clear_cpu_cap(X86_FEATURE_PSE);
895 return 0;
896#else
897 pr_warn("mem=nopentium ignored! (only supported on x86_32)\n");
898 return -EINVAL;
899#endif
900 }
901
902 userdef = 1;
903 mem_size = memparse(ptr: p, retptr: &p);
904
905 /* Don't remove all memory when getting "mem={invalid}" parameter: */
906 if (mem_size == 0)
907 return -EINVAL;
908
909 e820__range_remove(start: mem_size, ULLONG_MAX - mem_size, old_type: E820_TYPE_RAM, check_type: 1);
910
911#ifdef CONFIG_MEMORY_HOTPLUG
912 max_mem_size = mem_size;
913#endif
914
915 return 0;
916}
917early_param("mem", parse_memopt);
918
919static int __init parse_memmap_one(char *p)
920{
921 char *oldp;
922 u64 start_at, mem_size;
923
924 if (!p)
925 return -EINVAL;
926
927 if (!strncmp(p, "exactmap", 8)) {
928 e820_table->nr_entries = 0;
929 userdef = 1;
930 return 0;
931 }
932
933 oldp = p;
934 mem_size = memparse(ptr: p, retptr: &p);
935 if (p == oldp)
936 return -EINVAL;
937
938 userdef = 1;
939 if (*p == '@') {
940 start_at = memparse(ptr: p+1, retptr: &p);
941 e820__range_add(start: start_at, size: mem_size, type: E820_TYPE_RAM);
942 } else if (*p == '#') {
943 start_at = memparse(ptr: p+1, retptr: &p);
944 e820__range_add(start: start_at, size: mem_size, type: E820_TYPE_ACPI);
945 } else if (*p == '$') {
946 start_at = memparse(ptr: p+1, retptr: &p);
947 e820__range_add(start: start_at, size: mem_size, type: E820_TYPE_RESERVED);
948 } else if (*p == '!') {
949 start_at = memparse(ptr: p+1, retptr: &p);
950 e820__range_add(start: start_at, size: mem_size, type: E820_TYPE_PRAM);
951 } else if (*p == '%') {
952 enum e820_type from = 0, to = 0;
953
954 start_at = memparse(ptr: p + 1, retptr: &p);
955 if (*p == '-')
956 from = simple_strtoull(p + 1, &p, 0);
957 if (*p == '+')
958 to = simple_strtoull(p + 1, &p, 0);
959 if (*p != '\0')
960 return -EINVAL;
961 if (from && to)
962 e820__range_update(start: start_at, size: mem_size, old_type: from, new_type: to);
963 else if (to)
964 e820__range_add(start: start_at, size: mem_size, type: to);
965 else if (from)
966 e820__range_remove(start: start_at, size: mem_size, old_type: from, check_type: 1);
967 else
968 e820__range_remove(start: start_at, size: mem_size, old_type: 0, check_type: 0);
969 } else {
970 e820__range_remove(start: mem_size, ULLONG_MAX - mem_size, old_type: E820_TYPE_RAM, check_type: 1);
971 }
972
973 return *p == '\0' ? 0 : -EINVAL;
974}
975
976static int __init parse_memmap_opt(char *str)
977{
978 while (str) {
979 char *k = strchr(str, ',');
980
981 if (k)
982 *k++ = 0;
983
984 parse_memmap_one(p: str);
985 str = k;
986 }
987
988 return 0;
989}
990early_param("memmap", parse_memmap_opt);
991
992/*
993 * Called after parse_early_param(), after early parameters (such as mem=)
994 * have been processed, in which case we already have an E820 table filled in
995 * via the parameter callback function(s), but it's not sorted and printed yet:
996 */
997void __init e820__finish_early_params(void)
998{
999 if (userdef) {
1000 if (e820__update_table(table: e820_table) < 0)
1001 early_panic(msg: "Invalid user supplied memory map");
1002
1003 pr_info("user-defined physical RAM map:\n");
1004 e820__print_table(who: "user");
1005 }
1006}
1007
1008static const char *__init e820_type_to_string(struct e820_entry *entry)
1009{
1010 switch (entry->type) {
1011 case E820_TYPE_RAM: return "System RAM";
1012 case E820_TYPE_ACPI: return "ACPI Tables";
1013 case E820_TYPE_NVS: return "ACPI Non-volatile Storage";
1014 case E820_TYPE_UNUSABLE: return "Unusable memory";
1015 case E820_TYPE_PRAM: return "Persistent Memory (legacy)";
1016 case E820_TYPE_PMEM: return "Persistent Memory";
1017 case E820_TYPE_RESERVED: return "Reserved";
1018 case E820_TYPE_SOFT_RESERVED: return "Soft Reserved";
1019 default: return "Unknown E820 type";
1020 }
1021}
1022
1023static unsigned long __init e820_type_to_iomem_type(struct e820_entry *entry)
1024{
1025 switch (entry->type) {
1026 case E820_TYPE_RAM: return IORESOURCE_SYSTEM_RAM;
1027 case E820_TYPE_ACPI: /* Fall-through: */
1028 case E820_TYPE_NVS: /* Fall-through: */
1029 case E820_TYPE_UNUSABLE: /* Fall-through: */
1030 case E820_TYPE_PRAM: /* Fall-through: */
1031 case E820_TYPE_PMEM: /* Fall-through: */
1032 case E820_TYPE_RESERVED: /* Fall-through: */
1033 case E820_TYPE_SOFT_RESERVED: /* Fall-through: */
1034 default: return IORESOURCE_MEM;
1035 }
1036}
1037
1038static unsigned long __init e820_type_to_iores_desc(struct e820_entry *entry)
1039{
1040 switch (entry->type) {
1041 case E820_TYPE_ACPI: return IORES_DESC_ACPI_TABLES;
1042 case E820_TYPE_NVS: return IORES_DESC_ACPI_NV_STORAGE;
1043 case E820_TYPE_PMEM: return IORES_DESC_PERSISTENT_MEMORY;
1044 case E820_TYPE_PRAM: return IORES_DESC_PERSISTENT_MEMORY_LEGACY;
1045 case E820_TYPE_RESERVED: return IORES_DESC_RESERVED;
1046 case E820_TYPE_SOFT_RESERVED: return IORES_DESC_SOFT_RESERVED;
1047 case E820_TYPE_RAM: /* Fall-through: */
1048 case E820_TYPE_UNUSABLE: /* Fall-through: */
1049 default: return IORES_DESC_NONE;
1050 }
1051}
1052
1053static bool __init do_mark_busy(enum e820_type type, struct resource *res)
1054{
1055 /* this is the legacy bios/dos rom-shadow + mmio region */
1056 if (res->start < (1ULL<<20))
1057 return true;
1058
1059 /*
1060 * Treat persistent memory and other special memory ranges like
1061 * device memory, i.e. reserve it for exclusive use of a driver
1062 */
1063 switch (type) {
1064 case E820_TYPE_RESERVED:
1065 case E820_TYPE_SOFT_RESERVED:
1066 case E820_TYPE_PRAM:
1067 case E820_TYPE_PMEM:
1068 return false;
1069 case E820_TYPE_RAM:
1070 case E820_TYPE_ACPI:
1071 case E820_TYPE_NVS:
1072 case E820_TYPE_UNUSABLE:
1073 default:
1074 return true;
1075 }
1076}
1077
1078/*
1079 * Mark E820 reserved areas as busy for the resource manager:
1080 */
1081
1082static struct resource __initdata *e820_res;
1083
1084void __init e820__reserve_resources(void)
1085{
1086 int i;
1087 struct resource *res;
1088 u64 end;
1089
1090 res = memblock_alloc_or_panic(sizeof(*res) * e820_table->nr_entries,
1091 SMP_CACHE_BYTES);
1092 e820_res = res;
1093
1094 for (i = 0; i < e820_table->nr_entries; i++) {
1095 struct e820_entry *entry = e820_table->entries + i;
1096
1097 end = entry->addr + entry->size - 1;
1098 if (end != (resource_size_t)end) {
1099 res++;
1100 continue;
1101 }
1102 res->start = entry->addr;
1103 res->end = end;
1104 res->name = e820_type_to_string(entry);
1105 res->flags = e820_type_to_iomem_type(entry);
1106 res->desc = e820_type_to_iores_desc(entry);
1107
1108 /*
1109 * Don't register the region that could be conflicted with
1110 * PCI device BAR resources and insert them later in
1111 * pcibios_resource_survey():
1112 */
1113 if (do_mark_busy(type: entry->type, res)) {
1114 res->flags |= IORESOURCE_BUSY;
1115 insert_resource(parent: &iomem_resource, new: res);
1116 }
1117 res++;
1118 }
1119
1120 /* Expose the kexec e820 table to the sysfs. */
1121 for (i = 0; i < e820_table_kexec->nr_entries; i++) {
1122 struct e820_entry *entry = e820_table_kexec->entries + i;
1123
1124 firmware_map_add_early(start: entry->addr, end: entry->addr + entry->size, type: e820_type_to_string(entry));
1125 }
1126}
1127
1128/*
1129 * How much should we pad the end of RAM, depending on where it is?
1130 */
1131static unsigned long __init ram_alignment(resource_size_t pos)
1132{
1133 unsigned long mb = pos >> 20;
1134
1135 /* To 64kB in the first megabyte */
1136 if (!mb)
1137 return 64*1024;
1138
1139 /* To 1MB in the first 16MB */
1140 if (mb < 16)
1141 return 1024*1024;
1142
1143 /* To 64MB for anything above that */
1144 return 64*1024*1024;
1145}
1146
1147#define MAX_RESOURCE_SIZE ((resource_size_t)-1)
1148
1149void __init e820__reserve_resources_late(void)
1150{
1151 int i;
1152 struct resource *res;
1153
1154 res = e820_res;
1155 for (i = 0; i < e820_table->nr_entries; i++) {
1156 if (!res->parent && res->end)
1157 insert_resource_expand_to_fit(root: &iomem_resource, new: res);
1158 res++;
1159 }
1160
1161 /*
1162 * Try to bump up RAM regions to reasonable boundaries, to
1163 * avoid stolen RAM:
1164 */
1165 for (i = 0; i < e820_table->nr_entries; i++) {
1166 struct e820_entry *entry = &e820_table->entries[i];
1167 u64 start, end;
1168
1169 if (entry->type != E820_TYPE_RAM)
1170 continue;
1171
1172 start = entry->addr + entry->size;
1173 end = round_up(start, ram_alignment(start)) - 1;
1174 if (end > MAX_RESOURCE_SIZE)
1175 end = MAX_RESOURCE_SIZE;
1176 if (start >= end)
1177 continue;
1178
1179 printk(KERN_DEBUG "e820: reserve RAM buffer [mem %#010llx-%#010llx]\n", start, end);
1180 reserve_region_with_split(root: &iomem_resource, start, end, name: "RAM buffer");
1181 }
1182}
1183
1184/*
1185 * Pass the firmware (bootloader) E820 map to the kernel and process it:
1186 */
1187char *__init e820__memory_setup_default(void)
1188{
1189 char *who = "BIOS-e820";
1190
1191 /*
1192 * Try to copy the BIOS-supplied E820-map.
1193 *
1194 * Otherwise fake a memory map; one section from 0k->640k,
1195 * the next section from 1mb->appropriate_mem_k
1196 */
1197 if (append_e820_table(entries: boot_params.e820_table, nr_entries: boot_params.e820_entries) < 0) {
1198 u64 mem_size;
1199
1200 /* Compare results from other methods and take the one that gives more RAM: */
1201 if (boot_params.alt_mem_k < boot_params.screen_info.ext_mem_k) {
1202 mem_size = boot_params.screen_info.ext_mem_k;
1203 who = "BIOS-88";
1204 } else {
1205 mem_size = boot_params.alt_mem_k;
1206 who = "BIOS-e801";
1207 }
1208
1209 e820_table->nr_entries = 0;
1210 e820__range_add(start: 0, LOWMEMSIZE(), type: E820_TYPE_RAM);
1211 e820__range_add(HIGH_MEMORY, size: mem_size << 10, type: E820_TYPE_RAM);
1212 }
1213
1214 /* We just appended a lot of ranges, sanitize the table: */
1215 e820__update_table(table: e820_table);
1216
1217 return who;
1218}
1219
1220/*
1221 * Calls e820__memory_setup_default() in essence to pick up the firmware/bootloader
1222 * E820 map - with an optional platform quirk available for virtual platforms
1223 * to override this method of boot environment processing:
1224 */
1225void __init e820__memory_setup(void)
1226{
1227 char *who;
1228
1229 /* This is a firmware interface ABI - make sure we don't break it: */
1230 BUILD_BUG_ON(sizeof(struct boot_e820_entry) != 20);
1231
1232 who = x86_init.resources.memory_setup();
1233
1234 memcpy(e820_table_kexec, e820_table, sizeof(*e820_table_kexec));
1235 memcpy(e820_table_firmware, e820_table, sizeof(*e820_table_firmware));
1236
1237 pr_info("BIOS-provided physical RAM map:\n");
1238 e820__print_table(who);
1239}
1240
1241void __init e820__memblock_setup(void)
1242{
1243 int i;
1244 u64 end;
1245
1246#ifdef CONFIG_MEMORY_HOTPLUG
1247 /*
1248 * Memory used by the kernel cannot be hot-removed because Linux
1249 * cannot migrate the kernel pages. When memory hotplug is
1250 * enabled, we should prevent memblock from allocating memory
1251 * for the kernel.
1252 *
1253 * ACPI SRAT records all hotpluggable memory ranges. But before
1254 * SRAT is parsed, we don't know about it.
1255 *
1256 * The kernel image is loaded into memory at very early time. We
1257 * cannot prevent this anyway. So on NUMA system, we set any
1258 * node the kernel resides in as un-hotpluggable.
1259 *
1260 * Since on modern servers, one node could have double-digit
1261 * gigabytes memory, we can assume the memory around the kernel
1262 * image is also un-hotpluggable. So before SRAT is parsed, just
1263 * allocate memory near the kernel image to try the best to keep
1264 * the kernel away from hotpluggable memory.
1265 */
1266 if (movable_node_is_enabled())
1267 memblock_set_bottom_up(enable: true);
1268#endif
1269
1270 /*
1271 * At this point only the first megabyte is mapped for sure, the
1272 * rest of the memory cannot be used for memblock resizing
1273 */
1274 memblock_set_current_limit(ISA_END_ADDRESS);
1275
1276 /*
1277 * The bootstrap memblock region count maximum is 128 entries
1278 * (INIT_MEMBLOCK_REGIONS), but EFI might pass us more E820 entries
1279 * than that - so allow memblock resizing.
1280 *
1281 * This is safe, because this call happens pretty late during x86 setup,
1282 * so we know about reserved memory regions already. (This is important
1283 * so that memblock resizing does no stomp over reserved areas.)
1284 */
1285 memblock_allow_resize();
1286
1287 for (i = 0; i < e820_table->nr_entries; i++) {
1288 struct e820_entry *entry = &e820_table->entries[i];
1289
1290 end = entry->addr + entry->size;
1291 if (end != (resource_size_t)end)
1292 continue;
1293
1294 if (entry->type == E820_TYPE_SOFT_RESERVED)
1295 memblock_reserve(base: entry->addr, size: entry->size);
1296
1297 if (entry->type != E820_TYPE_RAM)
1298 continue;
1299
1300 memblock_add(base: entry->addr, size: entry->size);
1301 }
1302
1303 /*
1304 * At this point memblock is only allowed to allocate from memory
1305 * below 1M (aka ISA_END_ADDRESS) up until direct map is completely set
1306 * up in init_mem_mapping().
1307 *
1308 * KHO kernels are special and use only scratch memory for memblock
1309 * allocations, but memory below 1M is ignored by kernel after early
1310 * boot and cannot be naturally marked as scratch.
1311 *
1312 * To allow allocation of the real-mode trampoline and a few (if any)
1313 * other very early allocations from below 1M forcibly mark the memory
1314 * below 1M as scratch.
1315 *
1316 * After real mode trampoline is allocated, we clear that scratch
1317 * marking.
1318 */
1319 memblock_mark_kho_scratch(base: 0, SZ_1M);
1320
1321 /*
1322 * 32-bit systems are limited to 4BG of memory even with HIGHMEM and
1323 * to even less without it.
1324 * Discard memory after max_pfn - the actual limit detected at runtime.
1325 */
1326 if (IS_ENABLED(CONFIG_X86_32))
1327 memblock_remove(PFN_PHYS(max_pfn), size: -1);
1328
1329 /* Throw away partial pages: */
1330 memblock_trim_memory(PAGE_SIZE);
1331
1332 memblock_dump_all();
1333}
1334

source code of linux/arch/x86/kernel/e820.c