1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Bootmem core functions. |
4 | * |
5 | * Copyright (c) 2020, Bytedance. |
6 | * |
7 | * Author: Muchun Song <songmuchun@bytedance.com> |
8 | * |
9 | */ |
10 | #include <linux/mm.h> |
11 | #include <linux/compiler.h> |
12 | #include <linux/memblock.h> |
13 | #include <linux/bootmem_info.h> |
14 | #include <linux/memory_hotplug.h> |
15 | #include <linux/kmemleak.h> |
16 | |
17 | void get_page_bootmem(unsigned long info, struct page *page, unsigned long type) |
18 | { |
19 | page->index = type; |
20 | SetPagePrivate(page); |
21 | set_page_private(page, private: info); |
22 | page_ref_inc(page); |
23 | } |
24 | |
25 | void put_page_bootmem(struct page *page) |
26 | { |
27 | unsigned long type = page->index; |
28 | |
29 | BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE || |
30 | type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE); |
31 | |
32 | if (page_ref_dec_return(page) == 1) { |
33 | page->index = 0; |
34 | ClearPagePrivate(page); |
35 | set_page_private(page, private: 0); |
36 | INIT_LIST_HEAD(list: &page->lru); |
37 | kmemleak_free_part_phys(PFN_PHYS(page_to_pfn(page)), PAGE_SIZE); |
38 | free_reserved_page(page); |
39 | } |
40 | } |
41 | |
42 | #ifndef CONFIG_SPARSEMEM_VMEMMAP |
43 | static void __init register_page_bootmem_info_section(unsigned long start_pfn) |
44 | { |
45 | unsigned long mapsize, section_nr, i; |
46 | struct mem_section *ms; |
47 | struct page *page, *memmap; |
48 | struct mem_section_usage *usage; |
49 | |
50 | section_nr = pfn_to_section_nr(start_pfn); |
51 | ms = __nr_to_section(section_nr); |
52 | |
53 | /* Get section's memmap address */ |
54 | memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr); |
55 | |
56 | /* |
57 | * Get page for the memmap's phys address |
58 | * XXX: need more consideration for sparse_vmemmap... |
59 | */ |
60 | page = virt_to_page(memmap); |
61 | mapsize = sizeof(struct page) * PAGES_PER_SECTION; |
62 | mapsize = PAGE_ALIGN(mapsize) >> PAGE_SHIFT; |
63 | |
64 | /* remember memmap's page */ |
65 | for (i = 0; i < mapsize; i++, page++) |
66 | get_page_bootmem(section_nr, page, SECTION_INFO); |
67 | |
68 | usage = ms->usage; |
69 | page = virt_to_page(usage); |
70 | |
71 | mapsize = PAGE_ALIGN(mem_section_usage_size()) >> PAGE_SHIFT; |
72 | |
73 | for (i = 0; i < mapsize; i++, page++) |
74 | get_page_bootmem(section_nr, page, MIX_SECTION_INFO); |
75 | |
76 | } |
77 | #else /* CONFIG_SPARSEMEM_VMEMMAP */ |
78 | static void __init register_page_bootmem_info_section(unsigned long start_pfn) |
79 | { |
80 | unsigned long mapsize, section_nr, i; |
81 | struct mem_section *ms; |
82 | struct page *page, *memmap; |
83 | struct mem_section_usage *usage; |
84 | |
85 | section_nr = pfn_to_section_nr(pfn: start_pfn); |
86 | ms = __nr_to_section(nr: section_nr); |
87 | |
88 | memmap = sparse_decode_mem_map(coded_mem_map: ms->section_mem_map, pnum: section_nr); |
89 | |
90 | register_page_bootmem_memmap(section_nr, map: memmap, PAGES_PER_SECTION); |
91 | |
92 | usage = ms->usage; |
93 | page = virt_to_page(usage); |
94 | |
95 | mapsize = PAGE_ALIGN(mem_section_usage_size()) >> PAGE_SHIFT; |
96 | |
97 | for (i = 0; i < mapsize; i++, page++) |
98 | get_page_bootmem(info: section_nr, page, type: MIX_SECTION_INFO); |
99 | } |
100 | #endif /* !CONFIG_SPARSEMEM_VMEMMAP */ |
101 | |
102 | void __init register_page_bootmem_info_node(struct pglist_data *pgdat) |
103 | { |
104 | unsigned long i, pfn, end_pfn, nr_pages; |
105 | int node = pgdat->node_id; |
106 | struct page *page; |
107 | |
108 | nr_pages = PAGE_ALIGN(sizeof(struct pglist_data)) >> PAGE_SHIFT; |
109 | page = virt_to_page(pgdat); |
110 | |
111 | for (i = 0; i < nr_pages; i++, page++) |
112 | get_page_bootmem(info: node, page, type: NODE_INFO); |
113 | |
114 | pfn = pgdat->node_start_pfn; |
115 | end_pfn = pgdat_end_pfn(pgdat); |
116 | |
117 | /* register section info */ |
118 | for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) { |
119 | /* |
120 | * Some platforms can assign the same pfn to multiple nodes - on |
121 | * node0 as well as nodeN. To avoid registering a pfn against |
122 | * multiple nodes we check that this pfn does not already |
123 | * reside in some other nodes. |
124 | */ |
125 | if (pfn_valid(pfn) && (early_pfn_to_nid(pfn) == node)) |
126 | register_page_bootmem_info_section(start_pfn: pfn); |
127 | } |
128 | } |
129 | |