1 | // SPDX-License-Identifier: GPL-2.0 |
2 | #include <linux/processor.h> |
3 | #include <linux/errno.h> |
4 | #include <linux/init.h> |
5 | #include <asm/physmem_info.h> |
6 | #include <asm/stacktrace.h> |
7 | #include <asm/boot_data.h> |
8 | #include <asm/sparsemem.h> |
9 | #include <asm/sections.h> |
10 | #include <asm/setup.h> |
11 | #include <asm/sclp.h> |
12 | #include <asm/uv.h> |
13 | #include "decompressor.h" |
14 | #include "boot.h" |
15 | |
16 | struct physmem_info __bootdata(physmem_info); |
17 | static unsigned int physmem_alloc_ranges; |
18 | static unsigned long physmem_alloc_pos; |
19 | |
20 | /* up to 256 storage elements, 1020 subincrements each */ |
21 | #define ENTRIES_EXTENDED_MAX \ |
22 | (256 * (1020 / 2) * sizeof(struct physmem_range)) |
23 | |
24 | static struct physmem_range *__get_physmem_range_ptr(u32 n) |
25 | { |
26 | if (n < MEM_INLINED_ENTRIES) |
27 | return &physmem_info.online[n]; |
28 | if (unlikely(!physmem_info.online_extended)) { |
29 | physmem_info.online_extended = (struct physmem_range *)physmem_alloc_range( |
30 | RR_MEM_DETECT_EXTENDED, ENTRIES_EXTENDED_MAX, sizeof(long), 0, |
31 | physmem_alloc_pos, true); |
32 | } |
33 | return &physmem_info.online_extended[n - MEM_INLINED_ENTRIES]; |
34 | } |
35 | |
36 | /* |
37 | * sequential calls to add_physmem_online_range with adjacent memory ranges |
38 | * are merged together into single memory range. |
39 | */ |
40 | void add_physmem_online_range(u64 start, u64 end) |
41 | { |
42 | struct physmem_range *range; |
43 | |
44 | if (physmem_info.range_count) { |
45 | range = __get_physmem_range_ptr(n: physmem_info.range_count - 1); |
46 | if (range->end == start) { |
47 | range->end = end; |
48 | return; |
49 | } |
50 | } |
51 | |
52 | range = __get_physmem_range_ptr(n: physmem_info.range_count); |
53 | range->start = start; |
54 | range->end = end; |
55 | physmem_info.range_count++; |
56 | } |
57 | |
58 | static int __diag260(unsigned long rx1, unsigned long rx2) |
59 | { |
60 | unsigned long reg1, reg2, ry; |
61 | union register_pair rx; |
62 | psw_t old; |
63 | int rc; |
64 | |
65 | rx.even = rx1; |
66 | rx.odd = rx2; |
67 | ry = 0x10; /* storage configuration */ |
68 | rc = -1; /* fail */ |
69 | asm volatile( |
70 | " mvc 0(16,%[psw_old]),0(%[psw_pgm])\n" |
71 | " epsw %[reg1],%[reg2]\n" |
72 | " st %[reg1],0(%[psw_pgm])\n" |
73 | " st %[reg2],4(%[psw_pgm])\n" |
74 | " larl %[reg1],1f\n" |
75 | " stg %[reg1],8(%[psw_pgm])\n" |
76 | " diag %[rx],%[ry],0x260\n" |
77 | " ipm %[rc]\n" |
78 | " srl %[rc],28\n" |
79 | "1: mvc 0(16,%[psw_pgm]),0(%[psw_old])\n" |
80 | : [reg1] "=&d" (reg1), |
81 | [reg2] "=&a" (reg2), |
82 | [rc] "+&d" (rc), |
83 | [ry] "+&d" (ry), |
84 | "+Q" (S390_lowcore.program_new_psw), |
85 | "=Q" (old) |
86 | : [rx] "d" (rx.pair), |
87 | [psw_old] "a" (&old), |
88 | [psw_pgm] "a" (&S390_lowcore.program_new_psw) |
89 | : "cc" , "memory" ); |
90 | return rc == 0 ? ry : -1; |
91 | } |
92 | |
93 | static int diag260(void) |
94 | { |
95 | int rc, i; |
96 | |
97 | struct { |
98 | unsigned long start; |
99 | unsigned long end; |
100 | } storage_extents[8] __aligned(16); /* VM supports up to 8 extends */ |
101 | |
102 | memset(storage_extents, 0, sizeof(storage_extents)); |
103 | rc = __diag260(rx1: (unsigned long)storage_extents, rx2: sizeof(storage_extents)); |
104 | if (rc == -1) |
105 | return -1; |
106 | |
107 | for (i = 0; i < min_t(int, rc, ARRAY_SIZE(storage_extents)); i++) |
108 | add_physmem_online_range(start: storage_extents[i].start, end: storage_extents[i].end + 1); |
109 | return 0; |
110 | } |
111 | |
112 | static int tprot(unsigned long addr) |
113 | { |
114 | unsigned long reg1, reg2; |
115 | int rc = -EFAULT; |
116 | psw_t old; |
117 | |
118 | asm volatile( |
119 | " mvc 0(16,%[psw_old]),0(%[psw_pgm])\n" |
120 | " epsw %[reg1],%[reg2]\n" |
121 | " st %[reg1],0(%[psw_pgm])\n" |
122 | " st %[reg2],4(%[psw_pgm])\n" |
123 | " larl %[reg1],1f\n" |
124 | " stg %[reg1],8(%[psw_pgm])\n" |
125 | " tprot 0(%[addr]),0\n" |
126 | " ipm %[rc]\n" |
127 | " srl %[rc],28\n" |
128 | "1: mvc 0(16,%[psw_pgm]),0(%[psw_old])\n" |
129 | : [reg1] "=&d" (reg1), |
130 | [reg2] "=&a" (reg2), |
131 | [rc] "+&d" (rc), |
132 | "=Q" (S390_lowcore.program_new_psw.addr), |
133 | "=Q" (old) |
134 | : [psw_old] "a" (&old), |
135 | [psw_pgm] "a" (&S390_lowcore.program_new_psw), |
136 | [addr] "a" (addr) |
137 | : "cc" , "memory" ); |
138 | return rc; |
139 | } |
140 | |
141 | static unsigned long search_mem_end(void) |
142 | { |
143 | unsigned long range = 1 << (MAX_PHYSMEM_BITS - 20); /* in 1MB blocks */ |
144 | unsigned long offset = 0; |
145 | unsigned long pivot; |
146 | |
147 | while (range > 1) { |
148 | range >>= 1; |
149 | pivot = offset + range; |
150 | if (!tprot(addr: pivot << 20)) |
151 | offset = pivot; |
152 | } |
153 | return (offset + 1) << 20; |
154 | } |
155 | |
156 | unsigned long detect_max_physmem_end(void) |
157 | { |
158 | unsigned long max_physmem_end = 0; |
159 | |
160 | if (!sclp_early_get_memsize(&max_physmem_end)) { |
161 | physmem_info.info_source = MEM_DETECT_SCLP_READ_INFO; |
162 | } else { |
163 | max_physmem_end = search_mem_end(); |
164 | physmem_info.info_source = MEM_DETECT_BIN_SEARCH; |
165 | } |
166 | return max_physmem_end; |
167 | } |
168 | |
169 | void detect_physmem_online_ranges(unsigned long max_physmem_end) |
170 | { |
171 | if (!sclp_early_read_storage_info()) { |
172 | physmem_info.info_source = MEM_DETECT_SCLP_STOR_INFO; |
173 | } else if (!diag260()) { |
174 | physmem_info.info_source = MEM_DETECT_DIAG260; |
175 | } else if (max_physmem_end) { |
176 | add_physmem_online_range(start: 0, end: max_physmem_end); |
177 | } |
178 | } |
179 | |
180 | void physmem_set_usable_limit(unsigned long limit) |
181 | { |
182 | physmem_info.usable = limit; |
183 | physmem_alloc_pos = limit; |
184 | } |
185 | |
186 | static void die_oom(unsigned long size, unsigned long align, unsigned long min, unsigned long max) |
187 | { |
188 | unsigned long start, end, total_mem = 0, total_reserved_mem = 0; |
189 | struct reserved_range *range; |
190 | enum reserved_range_type t; |
191 | int i; |
192 | |
193 | decompressor_printk(fmt: "Linux version %s\n" , kernel_version); |
194 | if (!is_prot_virt_guest() && early_command_line[0]) |
195 | decompressor_printk(fmt: "Kernel command line: %s\n" , early_command_line); |
196 | decompressor_printk(fmt: "Out of memory allocating %lx bytes %lx aligned in range %lx:%lx\n" , |
197 | size, align, min, max); |
198 | decompressor_printk(fmt: "Reserved memory ranges:\n" ); |
199 | for_each_physmem_reserved_range(t, range, &start, &end) { |
200 | decompressor_printk(fmt: "%016lx %016lx %s\n" , start, end, get_rr_type_name(t)); |
201 | total_reserved_mem += end - start; |
202 | } |
203 | decompressor_printk(fmt: "Usable online memory ranges (info source: %s [%x]):\n" , |
204 | get_physmem_info_source(), physmem_info.info_source); |
205 | for_each_physmem_usable_range(i, &start, &end) { |
206 | decompressor_printk(fmt: "%016lx %016lx\n" , start, end); |
207 | total_mem += end - start; |
208 | } |
209 | decompressor_printk(fmt: "Usable online memory total: %lx Reserved: %lx Free: %lx\n" , |
210 | total_mem, total_reserved_mem, |
211 | total_mem > total_reserved_mem ? total_mem - total_reserved_mem : 0); |
212 | print_stacktrace(sp: current_frame_address()); |
213 | sclp_early_printk("\n\n -- System halted\n" ); |
214 | disabled_wait(); |
215 | } |
216 | |
217 | void physmem_reserve(enum reserved_range_type type, unsigned long addr, unsigned long size) |
218 | { |
219 | physmem_info.reserved[type].start = addr; |
220 | physmem_info.reserved[type].end = addr + size; |
221 | } |
222 | |
223 | void physmem_free(enum reserved_range_type type) |
224 | { |
225 | physmem_info.reserved[type].start = 0; |
226 | physmem_info.reserved[type].end = 0; |
227 | } |
228 | |
229 | static bool __physmem_alloc_intersects(unsigned long addr, unsigned long size, |
230 | unsigned long *intersection_start) |
231 | { |
232 | unsigned long res_addr, res_size; |
233 | int t; |
234 | |
235 | for (t = 0; t < RR_MAX; t++) { |
236 | if (!get_physmem_reserved(t, &res_addr, &res_size)) |
237 | continue; |
238 | if (intersects(addr0: addr, size0: size, addr1: res_addr, size1: res_size)) { |
239 | *intersection_start = res_addr; |
240 | return true; |
241 | } |
242 | } |
243 | return ipl_report_certs_intersects(addr, size, intersection_start); |
244 | } |
245 | |
246 | static unsigned long __physmem_alloc_range(unsigned long size, unsigned long align, |
247 | unsigned long min, unsigned long max, |
248 | unsigned int from_ranges, unsigned int *ranges_left, |
249 | bool die_on_oom) |
250 | { |
251 | unsigned int nranges = from_ranges ?: physmem_info.range_count; |
252 | unsigned long range_start, range_end; |
253 | unsigned long intersection_start; |
254 | unsigned long addr, pos = max; |
255 | |
256 | align = max(align, 8UL); |
257 | while (nranges) { |
258 | __get_physmem_range(nranges - 1, &range_start, &range_end, false); |
259 | pos = min(range_end, pos); |
260 | |
261 | if (round_up(min, align) + size > pos) |
262 | break; |
263 | addr = round_down(pos - size, align); |
264 | if (range_start > addr) { |
265 | nranges--; |
266 | continue; |
267 | } |
268 | if (__physmem_alloc_intersects(addr, size, intersection_start: &intersection_start)) { |
269 | pos = intersection_start; |
270 | continue; |
271 | } |
272 | |
273 | if (ranges_left) |
274 | *ranges_left = nranges; |
275 | return addr; |
276 | } |
277 | if (die_on_oom) |
278 | die_oom(size, align, min, max); |
279 | return 0; |
280 | } |
281 | |
282 | unsigned long physmem_alloc_range(enum reserved_range_type type, unsigned long size, |
283 | unsigned long align, unsigned long min, unsigned long max, |
284 | bool die_on_oom) |
285 | { |
286 | unsigned long addr; |
287 | |
288 | max = min(max, physmem_alloc_pos); |
289 | addr = __physmem_alloc_range(size, align, min, max, from_ranges: 0, NULL, die_on_oom); |
290 | if (addr) |
291 | physmem_reserve(type: type, addr, size); |
292 | return addr; |
293 | } |
294 | |
295 | unsigned long physmem_alloc_top_down(enum reserved_range_type type, unsigned long size, |
296 | unsigned long align) |
297 | { |
298 | struct reserved_range *range = &physmem_info.reserved[type]; |
299 | struct reserved_range *new_range; |
300 | unsigned int ranges_left; |
301 | unsigned long addr; |
302 | |
303 | addr = __physmem_alloc_range(size, align, min: 0, max: physmem_alloc_pos, from_ranges: physmem_alloc_ranges, |
304 | ranges_left: &ranges_left, die_on_oom: true); |
305 | /* if not a consecutive allocation of the same type or first allocation */ |
306 | if (range->start != addr + size) { |
307 | if (range->end) { |
308 | physmem_alloc_pos = __physmem_alloc_range( |
309 | sizeof(struct reserved_range), 0, 0, physmem_alloc_pos, |
310 | physmem_alloc_ranges, &ranges_left, true); |
311 | new_range = (struct reserved_range *)physmem_alloc_pos; |
312 | *new_range = *range; |
313 | range->chain = new_range; |
314 | addr = __physmem_alloc_range(size, align, min: 0, max: physmem_alloc_pos, |
315 | from_ranges: ranges_left, ranges_left: &ranges_left, die_on_oom: true); |
316 | } |
317 | range->end = addr + size; |
318 | } |
319 | range->start = addr; |
320 | physmem_alloc_pos = addr; |
321 | physmem_alloc_ranges = ranges_left; |
322 | return addr; |
323 | } |
324 | |
325 | unsigned long get_physmem_alloc_pos(void) |
326 | { |
327 | return physmem_alloc_pos; |
328 | } |
329 | |