1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* |
3 | * Firmware Assisted dump: A robust mechanism to get reliable kernel crash |
4 | * dump with assistance from firmware. This approach does not use kexec, |
5 | * instead firmware assists in booting the kdump kernel while preserving |
6 | * memory contents. The most of the code implementation has been adapted |
7 | * from phyp assisted dump implementation written by Linas Vepstas and |
8 | * Manish Ahuja |
9 | * |
10 | * Copyright 2011 IBM Corporation |
11 | * Author: Mahesh Salgaonkar <mahesh@linux.vnet.ibm.com> |
12 | */ |
13 | |
14 | #undef DEBUG |
15 | #define pr_fmt(fmt) "fadump: " fmt |
16 | |
17 | #include <linux/string.h> |
18 | #include <linux/memblock.h> |
19 | #include <linux/delay.h> |
20 | #include <linux/seq_file.h> |
21 | #include <linux/crash_dump.h> |
22 | #include <linux/kobject.h> |
23 | #include <linux/sysfs.h> |
24 | #include <linux/slab.h> |
25 | #include <linux/cma.h> |
26 | #include <linux/hugetlb.h> |
27 | #include <linux/debugfs.h> |
28 | #include <linux/of.h> |
29 | #include <linux/of_fdt.h> |
30 | |
31 | #include <asm/page.h> |
32 | #include <asm/fadump.h> |
33 | #include <asm/fadump-internal.h> |
34 | #include <asm/setup.h> |
35 | #include <asm/interrupt.h> |
36 | |
37 | /* |
38 | * The CPU who acquired the lock to trigger the fadump crash should |
39 | * wait for other CPUs to enter. |
40 | * |
41 | * The timeout is in milliseconds. |
42 | */ |
43 | #define CRASH_TIMEOUT 500 |
44 | |
45 | static struct fw_dump fw_dump; |
46 | |
47 | static void __init fadump_reserve_crash_area(u64 base); |
48 | |
49 | #ifndef CONFIG_PRESERVE_FA_DUMP |
50 | |
51 | static struct kobject *fadump_kobj; |
52 | |
53 | static atomic_t cpus_in_fadump; |
54 | static DEFINE_MUTEX(fadump_mutex); |
55 | |
56 | static struct fadump_mrange_info crash_mrange_info = { "crash" , NULL, 0, 0, 0, false }; |
57 | |
58 | #define RESERVED_RNGS_SZ 16384 /* 16K - 128 entries */ |
59 | #define RESERVED_RNGS_CNT (RESERVED_RNGS_SZ / \ |
60 | sizeof(struct fadump_memory_range)) |
61 | static struct fadump_memory_range rngs[RESERVED_RNGS_CNT]; |
62 | static struct fadump_mrange_info |
63 | reserved_mrange_info = { "reserved" , rngs, RESERVED_RNGS_SZ, 0, RESERVED_RNGS_CNT, true }; |
64 | |
65 | static void __init early_init_dt_scan_reserved_ranges(unsigned long node); |
66 | |
67 | #ifdef CONFIG_CMA |
68 | static struct cma *fadump_cma; |
69 | |
70 | /* |
71 | * fadump_cma_init() - Initialize CMA area from a fadump reserved memory |
72 | * |
73 | * This function initializes CMA area from fadump reserved memory. |
74 | * The total size of fadump reserved memory covers for boot memory size |
75 | * + cpu data size + hpte size and metadata. |
76 | * Initialize only the area equivalent to boot memory size for CMA use. |
77 | * The remaining portion of fadump reserved memory will be not given |
78 | * to CMA and pages for those will stay reserved. boot memory size is |
79 | * aligned per CMA requirement to satisy cma_init_reserved_mem() call. |
80 | * But for some reason even if it fails we still have the memory reservation |
81 | * with us and we can still continue doing fadump. |
82 | */ |
83 | static int __init fadump_cma_init(void) |
84 | { |
85 | unsigned long long base, size; |
86 | int rc; |
87 | |
88 | if (!fw_dump.fadump_enabled) |
89 | return 0; |
90 | |
91 | /* |
92 | * Do not use CMA if user has provided fadump=nocma kernel parameter. |
93 | * Return 1 to continue with fadump old behaviour. |
94 | */ |
95 | if (fw_dump.nocma) |
96 | return 1; |
97 | |
98 | base = fw_dump.reserve_dump_area_start; |
99 | size = fw_dump.boot_memory_size; |
100 | |
101 | if (!size) |
102 | return 0; |
103 | |
104 | rc = cma_init_reserved_mem(base, size, order_per_bit: 0, name: "fadump_cma" , res_cma: &fadump_cma); |
105 | if (rc) { |
106 | pr_err("Failed to init cma area for firmware-assisted dump,%d\n" , rc); |
107 | /* |
108 | * Though the CMA init has failed we still have memory |
109 | * reservation with us. The reserved memory will be |
110 | * blocked from production system usage. Hence return 1, |
111 | * so that we can continue with fadump. |
112 | */ |
113 | return 1; |
114 | } |
115 | |
116 | /* |
117 | * If CMA activation fails, keep the pages reserved, instead of |
118 | * exposing them to buddy allocator. Same as 'fadump=nocma' case. |
119 | */ |
120 | cma_reserve_pages_on_error(cma: fadump_cma); |
121 | |
122 | /* |
123 | * So we now have successfully initialized cma area for fadump. |
124 | */ |
125 | pr_info("Initialized 0x%lx bytes cma area at %ldMB from 0x%lx " |
126 | "bytes of memory reserved for firmware-assisted dump\n" , |
127 | cma_get_size(fadump_cma), |
128 | (unsigned long)cma_get_base(fadump_cma) >> 20, |
129 | fw_dump.reserve_dump_area_size); |
130 | return 1; |
131 | } |
132 | #else |
133 | static int __init fadump_cma_init(void) { return 1; } |
134 | #endif /* CONFIG_CMA */ |
135 | |
136 | /* Scan the Firmware Assisted dump configuration details. */ |
137 | int __init early_init_dt_scan_fw_dump(unsigned long node, const char *uname, |
138 | int depth, void *data) |
139 | { |
140 | if (depth == 0) { |
141 | early_init_dt_scan_reserved_ranges(node); |
142 | return 0; |
143 | } |
144 | |
145 | if (depth != 1) |
146 | return 0; |
147 | |
148 | if (strcmp(uname, "rtas" ) == 0) { |
149 | rtas_fadump_dt_scan(&fw_dump, node); |
150 | return 1; |
151 | } |
152 | |
153 | if (strcmp(uname, "ibm,opal" ) == 0) { |
154 | opal_fadump_dt_scan(&fw_dump, node); |
155 | return 1; |
156 | } |
157 | |
158 | return 0; |
159 | } |
160 | |
161 | /* |
162 | * If fadump is registered, check if the memory provided |
163 | * falls within boot memory area and reserved memory area. |
164 | */ |
165 | int is_fadump_memory_area(u64 addr, unsigned long size) |
166 | { |
167 | u64 d_start, d_end; |
168 | |
169 | if (!fw_dump.dump_registered) |
170 | return 0; |
171 | |
172 | if (!size) |
173 | return 0; |
174 | |
175 | d_start = fw_dump.reserve_dump_area_start; |
176 | d_end = d_start + fw_dump.reserve_dump_area_size; |
177 | if (((addr + size) > d_start) && (addr <= d_end)) |
178 | return 1; |
179 | |
180 | return (addr <= fw_dump.boot_mem_top); |
181 | } |
182 | |
183 | int should_fadump_crash(void) |
184 | { |
185 | if (!fw_dump.dump_registered || !fw_dump.fadumphdr_addr) |
186 | return 0; |
187 | return 1; |
188 | } |
189 | |
190 | int is_fadump_active(void) |
191 | { |
192 | return fw_dump.dump_active; |
193 | } |
194 | |
195 | /* |
196 | * Returns true, if there are no holes in memory area between d_start to d_end, |
197 | * false otherwise. |
198 | */ |
199 | static bool is_fadump_mem_area_contiguous(u64 d_start, u64 d_end) |
200 | { |
201 | phys_addr_t reg_start, reg_end; |
202 | bool ret = false; |
203 | u64 i, start, end; |
204 | |
205 | for_each_mem_range(i, ®_start, ®_end) { |
206 | start = max_t(u64, d_start, reg_start); |
207 | end = min_t(u64, d_end, reg_end); |
208 | if (d_start < end) { |
209 | /* Memory hole from d_start to start */ |
210 | if (start > d_start) |
211 | break; |
212 | |
213 | if (end == d_end) { |
214 | ret = true; |
215 | break; |
216 | } |
217 | |
218 | d_start = end + 1; |
219 | } |
220 | } |
221 | |
222 | return ret; |
223 | } |
224 | |
225 | /* |
226 | * Returns true, if there are no holes in boot memory area, |
227 | * false otherwise. |
228 | */ |
229 | bool is_fadump_boot_mem_contiguous(void) |
230 | { |
231 | unsigned long d_start, d_end; |
232 | bool ret = false; |
233 | int i; |
234 | |
235 | for (i = 0; i < fw_dump.boot_mem_regs_cnt; i++) { |
236 | d_start = fw_dump.boot_mem_addr[i]; |
237 | d_end = d_start + fw_dump.boot_mem_sz[i]; |
238 | |
239 | ret = is_fadump_mem_area_contiguous(d_start, d_end); |
240 | if (!ret) |
241 | break; |
242 | } |
243 | |
244 | return ret; |
245 | } |
246 | |
247 | /* |
248 | * Returns true, if there are no holes in reserved memory area, |
249 | * false otherwise. |
250 | */ |
251 | bool is_fadump_reserved_mem_contiguous(void) |
252 | { |
253 | u64 d_start, d_end; |
254 | |
255 | d_start = fw_dump.reserve_dump_area_start; |
256 | d_end = d_start + fw_dump.reserve_dump_area_size; |
257 | return is_fadump_mem_area_contiguous(d_start, d_end); |
258 | } |
259 | |
260 | /* Print firmware assisted dump configurations for debugging purpose. */ |
261 | static void __init fadump_show_config(void) |
262 | { |
263 | int i; |
264 | |
265 | pr_debug("Support for firmware-assisted dump (fadump): %s\n" , |
266 | (fw_dump.fadump_supported ? "present" : "no support" )); |
267 | |
268 | if (!fw_dump.fadump_supported) |
269 | return; |
270 | |
271 | pr_debug("Fadump enabled : %s\n" , |
272 | (fw_dump.fadump_enabled ? "yes" : "no" )); |
273 | pr_debug("Dump Active : %s\n" , |
274 | (fw_dump.dump_active ? "yes" : "no" )); |
275 | pr_debug("Dump section sizes:\n" ); |
276 | pr_debug(" CPU state data size: %lx\n" , fw_dump.cpu_state_data_size); |
277 | pr_debug(" HPTE region size : %lx\n" , fw_dump.hpte_region_size); |
278 | pr_debug(" Boot memory size : %lx\n" , fw_dump.boot_memory_size); |
279 | pr_debug(" Boot memory top : %llx\n" , fw_dump.boot_mem_top); |
280 | pr_debug("Boot memory regions cnt: %llx\n" , fw_dump.boot_mem_regs_cnt); |
281 | for (i = 0; i < fw_dump.boot_mem_regs_cnt; i++) { |
282 | pr_debug("[%03d] base = %llx, size = %llx\n" , i, |
283 | fw_dump.boot_mem_addr[i], fw_dump.boot_mem_sz[i]); |
284 | } |
285 | } |
286 | |
287 | /** |
288 | * fadump_calculate_reserve_size(): reserve variable boot area 5% of System RAM |
289 | * |
290 | * Function to find the largest memory size we need to reserve during early |
291 | * boot process. This will be the size of the memory that is required for a |
292 | * kernel to boot successfully. |
293 | * |
294 | * This function has been taken from phyp-assisted dump feature implementation. |
295 | * |
296 | * returns larger of 256MB or 5% rounded down to multiples of 256MB. |
297 | * |
298 | * TODO: Come up with better approach to find out more accurate memory size |
299 | * that is required for a kernel to boot successfully. |
300 | * |
301 | */ |
302 | static __init u64 fadump_calculate_reserve_size(void) |
303 | { |
304 | u64 base, size, bootmem_min; |
305 | int ret; |
306 | |
307 | if (fw_dump.reserve_bootvar) |
308 | pr_warn("'fadump_reserve_mem=' parameter is deprecated in favor of 'crashkernel=' parameter.\n" ); |
309 | |
310 | /* |
311 | * Check if the size is specified through crashkernel= cmdline |
312 | * option. If yes, then use that but ignore base as fadump reserves |
313 | * memory at a predefined offset. |
314 | */ |
315 | ret = parse_crashkernel(cmdline: boot_command_line, system_ram: memblock_phys_mem_size(), |
316 | crash_size: &size, crash_base: &base, NULL, NULL); |
317 | if (ret == 0 && size > 0) { |
318 | unsigned long max_size; |
319 | |
320 | if (fw_dump.reserve_bootvar) |
321 | pr_info("Using 'crashkernel=' parameter for memory reservation.\n" ); |
322 | |
323 | fw_dump.reserve_bootvar = (unsigned long)size; |
324 | |
325 | /* |
326 | * Adjust if the boot memory size specified is above |
327 | * the upper limit. |
328 | */ |
329 | max_size = memblock_phys_mem_size() / MAX_BOOT_MEM_RATIO; |
330 | if (fw_dump.reserve_bootvar > max_size) { |
331 | fw_dump.reserve_bootvar = max_size; |
332 | pr_info("Adjusted boot memory size to %luMB\n" , |
333 | (fw_dump.reserve_bootvar >> 20)); |
334 | } |
335 | |
336 | return fw_dump.reserve_bootvar; |
337 | } else if (fw_dump.reserve_bootvar) { |
338 | /* |
339 | * 'fadump_reserve_mem=' is being used to reserve memory |
340 | * for firmware-assisted dump. |
341 | */ |
342 | return fw_dump.reserve_bootvar; |
343 | } |
344 | |
345 | /* divide by 20 to get 5% of value */ |
346 | size = memblock_phys_mem_size() / 20; |
347 | |
348 | /* round it down in multiples of 256 */ |
349 | size = size & ~0x0FFFFFFFUL; |
350 | |
351 | /* Truncate to memory_limit. We don't want to over reserve the memory.*/ |
352 | if (memory_limit && size > memory_limit) |
353 | size = memory_limit; |
354 | |
355 | bootmem_min = fw_dump.ops->fadump_get_bootmem_min(); |
356 | return (size > bootmem_min ? size : bootmem_min); |
357 | } |
358 | |
359 | /* |
360 | * Calculate the total memory size required to be reserved for |
361 | * firmware-assisted dump registration. |
362 | */ |
363 | static unsigned long __init get_fadump_area_size(void) |
364 | { |
365 | unsigned long size = 0; |
366 | |
367 | size += fw_dump.cpu_state_data_size; |
368 | size += fw_dump.hpte_region_size; |
369 | /* |
370 | * Account for pagesize alignment of boot memory area destination address. |
371 | * This faciliates in mmap reading of first kernel's memory. |
372 | */ |
373 | size = PAGE_ALIGN(size); |
374 | size += fw_dump.boot_memory_size; |
375 | size += sizeof(struct fadump_crash_info_header); |
376 | size += sizeof(struct elfhdr); /* ELF core header.*/ |
377 | size += sizeof(struct elf_phdr); /* place holder for cpu notes */ |
378 | /* Program headers for crash memory regions. */ |
379 | size += sizeof(struct elf_phdr) * (memblock_num_regions(memory) + 2); |
380 | |
381 | size = PAGE_ALIGN(size); |
382 | |
383 | /* This is to hold kernel metadata on platforms that support it */ |
384 | size += (fw_dump.ops->fadump_get_metadata_size ? |
385 | fw_dump.ops->fadump_get_metadata_size() : 0); |
386 | return size; |
387 | } |
388 | |
389 | static int __init add_boot_mem_region(unsigned long rstart, |
390 | unsigned long rsize) |
391 | { |
392 | int i = fw_dump.boot_mem_regs_cnt++; |
393 | |
394 | if (fw_dump.boot_mem_regs_cnt > FADUMP_MAX_MEM_REGS) { |
395 | fw_dump.boot_mem_regs_cnt = FADUMP_MAX_MEM_REGS; |
396 | return 0; |
397 | } |
398 | |
399 | pr_debug("Added boot memory range[%d] [%#016lx-%#016lx)\n" , |
400 | i, rstart, (rstart + rsize)); |
401 | fw_dump.boot_mem_addr[i] = rstart; |
402 | fw_dump.boot_mem_sz[i] = rsize; |
403 | return 1; |
404 | } |
405 | |
406 | /* |
407 | * Firmware usually has a hard limit on the data it can copy per region. |
408 | * Honour that by splitting a memory range into multiple regions. |
409 | */ |
410 | static int __init add_boot_mem_regions(unsigned long mstart, |
411 | unsigned long msize) |
412 | { |
413 | unsigned long rstart, rsize, max_size; |
414 | int ret = 1; |
415 | |
416 | rstart = mstart; |
417 | max_size = fw_dump.max_copy_size ? fw_dump.max_copy_size : msize; |
418 | while (msize) { |
419 | if (msize > max_size) |
420 | rsize = max_size; |
421 | else |
422 | rsize = msize; |
423 | |
424 | ret = add_boot_mem_region(rstart, rsize); |
425 | if (!ret) |
426 | break; |
427 | |
428 | msize -= rsize; |
429 | rstart += rsize; |
430 | } |
431 | |
432 | return ret; |
433 | } |
434 | |
435 | static int __init fadump_get_boot_mem_regions(void) |
436 | { |
437 | unsigned long size, cur_size, hole_size, last_end; |
438 | unsigned long mem_size = fw_dump.boot_memory_size; |
439 | phys_addr_t reg_start, reg_end; |
440 | int ret = 1; |
441 | u64 i; |
442 | |
443 | fw_dump.boot_mem_regs_cnt = 0; |
444 | |
445 | last_end = 0; |
446 | hole_size = 0; |
447 | cur_size = 0; |
448 | for_each_mem_range(i, ®_start, ®_end) { |
449 | size = reg_end - reg_start; |
450 | hole_size += (reg_start - last_end); |
451 | |
452 | if ((cur_size + size) >= mem_size) { |
453 | size = (mem_size - cur_size); |
454 | ret = add_boot_mem_regions(mstart: reg_start, msize: size); |
455 | break; |
456 | } |
457 | |
458 | mem_size -= size; |
459 | cur_size += size; |
460 | ret = add_boot_mem_regions(mstart: reg_start, msize: size); |
461 | if (!ret) |
462 | break; |
463 | |
464 | last_end = reg_end; |
465 | } |
466 | fw_dump.boot_mem_top = PAGE_ALIGN(fw_dump.boot_memory_size + hole_size); |
467 | |
468 | return ret; |
469 | } |
470 | |
471 | /* |
472 | * Returns true, if the given range overlaps with reserved memory ranges |
473 | * starting at idx. Also, updates idx to index of overlapping memory range |
474 | * with the given memory range. |
475 | * False, otherwise. |
476 | */ |
477 | static bool __init overlaps_reserved_ranges(u64 base, u64 end, int *idx) |
478 | { |
479 | bool ret = false; |
480 | int i; |
481 | |
482 | for (i = *idx; i < reserved_mrange_info.mem_range_cnt; i++) { |
483 | u64 rbase = reserved_mrange_info.mem_ranges[i].base; |
484 | u64 rend = rbase + reserved_mrange_info.mem_ranges[i].size; |
485 | |
486 | if (end <= rbase) |
487 | break; |
488 | |
489 | if ((end > rbase) && (base < rend)) { |
490 | *idx = i; |
491 | ret = true; |
492 | break; |
493 | } |
494 | } |
495 | |
496 | return ret; |
497 | } |
498 | |
499 | /* |
500 | * Locate a suitable memory area to reserve memory for FADump. While at it, |
501 | * lookup reserved-ranges & avoid overlap with them, as they are used by F/W. |
502 | */ |
503 | static u64 __init fadump_locate_reserve_mem(u64 base, u64 size) |
504 | { |
505 | struct fadump_memory_range *mrngs; |
506 | phys_addr_t mstart, mend; |
507 | int idx = 0; |
508 | u64 i, ret = 0; |
509 | |
510 | mrngs = reserved_mrange_info.mem_ranges; |
511 | for_each_free_mem_range(i, NUMA_NO_NODE, MEMBLOCK_NONE, |
512 | &mstart, &mend, NULL) { |
513 | pr_debug("%llu) mstart: %llx, mend: %llx, base: %llx\n" , |
514 | i, mstart, mend, base); |
515 | |
516 | if (mstart > base) |
517 | base = PAGE_ALIGN(mstart); |
518 | |
519 | while ((mend > base) && ((mend - base) >= size)) { |
520 | if (!overlaps_reserved_ranges(base, end: base+size, idx: &idx)) { |
521 | ret = base; |
522 | goto out; |
523 | } |
524 | |
525 | base = mrngs[idx].base + mrngs[idx].size; |
526 | base = PAGE_ALIGN(base); |
527 | } |
528 | } |
529 | |
530 | out: |
531 | return ret; |
532 | } |
533 | |
534 | int __init fadump_reserve_mem(void) |
535 | { |
536 | u64 base, size, mem_boundary, bootmem_min; |
537 | int ret = 1; |
538 | |
539 | if (!fw_dump.fadump_enabled) |
540 | return 0; |
541 | |
542 | if (!fw_dump.fadump_supported) { |
543 | pr_info("Firmware-Assisted Dump is not supported on this hardware\n" ); |
544 | goto error_out; |
545 | } |
546 | |
547 | /* |
548 | * Initialize boot memory size |
549 | * If dump is active then we have already calculated the size during |
550 | * first kernel. |
551 | */ |
552 | if (!fw_dump.dump_active) { |
553 | fw_dump.boot_memory_size = |
554 | PAGE_ALIGN(fadump_calculate_reserve_size()); |
555 | #ifdef CONFIG_CMA |
556 | if (!fw_dump.nocma) { |
557 | fw_dump.boot_memory_size = |
558 | ALIGN(fw_dump.boot_memory_size, |
559 | CMA_MIN_ALIGNMENT_BYTES); |
560 | } |
561 | #endif |
562 | |
563 | bootmem_min = fw_dump.ops->fadump_get_bootmem_min(); |
564 | if (fw_dump.boot_memory_size < bootmem_min) { |
565 | pr_err("Can't enable fadump with boot memory size (0x%lx) less than 0x%llx\n" , |
566 | fw_dump.boot_memory_size, bootmem_min); |
567 | goto error_out; |
568 | } |
569 | |
570 | if (!fadump_get_boot_mem_regions()) { |
571 | pr_err("Too many holes in boot memory area to enable fadump\n" ); |
572 | goto error_out; |
573 | } |
574 | } |
575 | |
576 | /* |
577 | * Calculate the memory boundary. |
578 | * If memory_limit is less than actual memory boundary then reserve |
579 | * the memory for fadump beyond the memory_limit and adjust the |
580 | * memory_limit accordingly, so that the running kernel can run with |
581 | * specified memory_limit. |
582 | */ |
583 | if (memory_limit && memory_limit < memblock_end_of_DRAM()) { |
584 | size = get_fadump_area_size(); |
585 | if ((memory_limit + size) < memblock_end_of_DRAM()) |
586 | memory_limit += size; |
587 | else |
588 | memory_limit = memblock_end_of_DRAM(); |
589 | printk(KERN_INFO "Adjusted memory_limit for firmware-assisted" |
590 | " dump, now %#016llx\n" , memory_limit); |
591 | } |
592 | if (memory_limit) |
593 | mem_boundary = memory_limit; |
594 | else |
595 | mem_boundary = memblock_end_of_DRAM(); |
596 | |
597 | base = fw_dump.boot_mem_top; |
598 | size = get_fadump_area_size(); |
599 | fw_dump.reserve_dump_area_size = size; |
600 | if (fw_dump.dump_active) { |
601 | pr_info("Firmware-assisted dump is active.\n" ); |
602 | |
603 | #ifdef CONFIG_HUGETLB_PAGE |
604 | /* |
605 | * FADump capture kernel doesn't care much about hugepages. |
606 | * In fact, handling hugepages in capture kernel is asking for |
607 | * trouble. So, disable HugeTLB support when fadump is active. |
608 | */ |
609 | hugetlb_disabled = true; |
610 | #endif |
611 | /* |
612 | * If last boot has crashed then reserve all the memory |
613 | * above boot memory size so that we don't touch it until |
614 | * dump is written to disk by userspace tool. This memory |
615 | * can be released for general use by invalidating fadump. |
616 | */ |
617 | fadump_reserve_crash_area(base); |
618 | |
619 | pr_debug("fadumphdr_addr = %#016lx\n" , fw_dump.fadumphdr_addr); |
620 | pr_debug("Reserve dump area start address: 0x%lx\n" , |
621 | fw_dump.reserve_dump_area_start); |
622 | } else { |
623 | /* |
624 | * Reserve memory at an offset closer to bottom of the RAM to |
625 | * minimize the impact of memory hot-remove operation. |
626 | */ |
627 | base = fadump_locate_reserve_mem(base, size); |
628 | |
629 | if (!base || (base + size > mem_boundary)) { |
630 | pr_err("Failed to find memory chunk for reservation!\n" ); |
631 | goto error_out; |
632 | } |
633 | fw_dump.reserve_dump_area_start = base; |
634 | |
635 | /* |
636 | * Calculate the kernel metadata address and register it with |
637 | * f/w if the platform supports. |
638 | */ |
639 | if (fw_dump.ops->fadump_setup_metadata && |
640 | (fw_dump.ops->fadump_setup_metadata(&fw_dump) < 0)) |
641 | goto error_out; |
642 | |
643 | if (memblock_reserve(base, size)) { |
644 | pr_err("Failed to reserve memory!\n" ); |
645 | goto error_out; |
646 | } |
647 | |
648 | pr_info("Reserved %lldMB of memory at %#016llx (System RAM: %lldMB)\n" , |
649 | (size >> 20), base, (memblock_phys_mem_size() >> 20)); |
650 | |
651 | ret = fadump_cma_init(); |
652 | } |
653 | |
654 | return ret; |
655 | error_out: |
656 | fw_dump.fadump_enabled = 0; |
657 | fw_dump.reserve_dump_area_size = 0; |
658 | return 0; |
659 | } |
660 | |
661 | /* Look for fadump= cmdline option. */ |
662 | static int __init early_fadump_param(char *p) |
663 | { |
664 | if (!p) |
665 | return 1; |
666 | |
667 | if (strncmp(p, "on" , 2) == 0) |
668 | fw_dump.fadump_enabled = 1; |
669 | else if (strncmp(p, "off" , 3) == 0) |
670 | fw_dump.fadump_enabled = 0; |
671 | else if (strncmp(p, "nocma" , 5) == 0) { |
672 | fw_dump.fadump_enabled = 1; |
673 | fw_dump.nocma = 1; |
674 | } |
675 | |
676 | return 0; |
677 | } |
678 | early_param("fadump" , early_fadump_param); |
679 | |
680 | /* |
681 | * Look for fadump_reserve_mem= cmdline option |
682 | * TODO: Remove references to 'fadump_reserve_mem=' parameter, |
683 | * the sooner 'crashkernel=' parameter is accustomed to. |
684 | */ |
685 | static int __init early_fadump_reserve_mem(char *p) |
686 | { |
687 | if (p) |
688 | fw_dump.reserve_bootvar = memparse(ptr: p, retptr: &p); |
689 | return 0; |
690 | } |
691 | early_param("fadump_reserve_mem" , early_fadump_reserve_mem); |
692 | |
693 | void crash_fadump(struct pt_regs *regs, const char *str) |
694 | { |
695 | unsigned int msecs; |
696 | struct *fdh = NULL; |
697 | int old_cpu, this_cpu; |
698 | /* Do not include first CPU */ |
699 | unsigned int ncpus = num_online_cpus() - 1; |
700 | |
701 | if (!should_fadump_crash()) |
702 | return; |
703 | |
704 | /* |
705 | * old_cpu == -1 means this is the first CPU which has come here, |
706 | * go ahead and trigger fadump. |
707 | * |
708 | * old_cpu != -1 means some other CPU has already on it's way |
709 | * to trigger fadump, just keep looping here. |
710 | */ |
711 | this_cpu = smp_processor_id(); |
712 | old_cpu = cmpxchg(&crashing_cpu, -1, this_cpu); |
713 | |
714 | if (old_cpu != -1) { |
715 | atomic_inc(v: &cpus_in_fadump); |
716 | |
717 | /* |
718 | * We can't loop here indefinitely. Wait as long as fadump |
719 | * is in force. If we race with fadump un-registration this |
720 | * loop will break and then we go down to normal panic path |
721 | * and reboot. If fadump is in force the first crashing |
722 | * cpu will definitely trigger fadump. |
723 | */ |
724 | while (fw_dump.dump_registered) |
725 | cpu_relax(); |
726 | return; |
727 | } |
728 | |
729 | fdh = __va(fw_dump.fadumphdr_addr); |
730 | fdh->crashing_cpu = crashing_cpu; |
731 | crash_save_vmcoreinfo(); |
732 | |
733 | if (regs) |
734 | fdh->regs = *regs; |
735 | else |
736 | ppc_save_regs(&fdh->regs); |
737 | |
738 | fdh->cpu_mask = *cpu_online_mask; |
739 | |
740 | /* |
741 | * If we came in via system reset, wait a while for the secondary |
742 | * CPUs to enter. |
743 | */ |
744 | if (TRAP(&(fdh->regs)) == INTERRUPT_SYSTEM_RESET) { |
745 | msecs = CRASH_TIMEOUT; |
746 | while ((atomic_read(v: &cpus_in_fadump) < ncpus) && (--msecs > 0)) |
747 | mdelay(1); |
748 | } |
749 | |
750 | fw_dump.ops->fadump_trigger(fdh, str); |
751 | } |
752 | |
753 | u32 *__init fadump_regs_to_elf_notes(u32 *buf, struct pt_regs *regs) |
754 | { |
755 | struct elf_prstatus prstatus; |
756 | |
757 | memset(&prstatus, 0, sizeof(prstatus)); |
758 | /* |
759 | * FIXME: How do i get PID? Do I really need it? |
760 | * prstatus.pr_pid = ???? |
761 | */ |
762 | elf_core_copy_regs(elfregs: &prstatus.pr_reg, regs); |
763 | buf = append_elf_note(buf, CRASH_CORE_NOTE_NAME, NT_PRSTATUS, |
764 | data: &prstatus, data_len: sizeof(prstatus)); |
765 | return buf; |
766 | } |
767 | |
768 | void __init (char *bufp) |
769 | { |
770 | struct elf_phdr *phdr; |
771 | |
772 | bufp += sizeof(struct elfhdr); |
773 | |
774 | /* First note is a place holder for cpu notes info. */ |
775 | phdr = (struct elf_phdr *)bufp; |
776 | |
777 | if (phdr->p_type == PT_NOTE) { |
778 | phdr->p_paddr = __pa(fw_dump.cpu_notes_buf_vaddr); |
779 | phdr->p_offset = phdr->p_paddr; |
780 | phdr->p_filesz = fw_dump.cpu_notes_buf_size; |
781 | phdr->p_memsz = fw_dump.cpu_notes_buf_size; |
782 | } |
783 | return; |
784 | } |
785 | |
786 | static void *__init fadump_alloc_buffer(unsigned long size) |
787 | { |
788 | unsigned long count, i; |
789 | struct page *page; |
790 | void *vaddr; |
791 | |
792 | vaddr = alloc_pages_exact(size, GFP_KERNEL | __GFP_ZERO); |
793 | if (!vaddr) |
794 | return NULL; |
795 | |
796 | count = PAGE_ALIGN(size) / PAGE_SIZE; |
797 | page = virt_to_page(vaddr); |
798 | for (i = 0; i < count; i++) |
799 | mark_page_reserved(page: page + i); |
800 | return vaddr; |
801 | } |
802 | |
803 | static void fadump_free_buffer(unsigned long vaddr, unsigned long size) |
804 | { |
805 | free_reserved_area(start: (void *)vaddr, end: (void *)(vaddr + size), poison: -1, NULL); |
806 | } |
807 | |
808 | s32 __init fadump_setup_cpu_notes_buf(u32 num_cpus) |
809 | { |
810 | /* Allocate buffer to hold cpu crash notes. */ |
811 | fw_dump.cpu_notes_buf_size = num_cpus * sizeof(note_buf_t); |
812 | fw_dump.cpu_notes_buf_size = PAGE_ALIGN(fw_dump.cpu_notes_buf_size); |
813 | fw_dump.cpu_notes_buf_vaddr = |
814 | (unsigned long)fadump_alloc_buffer(size: fw_dump.cpu_notes_buf_size); |
815 | if (!fw_dump.cpu_notes_buf_vaddr) { |
816 | pr_err("Failed to allocate %ld bytes for CPU notes buffer\n" , |
817 | fw_dump.cpu_notes_buf_size); |
818 | return -ENOMEM; |
819 | } |
820 | |
821 | pr_debug("Allocated buffer for cpu notes of size %ld at 0x%lx\n" , |
822 | fw_dump.cpu_notes_buf_size, |
823 | fw_dump.cpu_notes_buf_vaddr); |
824 | return 0; |
825 | } |
826 | |
827 | void fadump_free_cpu_notes_buf(void) |
828 | { |
829 | if (!fw_dump.cpu_notes_buf_vaddr) |
830 | return; |
831 | |
832 | fadump_free_buffer(vaddr: fw_dump.cpu_notes_buf_vaddr, |
833 | size: fw_dump.cpu_notes_buf_size); |
834 | fw_dump.cpu_notes_buf_vaddr = 0; |
835 | fw_dump.cpu_notes_buf_size = 0; |
836 | } |
837 | |
838 | static void fadump_free_mem_ranges(struct fadump_mrange_info *mrange_info) |
839 | { |
840 | if (mrange_info->is_static) { |
841 | mrange_info->mem_range_cnt = 0; |
842 | return; |
843 | } |
844 | |
845 | kfree(objp: mrange_info->mem_ranges); |
846 | memset((void *)((u64)mrange_info + RNG_NAME_SZ), 0, |
847 | (sizeof(struct fadump_mrange_info) - RNG_NAME_SZ)); |
848 | } |
849 | |
850 | /* |
851 | * Allocate or reallocate mem_ranges array in incremental units |
852 | * of PAGE_SIZE. |
853 | */ |
854 | static int fadump_alloc_mem_ranges(struct fadump_mrange_info *mrange_info) |
855 | { |
856 | struct fadump_memory_range *new_array; |
857 | u64 new_size; |
858 | |
859 | new_size = mrange_info->mem_ranges_sz + PAGE_SIZE; |
860 | pr_debug("Allocating %llu bytes of memory for %s memory ranges\n" , |
861 | new_size, mrange_info->name); |
862 | |
863 | new_array = krealloc(objp: mrange_info->mem_ranges, new_size, GFP_KERNEL); |
864 | if (new_array == NULL) { |
865 | pr_err("Insufficient memory for setting up %s memory ranges\n" , |
866 | mrange_info->name); |
867 | fadump_free_mem_ranges(mrange_info); |
868 | return -ENOMEM; |
869 | } |
870 | |
871 | mrange_info->mem_ranges = new_array; |
872 | mrange_info->mem_ranges_sz = new_size; |
873 | mrange_info->max_mem_ranges = (new_size / |
874 | sizeof(struct fadump_memory_range)); |
875 | return 0; |
876 | } |
877 | static inline int fadump_add_mem_range(struct fadump_mrange_info *mrange_info, |
878 | u64 base, u64 end) |
879 | { |
880 | struct fadump_memory_range *mem_ranges = mrange_info->mem_ranges; |
881 | bool is_adjacent = false; |
882 | u64 start, size; |
883 | |
884 | if (base == end) |
885 | return 0; |
886 | |
887 | /* |
888 | * Fold adjacent memory ranges to bring down the memory ranges/ |
889 | * PT_LOAD segments count. |
890 | */ |
891 | if (mrange_info->mem_range_cnt) { |
892 | start = mem_ranges[mrange_info->mem_range_cnt - 1].base; |
893 | size = mem_ranges[mrange_info->mem_range_cnt - 1].size; |
894 | |
895 | /* |
896 | * Boot memory area needs separate PT_LOAD segment(s) as it |
897 | * is moved to a different location at the time of crash. |
898 | * So, fold only if the region is not boot memory area. |
899 | */ |
900 | if ((start + size) == base && start >= fw_dump.boot_mem_top) |
901 | is_adjacent = true; |
902 | } |
903 | if (!is_adjacent) { |
904 | /* resize the array on reaching the limit */ |
905 | if (mrange_info->mem_range_cnt == mrange_info->max_mem_ranges) { |
906 | int ret; |
907 | |
908 | if (mrange_info->is_static) { |
909 | pr_err("Reached array size limit for %s memory ranges\n" , |
910 | mrange_info->name); |
911 | return -ENOSPC; |
912 | } |
913 | |
914 | ret = fadump_alloc_mem_ranges(mrange_info); |
915 | if (ret) |
916 | return ret; |
917 | |
918 | /* Update to the new resized array */ |
919 | mem_ranges = mrange_info->mem_ranges; |
920 | } |
921 | |
922 | start = base; |
923 | mem_ranges[mrange_info->mem_range_cnt].base = start; |
924 | mrange_info->mem_range_cnt++; |
925 | } |
926 | |
927 | mem_ranges[mrange_info->mem_range_cnt - 1].size = (end - start); |
928 | pr_debug("%s_memory_range[%d] [%#016llx-%#016llx], %#llx bytes\n" , |
929 | mrange_info->name, (mrange_info->mem_range_cnt - 1), |
930 | start, end - 1, (end - start)); |
931 | return 0; |
932 | } |
933 | |
934 | static int fadump_exclude_reserved_area(u64 start, u64 end) |
935 | { |
936 | u64 ra_start, ra_end; |
937 | int ret = 0; |
938 | |
939 | ra_start = fw_dump.reserve_dump_area_start; |
940 | ra_end = ra_start + fw_dump.reserve_dump_area_size; |
941 | |
942 | if ((ra_start < end) && (ra_end > start)) { |
943 | if ((start < ra_start) && (end > ra_end)) { |
944 | ret = fadump_add_mem_range(mrange_info: &crash_mrange_info, |
945 | base: start, end: ra_start); |
946 | if (ret) |
947 | return ret; |
948 | |
949 | ret = fadump_add_mem_range(mrange_info: &crash_mrange_info, |
950 | base: ra_end, end); |
951 | } else if (start < ra_start) { |
952 | ret = fadump_add_mem_range(mrange_info: &crash_mrange_info, |
953 | base: start, end: ra_start); |
954 | } else if (ra_end < end) { |
955 | ret = fadump_add_mem_range(mrange_info: &crash_mrange_info, |
956 | base: ra_end, end); |
957 | } |
958 | } else |
959 | ret = fadump_add_mem_range(mrange_info: &crash_mrange_info, base: start, end); |
960 | |
961 | return ret; |
962 | } |
963 | |
964 | static int (char *bufp) |
965 | { |
966 | struct elfhdr *elf; |
967 | |
968 | elf = (struct elfhdr *) bufp; |
969 | bufp += sizeof(struct elfhdr); |
970 | memcpy(elf->e_ident, ELFMAG, SELFMAG); |
971 | elf->e_ident[EI_CLASS] = ELF_CLASS; |
972 | elf->e_ident[EI_DATA] = ELF_DATA; |
973 | elf->e_ident[EI_VERSION] = EV_CURRENT; |
974 | elf->e_ident[EI_OSABI] = ELF_OSABI; |
975 | memset(elf->e_ident+EI_PAD, 0, EI_NIDENT-EI_PAD); |
976 | elf->e_type = ET_CORE; |
977 | elf->e_machine = ELF_ARCH; |
978 | elf->e_version = EV_CURRENT; |
979 | elf->e_entry = 0; |
980 | elf->e_phoff = sizeof(struct elfhdr); |
981 | elf->e_shoff = 0; |
982 | |
983 | if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2)) |
984 | elf->e_flags = 2; |
985 | else if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V1)) |
986 | elf->e_flags = 1; |
987 | else |
988 | elf->e_flags = 0; |
989 | |
990 | elf->e_ehsize = sizeof(struct elfhdr); |
991 | elf->e_phentsize = sizeof(struct elf_phdr); |
992 | elf->e_phnum = 0; |
993 | elf->e_shentsize = 0; |
994 | elf->e_shnum = 0; |
995 | elf->e_shstrndx = 0; |
996 | |
997 | return 0; |
998 | } |
999 | |
1000 | /* |
1001 | * Traverse through memblock structure and setup crash memory ranges. These |
1002 | * ranges will be used create PT_LOAD program headers in elfcore header. |
1003 | */ |
1004 | static int fadump_setup_crash_memory_ranges(void) |
1005 | { |
1006 | u64 i, start, end; |
1007 | int ret; |
1008 | |
1009 | pr_debug("Setup crash memory ranges.\n" ); |
1010 | crash_mrange_info.mem_range_cnt = 0; |
1011 | |
1012 | /* |
1013 | * Boot memory region(s) registered with firmware are moved to |
1014 | * different location at the time of crash. Create separate program |
1015 | * header(s) for this memory chunk(s) with the correct offset. |
1016 | */ |
1017 | for (i = 0; i < fw_dump.boot_mem_regs_cnt; i++) { |
1018 | start = fw_dump.boot_mem_addr[i]; |
1019 | end = start + fw_dump.boot_mem_sz[i]; |
1020 | ret = fadump_add_mem_range(mrange_info: &crash_mrange_info, base: start, end); |
1021 | if (ret) |
1022 | return ret; |
1023 | } |
1024 | |
1025 | for_each_mem_range(i, &start, &end) { |
1026 | /* |
1027 | * skip the memory chunk that is already added |
1028 | * (0 through boot_memory_top). |
1029 | */ |
1030 | if (start < fw_dump.boot_mem_top) { |
1031 | if (end > fw_dump.boot_mem_top) |
1032 | start = fw_dump.boot_mem_top; |
1033 | else |
1034 | continue; |
1035 | } |
1036 | |
1037 | /* add this range excluding the reserved dump area. */ |
1038 | ret = fadump_exclude_reserved_area(start, end); |
1039 | if (ret) |
1040 | return ret; |
1041 | } |
1042 | |
1043 | return 0; |
1044 | } |
1045 | |
1046 | /* |
1047 | * If the given physical address falls within the boot memory region then |
1048 | * return the relocated address that points to the dump region reserved |
1049 | * for saving initial boot memory contents. |
1050 | */ |
1051 | static inline unsigned long fadump_relocate(unsigned long paddr) |
1052 | { |
1053 | unsigned long raddr, rstart, rend, rlast, hole_size; |
1054 | int i; |
1055 | |
1056 | hole_size = 0; |
1057 | rlast = 0; |
1058 | raddr = paddr; |
1059 | for (i = 0; i < fw_dump.boot_mem_regs_cnt; i++) { |
1060 | rstart = fw_dump.boot_mem_addr[i]; |
1061 | rend = rstart + fw_dump.boot_mem_sz[i]; |
1062 | hole_size += (rstart - rlast); |
1063 | |
1064 | if (paddr >= rstart && paddr < rend) { |
1065 | raddr += fw_dump.boot_mem_dest_addr - hole_size; |
1066 | break; |
1067 | } |
1068 | |
1069 | rlast = rend; |
1070 | } |
1071 | |
1072 | pr_debug("vmcoreinfo: paddr = 0x%lx, raddr = 0x%lx\n" , paddr, raddr); |
1073 | return raddr; |
1074 | } |
1075 | |
1076 | static int (char *bufp) |
1077 | { |
1078 | unsigned long long raddr, offset; |
1079 | struct elf_phdr *phdr; |
1080 | struct elfhdr *elf; |
1081 | int i, j; |
1082 | |
1083 | fadump_init_elfcore_header(bufp); |
1084 | elf = (struct elfhdr *)bufp; |
1085 | bufp += sizeof(struct elfhdr); |
1086 | |
1087 | /* |
1088 | * setup ELF PT_NOTE, place holder for cpu notes info. The notes info |
1089 | * will be populated during second kernel boot after crash. Hence |
1090 | * this PT_NOTE will always be the first elf note. |
1091 | * |
1092 | * NOTE: Any new ELF note addition should be placed after this note. |
1093 | */ |
1094 | phdr = (struct elf_phdr *)bufp; |
1095 | bufp += sizeof(struct elf_phdr); |
1096 | phdr->p_type = PT_NOTE; |
1097 | phdr->p_flags = 0; |
1098 | phdr->p_vaddr = 0; |
1099 | phdr->p_align = 0; |
1100 | |
1101 | phdr->p_offset = 0; |
1102 | phdr->p_paddr = 0; |
1103 | phdr->p_filesz = 0; |
1104 | phdr->p_memsz = 0; |
1105 | |
1106 | (elf->e_phnum)++; |
1107 | |
1108 | /* setup ELF PT_NOTE for vmcoreinfo */ |
1109 | phdr = (struct elf_phdr *)bufp; |
1110 | bufp += sizeof(struct elf_phdr); |
1111 | phdr->p_type = PT_NOTE; |
1112 | phdr->p_flags = 0; |
1113 | phdr->p_vaddr = 0; |
1114 | phdr->p_align = 0; |
1115 | |
1116 | phdr->p_paddr = fadump_relocate(paddr: paddr_vmcoreinfo_note()); |
1117 | phdr->p_offset = phdr->p_paddr; |
1118 | phdr->p_memsz = phdr->p_filesz = VMCOREINFO_NOTE_SIZE; |
1119 | |
1120 | /* Increment number of program headers. */ |
1121 | (elf->e_phnum)++; |
1122 | |
1123 | /* setup PT_LOAD sections. */ |
1124 | j = 0; |
1125 | offset = 0; |
1126 | raddr = fw_dump.boot_mem_addr[0]; |
1127 | for (i = 0; i < crash_mrange_info.mem_range_cnt; i++) { |
1128 | u64 mbase, msize; |
1129 | |
1130 | mbase = crash_mrange_info.mem_ranges[i].base; |
1131 | msize = crash_mrange_info.mem_ranges[i].size; |
1132 | if (!msize) |
1133 | continue; |
1134 | |
1135 | phdr = (struct elf_phdr *)bufp; |
1136 | bufp += sizeof(struct elf_phdr); |
1137 | phdr->p_type = PT_LOAD; |
1138 | phdr->p_flags = PF_R|PF_W|PF_X; |
1139 | phdr->p_offset = mbase; |
1140 | |
1141 | if (mbase == raddr) { |
1142 | /* |
1143 | * The entire real memory region will be moved by |
1144 | * firmware to the specified destination_address. |
1145 | * Hence set the correct offset. |
1146 | */ |
1147 | phdr->p_offset = fw_dump.boot_mem_dest_addr + offset; |
1148 | if (j < (fw_dump.boot_mem_regs_cnt - 1)) { |
1149 | offset += fw_dump.boot_mem_sz[j]; |
1150 | raddr = fw_dump.boot_mem_addr[++j]; |
1151 | } |
1152 | } |
1153 | |
1154 | phdr->p_paddr = mbase; |
1155 | phdr->p_vaddr = (unsigned long)__va(mbase); |
1156 | phdr->p_filesz = msize; |
1157 | phdr->p_memsz = msize; |
1158 | phdr->p_align = 0; |
1159 | |
1160 | /* Increment number of program headers. */ |
1161 | (elf->e_phnum)++; |
1162 | } |
1163 | return 0; |
1164 | } |
1165 | |
1166 | static unsigned long (unsigned long addr) |
1167 | { |
1168 | struct *fdh; |
1169 | |
1170 | if (!addr) |
1171 | return 0; |
1172 | |
1173 | fdh = __va(addr); |
1174 | addr += sizeof(struct fadump_crash_info_header); |
1175 | |
1176 | memset(fdh, 0, sizeof(struct fadump_crash_info_header)); |
1177 | fdh->magic_number = FADUMP_CRASH_INFO_MAGIC; |
1178 | fdh->elfcorehdr_addr = addr; |
1179 | /* We will set the crashing cpu id in crash_fadump() during crash. */ |
1180 | fdh->crashing_cpu = FADUMP_CPU_UNKNOWN; |
1181 | /* |
1182 | * When LPAR is terminated by PYHP, ensure all possible CPUs' |
1183 | * register data is processed while exporting the vmcore. |
1184 | */ |
1185 | fdh->cpu_mask = *cpu_possible_mask; |
1186 | |
1187 | return addr; |
1188 | } |
1189 | |
1190 | static int register_fadump(void) |
1191 | { |
1192 | unsigned long addr; |
1193 | void *vaddr; |
1194 | int ret; |
1195 | |
1196 | /* |
1197 | * If no memory is reserved then we can not register for firmware- |
1198 | * assisted dump. |
1199 | */ |
1200 | if (!fw_dump.reserve_dump_area_size) |
1201 | return -ENODEV; |
1202 | |
1203 | ret = fadump_setup_crash_memory_ranges(); |
1204 | if (ret) |
1205 | return ret; |
1206 | |
1207 | addr = fw_dump.fadumphdr_addr; |
1208 | |
1209 | /* Initialize fadump crash info header. */ |
1210 | addr = init_fadump_header(addr); |
1211 | vaddr = __va(addr); |
1212 | |
1213 | pr_debug("Creating ELF core headers at %#016lx\n" , addr); |
1214 | fadump_create_elfcore_headers(bufp: vaddr); |
1215 | |
1216 | /* register the future kernel dump with firmware. */ |
1217 | pr_debug("Registering for firmware-assisted kernel dump...\n" ); |
1218 | return fw_dump.ops->fadump_register(&fw_dump); |
1219 | } |
1220 | |
1221 | void fadump_cleanup(void) |
1222 | { |
1223 | if (!fw_dump.fadump_supported) |
1224 | return; |
1225 | |
1226 | /* Invalidate the registration only if dump is active. */ |
1227 | if (fw_dump.dump_active) { |
1228 | pr_debug("Invalidating firmware-assisted dump registration\n" ); |
1229 | fw_dump.ops->fadump_invalidate(&fw_dump); |
1230 | } else if (fw_dump.dump_registered) { |
1231 | /* Un-register Firmware-assisted dump if it was registered. */ |
1232 | fw_dump.ops->fadump_unregister(&fw_dump); |
1233 | fadump_free_mem_ranges(mrange_info: &crash_mrange_info); |
1234 | } |
1235 | |
1236 | if (fw_dump.ops->fadump_cleanup) |
1237 | fw_dump.ops->fadump_cleanup(&fw_dump); |
1238 | } |
1239 | |
1240 | static void fadump_free_reserved_memory(unsigned long start_pfn, |
1241 | unsigned long end_pfn) |
1242 | { |
1243 | unsigned long pfn; |
1244 | unsigned long time_limit = jiffies + HZ; |
1245 | |
1246 | pr_info("freeing reserved memory (0x%llx - 0x%llx)\n" , |
1247 | PFN_PHYS(start_pfn), PFN_PHYS(end_pfn)); |
1248 | |
1249 | for (pfn = start_pfn; pfn < end_pfn; pfn++) { |
1250 | free_reserved_page(pfn_to_page(pfn)); |
1251 | |
1252 | if (time_after(jiffies, time_limit)) { |
1253 | cond_resched(); |
1254 | time_limit = jiffies + HZ; |
1255 | } |
1256 | } |
1257 | } |
1258 | |
1259 | /* |
1260 | * Skip memory holes and free memory that was actually reserved. |
1261 | */ |
1262 | static void fadump_release_reserved_area(u64 start, u64 end) |
1263 | { |
1264 | unsigned long reg_spfn, reg_epfn; |
1265 | u64 tstart, tend, spfn, epfn; |
1266 | int i; |
1267 | |
1268 | spfn = PHYS_PFN(start); |
1269 | epfn = PHYS_PFN(end); |
1270 | |
1271 | for_each_mem_pfn_range(i, MAX_NUMNODES, ®_spfn, ®_epfn, NULL) { |
1272 | tstart = max_t(u64, spfn, reg_spfn); |
1273 | tend = min_t(u64, epfn, reg_epfn); |
1274 | |
1275 | if (tstart < tend) { |
1276 | fadump_free_reserved_memory(start_pfn: tstart, end_pfn: tend); |
1277 | |
1278 | if (tend == epfn) |
1279 | break; |
1280 | |
1281 | spfn = tend; |
1282 | } |
1283 | } |
1284 | } |
1285 | |
1286 | /* |
1287 | * Sort the mem ranges in-place and merge adjacent ranges |
1288 | * to minimize the memory ranges count. |
1289 | */ |
1290 | static void sort_and_merge_mem_ranges(struct fadump_mrange_info *mrange_info) |
1291 | { |
1292 | struct fadump_memory_range *mem_ranges; |
1293 | u64 base, size; |
1294 | int i, j, idx; |
1295 | |
1296 | if (!reserved_mrange_info.mem_range_cnt) |
1297 | return; |
1298 | |
1299 | /* Sort the memory ranges */ |
1300 | mem_ranges = mrange_info->mem_ranges; |
1301 | for (i = 0; i < mrange_info->mem_range_cnt; i++) { |
1302 | idx = i; |
1303 | for (j = (i + 1); j < mrange_info->mem_range_cnt; j++) { |
1304 | if (mem_ranges[idx].base > mem_ranges[j].base) |
1305 | idx = j; |
1306 | } |
1307 | if (idx != i) |
1308 | swap(mem_ranges[idx], mem_ranges[i]); |
1309 | } |
1310 | |
1311 | /* Merge adjacent reserved ranges */ |
1312 | idx = 0; |
1313 | for (i = 1; i < mrange_info->mem_range_cnt; i++) { |
1314 | base = mem_ranges[i-1].base; |
1315 | size = mem_ranges[i-1].size; |
1316 | if (mem_ranges[i].base == (base + size)) |
1317 | mem_ranges[idx].size += mem_ranges[i].size; |
1318 | else { |
1319 | idx++; |
1320 | if (i == idx) |
1321 | continue; |
1322 | |
1323 | mem_ranges[idx] = mem_ranges[i]; |
1324 | } |
1325 | } |
1326 | mrange_info->mem_range_cnt = idx + 1; |
1327 | } |
1328 | |
1329 | /* |
1330 | * Scan reserved-ranges to consider them while reserving/releasing |
1331 | * memory for FADump. |
1332 | */ |
1333 | static void __init early_init_dt_scan_reserved_ranges(unsigned long node) |
1334 | { |
1335 | const __be32 *prop; |
1336 | int len, ret = -1; |
1337 | unsigned long i; |
1338 | |
1339 | /* reserved-ranges already scanned */ |
1340 | if (reserved_mrange_info.mem_range_cnt != 0) |
1341 | return; |
1342 | |
1343 | prop = of_get_flat_dt_prop(node, name: "reserved-ranges" , size: &len); |
1344 | if (!prop) |
1345 | return; |
1346 | |
1347 | /* |
1348 | * Each reserved range is an (address,size) pair, 2 cells each, |
1349 | * totalling 4 cells per range. |
1350 | */ |
1351 | for (i = 0; i < len / (sizeof(*prop) * 4); i++) { |
1352 | u64 base, size; |
1353 | |
1354 | base = of_read_number(cell: prop + (i * 4) + 0, size: 2); |
1355 | size = of_read_number(cell: prop + (i * 4) + 2, size: 2); |
1356 | |
1357 | if (size) { |
1358 | ret = fadump_add_mem_range(mrange_info: &reserved_mrange_info, |
1359 | base, end: base + size); |
1360 | if (ret < 0) { |
1361 | pr_warn("some reserved ranges are ignored!\n" ); |
1362 | break; |
1363 | } |
1364 | } |
1365 | } |
1366 | |
1367 | /* Compact reserved ranges */ |
1368 | sort_and_merge_mem_ranges(mrange_info: &reserved_mrange_info); |
1369 | } |
1370 | |
1371 | /* |
1372 | * Release the memory that was reserved during early boot to preserve the |
1373 | * crash'ed kernel's memory contents except reserved dump area (permanent |
1374 | * reservation) and reserved ranges used by F/W. The released memory will |
1375 | * be available for general use. |
1376 | */ |
1377 | static void fadump_release_memory(u64 begin, u64 end) |
1378 | { |
1379 | u64 ra_start, ra_end, tstart; |
1380 | int i, ret; |
1381 | |
1382 | ra_start = fw_dump.reserve_dump_area_start; |
1383 | ra_end = ra_start + fw_dump.reserve_dump_area_size; |
1384 | |
1385 | /* |
1386 | * If reserved ranges array limit is hit, overwrite the last reserved |
1387 | * memory range with reserved dump area to ensure it is excluded from |
1388 | * the memory being released (reused for next FADump registration). |
1389 | */ |
1390 | if (reserved_mrange_info.mem_range_cnt == |
1391 | reserved_mrange_info.max_mem_ranges) |
1392 | reserved_mrange_info.mem_range_cnt--; |
1393 | |
1394 | ret = fadump_add_mem_range(mrange_info: &reserved_mrange_info, base: ra_start, end: ra_end); |
1395 | if (ret != 0) |
1396 | return; |
1397 | |
1398 | /* Get the reserved ranges list in order first. */ |
1399 | sort_and_merge_mem_ranges(mrange_info: &reserved_mrange_info); |
1400 | |
1401 | /* Exclude reserved ranges and release remaining memory */ |
1402 | tstart = begin; |
1403 | for (i = 0; i < reserved_mrange_info.mem_range_cnt; i++) { |
1404 | ra_start = reserved_mrange_info.mem_ranges[i].base; |
1405 | ra_end = ra_start + reserved_mrange_info.mem_ranges[i].size; |
1406 | |
1407 | if (tstart >= ra_end) |
1408 | continue; |
1409 | |
1410 | if (tstart < ra_start) |
1411 | fadump_release_reserved_area(start: tstart, end: ra_start); |
1412 | tstart = ra_end; |
1413 | } |
1414 | |
1415 | if (tstart < end) |
1416 | fadump_release_reserved_area(start: tstart, end); |
1417 | } |
1418 | |
1419 | static void fadump_invalidate_release_mem(void) |
1420 | { |
1421 | mutex_lock(&fadump_mutex); |
1422 | if (!fw_dump.dump_active) { |
1423 | mutex_unlock(lock: &fadump_mutex); |
1424 | return; |
1425 | } |
1426 | |
1427 | fadump_cleanup(); |
1428 | mutex_unlock(lock: &fadump_mutex); |
1429 | |
1430 | fadump_release_memory(begin: fw_dump.boot_mem_top, end: memblock_end_of_DRAM()); |
1431 | fadump_free_cpu_notes_buf(); |
1432 | |
1433 | /* |
1434 | * Setup kernel metadata and initialize the kernel dump |
1435 | * memory structure for FADump re-registration. |
1436 | */ |
1437 | if (fw_dump.ops->fadump_setup_metadata && |
1438 | (fw_dump.ops->fadump_setup_metadata(&fw_dump) < 0)) |
1439 | pr_warn("Failed to setup kernel metadata!\n" ); |
1440 | fw_dump.ops->fadump_init_mem_struct(&fw_dump); |
1441 | } |
1442 | |
1443 | static ssize_t release_mem_store(struct kobject *kobj, |
1444 | struct kobj_attribute *attr, |
1445 | const char *buf, size_t count) |
1446 | { |
1447 | int input = -1; |
1448 | |
1449 | if (!fw_dump.dump_active) |
1450 | return -EPERM; |
1451 | |
1452 | if (kstrtoint(s: buf, base: 0, res: &input)) |
1453 | return -EINVAL; |
1454 | |
1455 | if (input == 1) { |
1456 | /* |
1457 | * Take away the '/proc/vmcore'. We are releasing the dump |
1458 | * memory, hence it will not be valid anymore. |
1459 | */ |
1460 | #ifdef CONFIG_PROC_VMCORE |
1461 | vmcore_cleanup(); |
1462 | #endif |
1463 | fadump_invalidate_release_mem(); |
1464 | |
1465 | } else |
1466 | return -EINVAL; |
1467 | return count; |
1468 | } |
1469 | |
1470 | /* Release the reserved memory and disable the FADump */ |
1471 | static void __init unregister_fadump(void) |
1472 | { |
1473 | fadump_cleanup(); |
1474 | fadump_release_memory(begin: fw_dump.reserve_dump_area_start, |
1475 | end: fw_dump.reserve_dump_area_size); |
1476 | fw_dump.fadump_enabled = 0; |
1477 | kobject_put(kobj: fadump_kobj); |
1478 | } |
1479 | |
1480 | static ssize_t enabled_show(struct kobject *kobj, |
1481 | struct kobj_attribute *attr, |
1482 | char *buf) |
1483 | { |
1484 | return sprintf(buf, fmt: "%d\n" , fw_dump.fadump_enabled); |
1485 | } |
1486 | |
1487 | static ssize_t mem_reserved_show(struct kobject *kobj, |
1488 | struct kobj_attribute *attr, |
1489 | char *buf) |
1490 | { |
1491 | return sprintf(buf, fmt: "%ld\n" , fw_dump.reserve_dump_area_size); |
1492 | } |
1493 | |
1494 | static ssize_t registered_show(struct kobject *kobj, |
1495 | struct kobj_attribute *attr, |
1496 | char *buf) |
1497 | { |
1498 | return sprintf(buf, fmt: "%d\n" , fw_dump.dump_registered); |
1499 | } |
1500 | |
1501 | static ssize_t registered_store(struct kobject *kobj, |
1502 | struct kobj_attribute *attr, |
1503 | const char *buf, size_t count) |
1504 | { |
1505 | int ret = 0; |
1506 | int input = -1; |
1507 | |
1508 | if (!fw_dump.fadump_enabled || fw_dump.dump_active) |
1509 | return -EPERM; |
1510 | |
1511 | if (kstrtoint(s: buf, base: 0, res: &input)) |
1512 | return -EINVAL; |
1513 | |
1514 | mutex_lock(&fadump_mutex); |
1515 | |
1516 | switch (input) { |
1517 | case 0: |
1518 | if (fw_dump.dump_registered == 0) { |
1519 | goto unlock_out; |
1520 | } |
1521 | |
1522 | /* Un-register Firmware-assisted dump */ |
1523 | pr_debug("Un-register firmware-assisted dump\n" ); |
1524 | fw_dump.ops->fadump_unregister(&fw_dump); |
1525 | break; |
1526 | case 1: |
1527 | if (fw_dump.dump_registered == 1) { |
1528 | /* Un-register Firmware-assisted dump */ |
1529 | fw_dump.ops->fadump_unregister(&fw_dump); |
1530 | } |
1531 | /* Register Firmware-assisted dump */ |
1532 | ret = register_fadump(); |
1533 | break; |
1534 | default: |
1535 | ret = -EINVAL; |
1536 | break; |
1537 | } |
1538 | |
1539 | unlock_out: |
1540 | mutex_unlock(lock: &fadump_mutex); |
1541 | return ret < 0 ? ret : count; |
1542 | } |
1543 | |
1544 | static int fadump_region_show(struct seq_file *m, void *private) |
1545 | { |
1546 | if (!fw_dump.fadump_enabled) |
1547 | return 0; |
1548 | |
1549 | mutex_lock(&fadump_mutex); |
1550 | fw_dump.ops->fadump_region_show(&fw_dump, m); |
1551 | mutex_unlock(lock: &fadump_mutex); |
1552 | return 0; |
1553 | } |
1554 | |
1555 | static struct kobj_attribute release_attr = __ATTR_WO(release_mem); |
1556 | static struct kobj_attribute enable_attr = __ATTR_RO(enabled); |
1557 | static struct kobj_attribute register_attr = __ATTR_RW(registered); |
1558 | static struct kobj_attribute mem_reserved_attr = __ATTR_RO(mem_reserved); |
1559 | |
1560 | static struct attribute *fadump_attrs[] = { |
1561 | &enable_attr.attr, |
1562 | ®ister_attr.attr, |
1563 | &mem_reserved_attr.attr, |
1564 | NULL, |
1565 | }; |
1566 | |
1567 | ATTRIBUTE_GROUPS(fadump); |
1568 | |
1569 | DEFINE_SHOW_ATTRIBUTE(fadump_region); |
1570 | |
1571 | static void __init fadump_init_files(void) |
1572 | { |
1573 | int rc = 0; |
1574 | |
1575 | fadump_kobj = kobject_create_and_add(name: "fadump" , parent: kernel_kobj); |
1576 | if (!fadump_kobj) { |
1577 | pr_err("failed to create fadump kobject\n" ); |
1578 | return; |
1579 | } |
1580 | |
1581 | debugfs_create_file(name: "fadump_region" , mode: 0444, parent: arch_debugfs_dir, NULL, |
1582 | fops: &fadump_region_fops); |
1583 | |
1584 | if (fw_dump.dump_active) { |
1585 | rc = sysfs_create_file(kobj: fadump_kobj, attr: &release_attr.attr); |
1586 | if (rc) |
1587 | pr_err("unable to create release_mem sysfs file (%d)\n" , |
1588 | rc); |
1589 | } |
1590 | |
1591 | rc = sysfs_create_groups(kobj: fadump_kobj, groups: fadump_groups); |
1592 | if (rc) { |
1593 | pr_err("sysfs group creation failed (%d), unregistering FADump" , |
1594 | rc); |
1595 | unregister_fadump(); |
1596 | return; |
1597 | } |
1598 | |
1599 | /* |
1600 | * The FADump sysfs are moved from kernel_kobj to fadump_kobj need to |
1601 | * create symlink at old location to maintain backward compatibility. |
1602 | * |
1603 | * - fadump_enabled -> fadump/enabled |
1604 | * - fadump_registered -> fadump/registered |
1605 | * - fadump_release_mem -> fadump/release_mem |
1606 | */ |
1607 | rc = compat_only_sysfs_link_entry_to_kobj(kobj: kernel_kobj, target_kobj: fadump_kobj, |
1608 | target_name: "enabled" , symlink_name: "fadump_enabled" ); |
1609 | if (rc) { |
1610 | pr_err("unable to create fadump_enabled symlink (%d)" , rc); |
1611 | return; |
1612 | } |
1613 | |
1614 | rc = compat_only_sysfs_link_entry_to_kobj(kobj: kernel_kobj, target_kobj: fadump_kobj, |
1615 | target_name: "registered" , |
1616 | symlink_name: "fadump_registered" ); |
1617 | if (rc) { |
1618 | pr_err("unable to create fadump_registered symlink (%d)" , rc); |
1619 | sysfs_remove_link(kobj: kernel_kobj, name: "fadump_enabled" ); |
1620 | return; |
1621 | } |
1622 | |
1623 | if (fw_dump.dump_active) { |
1624 | rc = compat_only_sysfs_link_entry_to_kobj(kobj: kernel_kobj, |
1625 | target_kobj: fadump_kobj, |
1626 | target_name: "release_mem" , |
1627 | symlink_name: "fadump_release_mem" ); |
1628 | if (rc) |
1629 | pr_err("unable to create fadump_release_mem symlink (%d)" , |
1630 | rc); |
1631 | } |
1632 | return; |
1633 | } |
1634 | |
1635 | /* |
1636 | * Prepare for firmware-assisted dump. |
1637 | */ |
1638 | int __init setup_fadump(void) |
1639 | { |
1640 | if (!fw_dump.fadump_supported) |
1641 | return 0; |
1642 | |
1643 | fadump_init_files(); |
1644 | fadump_show_config(); |
1645 | |
1646 | if (!fw_dump.fadump_enabled) |
1647 | return 1; |
1648 | |
1649 | /* |
1650 | * If dump data is available then see if it is valid and prepare for |
1651 | * saving it to the disk. |
1652 | */ |
1653 | if (fw_dump.dump_active) { |
1654 | /* |
1655 | * if dump process fails then invalidate the registration |
1656 | * and release memory before proceeding for re-registration. |
1657 | */ |
1658 | if (fw_dump.ops->fadump_process(&fw_dump) < 0) |
1659 | fadump_invalidate_release_mem(); |
1660 | } |
1661 | /* Initialize the kernel dump memory structure and register with f/w */ |
1662 | else if (fw_dump.reserve_dump_area_size) { |
1663 | fw_dump.ops->fadump_init_mem_struct(&fw_dump); |
1664 | register_fadump(); |
1665 | } |
1666 | |
1667 | /* |
1668 | * In case of panic, fadump is triggered via ppc_panic_event() |
1669 | * panic notifier. Setting crash_kexec_post_notifiers to 'true' |
1670 | * lets panic() function take crash friendly path before panic |
1671 | * notifiers are invoked. |
1672 | */ |
1673 | crash_kexec_post_notifiers = true; |
1674 | |
1675 | return 1; |
1676 | } |
1677 | /* |
1678 | * Use subsys_initcall_sync() here because there is dependency with |
1679 | * crash_save_vmcoreinfo_init(), which must run first to ensure vmcoreinfo initialization |
1680 | * is done before registering with f/w. |
1681 | */ |
1682 | subsys_initcall_sync(setup_fadump); |
1683 | #else /* !CONFIG_PRESERVE_FA_DUMP */ |
1684 | |
1685 | /* Scan the Firmware Assisted dump configuration details. */ |
1686 | int __init early_init_dt_scan_fw_dump(unsigned long node, const char *uname, |
1687 | int depth, void *data) |
1688 | { |
1689 | if ((depth != 1) || (strcmp(uname, "ibm,opal" ) != 0)) |
1690 | return 0; |
1691 | |
1692 | opal_fadump_dt_scan(&fw_dump, node); |
1693 | return 1; |
1694 | } |
1695 | |
1696 | /* |
1697 | * When dump is active but PRESERVE_FA_DUMP is enabled on the kernel, |
1698 | * preserve crash data. The subsequent memory preserving kernel boot |
1699 | * is likely to process this crash data. |
1700 | */ |
1701 | int __init fadump_reserve_mem(void) |
1702 | { |
1703 | if (fw_dump.dump_active) { |
1704 | /* |
1705 | * If last boot has crashed then reserve all the memory |
1706 | * above boot memory to preserve crash data. |
1707 | */ |
1708 | pr_info("Preserving crash data for processing in next boot.\n" ); |
1709 | fadump_reserve_crash_area(fw_dump.boot_mem_top); |
1710 | } else |
1711 | pr_debug("FADump-aware kernel..\n" ); |
1712 | |
1713 | return 1; |
1714 | } |
1715 | #endif /* CONFIG_PRESERVE_FA_DUMP */ |
1716 | |
1717 | /* Preserve everything above the base address */ |
1718 | static void __init fadump_reserve_crash_area(u64 base) |
1719 | { |
1720 | u64 i, mstart, mend, msize; |
1721 | |
1722 | for_each_mem_range(i, &mstart, &mend) { |
1723 | msize = mend - mstart; |
1724 | |
1725 | if ((mstart + msize) < base) |
1726 | continue; |
1727 | |
1728 | if (mstart < base) { |
1729 | msize -= (base - mstart); |
1730 | mstart = base; |
1731 | } |
1732 | |
1733 | pr_info("Reserving %lluMB of memory at %#016llx for preserving crash data" , |
1734 | (msize >> 20), mstart); |
1735 | memblock_reserve(base: mstart, size: msize); |
1736 | } |
1737 | } |
1738 | |
1739 | unsigned long __init arch_reserved_kernel_pages(void) |
1740 | { |
1741 | return memblock_reserved_size() / PAGE_SIZE; |
1742 | } |
1743 | |