1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Hibernation support for RISCV
4 *
5 * Copyright (C) 2023 StarFive Technology Co., Ltd.
6 *
7 * Author: Jee Heng Sia <jeeheng.sia@starfivetech.com>
8 */
9
10#include <asm/barrier.h>
11#include <asm/cacheflush.h>
12#include <asm/mmu_context.h>
13#include <asm/page.h>
14#include <asm/pgalloc.h>
15#include <asm/pgtable.h>
16#include <asm/sections.h>
17#include <asm/set_memory.h>
18#include <asm/smp.h>
19#include <asm/suspend.h>
20
21#include <linux/cpu.h>
22#include <linux/memblock.h>
23#include <linux/pm.h>
24#include <linux/sched.h>
25#include <linux/suspend.h>
26#include <linux/utsname.h>
27
28/* The logical cpu number we should resume on, initialised to a non-cpu number. */
29static int sleep_cpu = -EINVAL;
30
31/* Pointer to the temporary resume page table. */
32static pgd_t *resume_pg_dir;
33
34/* CPU context to be saved. */
35struct suspend_context *hibernate_cpu_context;
36EXPORT_SYMBOL_GPL(hibernate_cpu_context);
37
38unsigned long relocated_restore_code;
39EXPORT_SYMBOL_GPL(relocated_restore_code);
40
41/**
42 * struct arch_hibernate_hdr_invariants - container to store kernel build version.
43 * @uts_version: to save the build number and date so that we do not resume with
44 * a different kernel.
45 */
46struct arch_hibernate_hdr_invariants {
47 char uts_version[__NEW_UTS_LEN + 1];
48};
49
50/**
51 * struct arch_hibernate_hdr - helper parameters that help us to restore the image.
52 * @invariants: container to store kernel build version.
53 * @hartid: to make sure same boot_cpu executes the hibernate/restore code.
54 * @saved_satp: original page table used by the hibernated image.
55 * @restore_cpu_addr: the kernel's image address to restore the CPU context.
56 */
57static struct arch_hibernate_hdr {
58 struct arch_hibernate_hdr_invariants invariants;
59 unsigned long hartid;
60 unsigned long saved_satp;
61 unsigned long restore_cpu_addr;
62} resume_hdr;
63
64static void arch_hdr_invariants(struct arch_hibernate_hdr_invariants *i)
65{
66 memset(i, 0, sizeof(*i));
67 memcpy(i->uts_version, init_utsname()->version, sizeof(i->uts_version));
68}
69
70/*
71 * Check if the given pfn is in the 'nosave' section.
72 */
73int pfn_is_nosave(unsigned long pfn)
74{
75 unsigned long nosave_begin_pfn = sym_to_pfn(&__nosave_begin);
76 unsigned long nosave_end_pfn = sym_to_pfn(&__nosave_end - 1);
77
78 return ((pfn >= nosave_begin_pfn) && (pfn <= nosave_end_pfn));
79}
80
81void notrace save_processor_state(void)
82{
83}
84
85void notrace restore_processor_state(void)
86{
87}
88
89/*
90 * Helper parameters need to be saved to the hibernation image header.
91 */
92int arch_hibernation_header_save(void *addr, unsigned int max_size)
93{
94 struct arch_hibernate_hdr *hdr = addr;
95
96 if (max_size < sizeof(*hdr))
97 return -EOVERFLOW;
98
99 arch_hdr_invariants(i: &hdr->invariants);
100
101 hdr->hartid = cpuid_to_hartid_map(sleep_cpu);
102 hdr->saved_satp = csr_read(CSR_SATP);
103 hdr->restore_cpu_addr = (unsigned long)__hibernate_cpu_resume;
104
105 return 0;
106}
107EXPORT_SYMBOL_GPL(arch_hibernation_header_save);
108
109/*
110 * Retrieve the helper parameters from the hibernation image header.
111 */
112int arch_hibernation_header_restore(void *addr)
113{
114 struct arch_hibernate_hdr_invariants invariants;
115 struct arch_hibernate_hdr *hdr = addr;
116 int ret = 0;
117
118 arch_hdr_invariants(i: &invariants);
119
120 if (memcmp(p: &hdr->invariants, q: &invariants, size: sizeof(invariants))) {
121 pr_crit("Hibernate image not generated by this kernel!\n");
122 return -EINVAL;
123 }
124
125 sleep_cpu = riscv_hartid_to_cpuid(hdr->hartid);
126 if (sleep_cpu < 0) {
127 pr_crit("Hibernated on a CPU not known to this kernel!\n");
128 sleep_cpu = -EINVAL;
129 return -EINVAL;
130 }
131
132#ifdef CONFIG_SMP
133 ret = bringup_hibernate_cpu(sleep_cpu);
134 if (ret) {
135 sleep_cpu = -EINVAL;
136 return ret;
137 }
138#endif
139 resume_hdr = *hdr;
140
141 return ret;
142}
143EXPORT_SYMBOL_GPL(arch_hibernation_header_restore);
144
145int swsusp_arch_suspend(void)
146{
147 int ret = 0;
148
149 if (__cpu_suspend_enter(hibernate_cpu_context)) {
150 sleep_cpu = smp_processor_id();
151 suspend_save_csrs(hibernate_cpu_context);
152 ret = swsusp_save();
153 } else {
154 suspend_restore_csrs(hibernate_cpu_context);
155 flush_tlb_all();
156 flush_icache_all();
157
158 /*
159 * Tell the hibernation core that we've just restored the memory.
160 */
161 in_suspend = 0;
162 sleep_cpu = -EINVAL;
163 }
164
165 return ret;
166}
167
168static int temp_pgtable_map_pte(pmd_t *dst_pmdp, pmd_t *src_pmdp, unsigned long start,
169 unsigned long end, pgprot_t prot)
170{
171 pte_t *src_ptep;
172 pte_t *dst_ptep;
173
174 if (pmd_none(READ_ONCE(*dst_pmdp))) {
175 dst_ptep = (pte_t *)get_safe_page(GFP_ATOMIC);
176 if (!dst_ptep)
177 return -ENOMEM;
178
179 pmd_populate_kernel(NULL, pmd: dst_pmdp, pte: dst_ptep);
180 }
181
182 dst_ptep = pte_offset_kernel(pmd: dst_pmdp, address: start);
183 src_ptep = pte_offset_kernel(pmd: src_pmdp, address: start);
184
185 do {
186 pte_t pte = READ_ONCE(*src_ptep);
187
188 if (pte_present(a: pte))
189 set_pte(ptep: dst_ptep, pte: __pte(val: pte_val(pte) | pgprot_val(prot)));
190 } while (dst_ptep++, src_ptep++, start += PAGE_SIZE, start < end);
191
192 return 0;
193}
194
195static int temp_pgtable_map_pmd(pud_t *dst_pudp, pud_t *src_pudp, unsigned long start,
196 unsigned long end, pgprot_t prot)
197{
198 unsigned long next;
199 unsigned long ret;
200 pmd_t *src_pmdp;
201 pmd_t *dst_pmdp;
202
203 if (pud_none(READ_ONCE(*dst_pudp))) {
204 dst_pmdp = (pmd_t *)get_safe_page(GFP_ATOMIC);
205 if (!dst_pmdp)
206 return -ENOMEM;
207
208 pud_populate(NULL, pud: dst_pudp, pmd: dst_pmdp);
209 }
210
211 dst_pmdp = pmd_offset(pud: dst_pudp, address: start);
212 src_pmdp = pmd_offset(pud: src_pudp, address: start);
213
214 do {
215 pmd_t pmd = READ_ONCE(*src_pmdp);
216
217 next = pmd_addr_end(start, end);
218
219 if (pmd_none(pmd))
220 continue;
221
222 if (pmd_leaf(pte: pmd)) {
223 set_pmd(pmdp: dst_pmdp, pmd: __pmd(val: pmd_val(pmd) | pgprot_val(prot)));
224 } else {
225 ret = temp_pgtable_map_pte(dst_pmdp, src_pmdp, start, end: next, prot);
226 if (ret)
227 return -ENOMEM;
228 }
229 } while (dst_pmdp++, src_pmdp++, start = next, start != end);
230
231 return 0;
232}
233
234static int temp_pgtable_map_pud(p4d_t *dst_p4dp, p4d_t *src_p4dp, unsigned long start,
235 unsigned long end, pgprot_t prot)
236{
237 unsigned long next;
238 unsigned long ret;
239 pud_t *dst_pudp;
240 pud_t *src_pudp;
241
242 if (p4d_none(READ_ONCE(*dst_p4dp))) {
243 dst_pudp = (pud_t *)get_safe_page(GFP_ATOMIC);
244 if (!dst_pudp)
245 return -ENOMEM;
246
247 p4d_populate(NULL, p4d: dst_p4dp, pud: dst_pudp);
248 }
249
250 dst_pudp = pud_offset(p4d: dst_p4dp, address: start);
251 src_pudp = pud_offset(p4d: src_p4dp, address: start);
252
253 do {
254 pud_t pud = READ_ONCE(*src_pudp);
255
256 next = pud_addr_end(start, end);
257
258 if (pud_none(pud))
259 continue;
260
261 if (pud_leaf(pud)) {
262 set_pud(pudp: dst_pudp, pud: __pud(val: pud_val(pud) | pgprot_val(prot)));
263 } else {
264 ret = temp_pgtable_map_pmd(dst_pudp, src_pudp, start, end: next, prot);
265 if (ret)
266 return -ENOMEM;
267 }
268 } while (dst_pudp++, src_pudp++, start = next, start != end);
269
270 return 0;
271}
272
273static int temp_pgtable_map_p4d(pgd_t *dst_pgdp, pgd_t *src_pgdp, unsigned long start,
274 unsigned long end, pgprot_t prot)
275{
276 unsigned long next;
277 unsigned long ret;
278 p4d_t *dst_p4dp;
279 p4d_t *src_p4dp;
280
281 if (pgd_none(READ_ONCE(*dst_pgdp))) {
282 dst_p4dp = (p4d_t *)get_safe_page(GFP_ATOMIC);
283 if (!dst_p4dp)
284 return -ENOMEM;
285
286 pgd_populate(NULL, pgd: dst_pgdp, p4d: dst_p4dp);
287 }
288
289 dst_p4dp = p4d_offset(pgd: dst_pgdp, address: start);
290 src_p4dp = p4d_offset(pgd: src_pgdp, address: start);
291
292 do {
293 p4d_t p4d = READ_ONCE(*src_p4dp);
294
295 next = p4d_addr_end(start, end);
296
297 if (p4d_none(p4d))
298 continue;
299
300 if (p4d_leaf(p4d)) {
301 set_p4d(p4dp: dst_p4dp, p4d: __p4d(val: p4d_val(p4d) | pgprot_val(prot)));
302 } else {
303 ret = temp_pgtable_map_pud(dst_p4dp, src_p4dp, start, end: next, prot);
304 if (ret)
305 return -ENOMEM;
306 }
307 } while (dst_p4dp++, src_p4dp++, start = next, start != end);
308
309 return 0;
310}
311
312static int temp_pgtable_mapping(pgd_t *pgdp, unsigned long start, unsigned long end, pgprot_t prot)
313{
314 pgd_t *dst_pgdp = pgd_offset_pgd(pgd: pgdp, address: start);
315 pgd_t *src_pgdp = pgd_offset_k(start);
316 unsigned long next;
317 unsigned long ret;
318
319 do {
320 pgd_t pgd = READ_ONCE(*src_pgdp);
321
322 next = pgd_addr_end(start, end);
323
324 if (pgd_none(pgd))
325 continue;
326
327 if (pgd_leaf(pgd)) {
328 set_pgd(dst_pgdp, __pgd(pgd_val(pgd) | pgprot_val(prot)));
329 } else {
330 ret = temp_pgtable_map_p4d(dst_pgdp, src_pgdp, start, end: next, prot);
331 if (ret)
332 return -ENOMEM;
333 }
334 } while (dst_pgdp++, src_pgdp++, start = next, start != end);
335
336 return 0;
337}
338
339static unsigned long relocate_restore_code(void)
340{
341 void *page = (void *)get_safe_page(GFP_ATOMIC);
342
343 if (!page)
344 return -ENOMEM;
345
346 copy_page(to: page, from: hibernate_core_restore_code);
347
348 /* Make the page containing the relocated code executable. */
349 set_memory_x(addr: (unsigned long)page, numpages: 1);
350
351 return (unsigned long)page;
352}
353
354int swsusp_arch_resume(void)
355{
356 unsigned long end = (unsigned long)pfn_to_virt(max_low_pfn);
357 unsigned long start = PAGE_OFFSET;
358 int ret;
359
360 /*
361 * Memory allocated by get_safe_page() will be dealt with by the hibernation core,
362 * we don't need to free it here.
363 */
364 resume_pg_dir = (pgd_t *)get_safe_page(GFP_ATOMIC);
365 if (!resume_pg_dir)
366 return -ENOMEM;
367
368 /*
369 * Create a temporary page table and map the whole linear region as executable and
370 * writable.
371 */
372 ret = temp_pgtable_mapping(resume_pg_dir, start, end, __pgprot(_PAGE_WRITE | _PAGE_EXEC));
373 if (ret)
374 return ret;
375
376 /* Move the restore code to a new page so that it doesn't get overwritten by itself. */
377 relocated_restore_code = relocate_restore_code();
378 if (relocated_restore_code == -ENOMEM)
379 return -ENOMEM;
380
381 /*
382 * Map the __hibernate_cpu_resume() address to the temporary page table so that the
383 * restore code can jumps to it after finished restore the image. The next execution
384 * code doesn't find itself in a different address space after switching over to the
385 * original page table used by the hibernated image.
386 * The __hibernate_cpu_resume() mapping is unnecessary for RV32 since the kernel and
387 * linear addresses are identical, but different for RV64. To ensure consistency, we
388 * map it for both RV32 and RV64 kernels.
389 * Additionally, we should ensure that the page is writable before restoring the image.
390 */
391 start = (unsigned long)resume_hdr.restore_cpu_addr;
392 end = start + PAGE_SIZE;
393
394 ret = temp_pgtable_mapping(pgdp: resume_pg_dir, start, end, __pgprot(_PAGE_WRITE));
395 if (ret)
396 return ret;
397
398 hibernate_restore_image(resume_hdr.saved_satp, (PFN_DOWN(__pa(resume_pg_dir)) | satp_mode),
399 resume_hdr.restore_cpu_addr);
400
401 return 0;
402}
403
404#ifdef CONFIG_PM_SLEEP_SMP
405int hibernate_resume_nonboot_cpu_disable(void)
406{
407 if (sleep_cpu < 0) {
408 pr_err("Failing to resume from hibernate on an unknown CPU\n");
409 return -ENODEV;
410 }
411
412 return freeze_secondary_cpus(primary: sleep_cpu);
413}
414#endif
415
416static int __init riscv_hibernate_init(void)
417{
418 hibernate_cpu_context = kzalloc(sizeof(*hibernate_cpu_context), GFP_KERNEL);
419
420 if (WARN_ON(!hibernate_cpu_context))
421 return -ENOMEM;
422
423 return 0;
424}
425
426early_initcall(riscv_hibernate_init);
427

source code of linux/arch/riscv/kernel/hibernate.c