1 | // SPDX-License-Identifier: GPL-2.0 |
2 | #include <linux/ftrace.h> |
3 | #include <linux/percpu.h> |
4 | #include <linux/slab.h> |
5 | #include <linux/uaccess.h> |
6 | #include <linux/pgtable.h> |
7 | #include <linux/cpuidle.h> |
8 | #include <asm/alternative.h> |
9 | #include <asm/cacheflush.h> |
10 | #include <asm/cpufeature.h> |
11 | #include <asm/cpuidle.h> |
12 | #include <asm/daifflags.h> |
13 | #include <asm/debug-monitors.h> |
14 | #include <asm/exec.h> |
15 | #include <asm/fpsimd.h> |
16 | #include <asm/mte.h> |
17 | #include <asm/memory.h> |
18 | #include <asm/mmu_context.h> |
19 | #include <asm/smp_plat.h> |
20 | #include <asm/suspend.h> |
21 | |
22 | /* |
23 | * This is allocated by cpu_suspend_init(), and used to store a pointer to |
24 | * the 'struct sleep_stack_data' the contains a particular CPUs state. |
25 | */ |
26 | unsigned long *sleep_save_stash; |
27 | |
28 | /* |
29 | * This hook is provided so that cpu_suspend code can restore HW |
30 | * breakpoints as early as possible in the resume path, before reenabling |
31 | * debug exceptions. Code cannot be run from a CPU PM notifier since by the |
32 | * time the notifier runs debug exceptions might have been enabled already, |
33 | * with HW breakpoints registers content still in an unknown state. |
34 | */ |
35 | static int (*hw_breakpoint_restore)(unsigned int); |
36 | void __init cpu_suspend_set_dbg_restorer(int (*hw_bp_restore)(unsigned int)) |
37 | { |
38 | /* Prevent multiple restore hook initializations */ |
39 | if (WARN_ON(hw_breakpoint_restore)) |
40 | return; |
41 | hw_breakpoint_restore = hw_bp_restore; |
42 | } |
43 | |
44 | void notrace __cpu_suspend_exit(void) |
45 | { |
46 | unsigned int cpu = smp_processor_id(); |
47 | |
48 | mte_suspend_exit(); |
49 | |
50 | /* |
51 | * We are resuming from reset with the idmap active in TTBR0_EL1. |
52 | * We must uninstall the idmap and restore the expected MMU |
53 | * state before we can possibly return to userspace. |
54 | */ |
55 | cpu_uninstall_idmap(); |
56 | |
57 | /* Restore CnP bit in TTBR1_EL1 */ |
58 | if (system_supports_cnp()) |
59 | cpu_enable_swapper_cnp(); |
60 | |
61 | /* |
62 | * PSTATE was not saved over suspend/resume, re-enable any detected |
63 | * features that might not have been set correctly. |
64 | */ |
65 | if (alternative_has_cap_unlikely(ARM64_HAS_DIT)) |
66 | set_pstate_dit(1); |
67 | __uaccess_enable_hw_pan(); |
68 | |
69 | /* |
70 | * Restore HW breakpoint registers to sane values |
71 | * before debug exceptions are possibly reenabled |
72 | * by cpu_suspend()s local_daif_restore() call. |
73 | */ |
74 | if (hw_breakpoint_restore) |
75 | hw_breakpoint_restore(cpu); |
76 | |
77 | /* |
78 | * On resume, firmware implementing dynamic mitigation will |
79 | * have turned the mitigation on. If the user has forcefully |
80 | * disabled it, make sure their wishes are obeyed. |
81 | */ |
82 | spectre_v4_enable_mitigation(NULL); |
83 | |
84 | sme_suspend_exit(); |
85 | |
86 | /* Restore additional feature-specific configuration */ |
87 | ptrauth_suspend_exit(); |
88 | } |
89 | |
90 | /* |
91 | * cpu_suspend |
92 | * |
93 | * arg: argument to pass to the finisher function |
94 | * fn: finisher function pointer |
95 | * |
96 | */ |
97 | int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) |
98 | { |
99 | int ret = 0; |
100 | unsigned long flags; |
101 | struct sleep_stack_data state; |
102 | struct arm_cpuidle_irq_context context; |
103 | |
104 | /* |
105 | * Some portions of CPU state (e.g. PSTATE.{PAN,DIT}) are initialized |
106 | * before alternatives are patched, but are only restored by |
107 | * __cpu_suspend_exit() after alternatives are patched. To avoid |
108 | * accidentally losing these bits we must not attempt to suspend until |
109 | * after alternatives have been patched. |
110 | */ |
111 | WARN_ON(!system_capabilities_finalized()); |
112 | |
113 | /* Report any MTE async fault before going to suspend */ |
114 | mte_suspend_enter(); |
115 | |
116 | /* |
117 | * From this point debug exceptions are disabled to prevent |
118 | * updates to mdscr register (saved and restored along with |
119 | * general purpose registers) from kernel debuggers. |
120 | * |
121 | * Strictly speaking the trace_hardirqs_off() here is superfluous, |
122 | * hardirqs should be firmly off by now. This really ought to use |
123 | * something like raw_local_daif_save(). |
124 | */ |
125 | flags = local_daif_save(); |
126 | |
127 | /* |
128 | * Function graph tracer state gets inconsistent when the kernel |
129 | * calls functions that never return (aka suspend finishers) hence |
130 | * disable graph tracing during their execution. |
131 | */ |
132 | pause_graph_tracing(); |
133 | |
134 | /* |
135 | * Switch to using DAIF.IF instead of PMR in order to reliably |
136 | * resume if we're using pseudo-NMIs. |
137 | */ |
138 | arm_cpuidle_save_irq_context(&context); |
139 | |
140 | ct_cpuidle_enter(); |
141 | |
142 | if (__cpu_suspend_enter(&state)) { |
143 | /* Call the suspend finisher */ |
144 | ret = fn(arg); |
145 | |
146 | /* |
147 | * Never gets here, unless the suspend finisher fails. |
148 | * Successful cpu_suspend() should return from cpu_resume(), |
149 | * returning through this code path is considered an error |
150 | * If the return value is set to 0 force ret = -EOPNOTSUPP |
151 | * to make sure a proper error condition is propagated |
152 | */ |
153 | if (!ret) |
154 | ret = -EOPNOTSUPP; |
155 | |
156 | ct_cpuidle_exit(); |
157 | } else { |
158 | ct_cpuidle_exit(); |
159 | __cpu_suspend_exit(); |
160 | } |
161 | |
162 | arm_cpuidle_restore_irq_context(&context); |
163 | |
164 | unpause_graph_tracing(); |
165 | |
166 | /* |
167 | * Restore pstate flags. OS lock and mdscr have been already |
168 | * restored, so from this point onwards, debugging is fully |
169 | * reenabled if it was enabled when core started shutdown. |
170 | */ |
171 | local_daif_restore(flags); |
172 | |
173 | return ret; |
174 | } |
175 | |
176 | static int __init cpu_suspend_init(void) |
177 | { |
178 | /* ctx_ptr is an array of physical addresses */ |
179 | sleep_save_stash = kcalloc(n: mpidr_hash_size(), size: sizeof(*sleep_save_stash), |
180 | GFP_KERNEL); |
181 | |
182 | if (WARN_ON(!sleep_save_stash)) |
183 | return -ENOMEM; |
184 | |
185 | return 0; |
186 | } |
187 | early_initcall(cpu_suspend_init); |
188 | |