1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright (C) 2012 - ARM Ltd |
4 | * Author: Marc Zyngier <marc.zyngier@arm.com> |
5 | */ |
6 | |
7 | #include <linux/arm-smccc.h> |
8 | #include <linux/preempt.h> |
9 | #include <linux/kvm_host.h> |
10 | #include <linux/uaccess.h> |
11 | #include <linux/wait.h> |
12 | |
13 | #include <asm/cputype.h> |
14 | #include <asm/kvm_emulate.h> |
15 | |
16 | #include <kvm/arm_psci.h> |
17 | #include <kvm/arm_hypercalls.h> |
18 | |
19 | /* |
20 | * This is an implementation of the Power State Coordination Interface |
21 | * as described in ARM document number ARM DEN 0022A. |
22 | */ |
23 | |
24 | #define AFFINITY_MASK(level) ~((0x1UL << ((level) * MPIDR_LEVEL_BITS)) - 1) |
25 | |
26 | static unsigned long psci_affinity_mask(unsigned long affinity_level) |
27 | { |
28 | if (affinity_level <= 3) |
29 | return MPIDR_HWID_BITMASK & AFFINITY_MASK(affinity_level); |
30 | |
31 | return 0; |
32 | } |
33 | |
34 | static unsigned long kvm_psci_vcpu_suspend(struct kvm_vcpu *vcpu) |
35 | { |
36 | /* |
37 | * NOTE: For simplicity, we make VCPU suspend emulation to be |
38 | * same-as WFI (Wait-for-interrupt) emulation. |
39 | * |
40 | * This means for KVM the wakeup events are interrupts and |
41 | * this is consistent with intended use of StateID as described |
42 | * in section 5.4.1 of PSCI v0.2 specification (ARM DEN 0022A). |
43 | * |
44 | * Further, we also treat power-down request to be same as |
45 | * stand-by request as-per section 5.4.2 clause 3 of PSCI v0.2 |
46 | * specification (ARM DEN 0022A). This means all suspend states |
47 | * for KVM will preserve the register state. |
48 | */ |
49 | kvm_vcpu_wfi(vcpu); |
50 | |
51 | return PSCI_RET_SUCCESS; |
52 | } |
53 | |
54 | static inline bool kvm_psci_valid_affinity(struct kvm_vcpu *vcpu, |
55 | unsigned long affinity) |
56 | { |
57 | return !(affinity & ~MPIDR_HWID_BITMASK); |
58 | } |
59 | |
60 | static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu) |
61 | { |
62 | struct vcpu_reset_state *reset_state; |
63 | struct kvm *kvm = source_vcpu->kvm; |
64 | struct kvm_vcpu *vcpu = NULL; |
65 | int ret = PSCI_RET_SUCCESS; |
66 | unsigned long cpu_id; |
67 | |
68 | cpu_id = smccc_get_arg1(vcpu: source_vcpu); |
69 | if (!kvm_psci_valid_affinity(vcpu: source_vcpu, affinity: cpu_id)) |
70 | return PSCI_RET_INVALID_PARAMS; |
71 | |
72 | vcpu = kvm_mpidr_to_vcpu(kvm, cpu_id); |
73 | |
74 | /* |
75 | * Make sure the caller requested a valid CPU and that the CPU is |
76 | * turned off. |
77 | */ |
78 | if (!vcpu) |
79 | return PSCI_RET_INVALID_PARAMS; |
80 | |
81 | spin_lock(lock: &vcpu->arch.mp_state_lock); |
82 | if (!kvm_arm_vcpu_stopped(vcpu)) { |
83 | if (kvm_psci_version(vcpu: source_vcpu) != KVM_ARM_PSCI_0_1) |
84 | ret = PSCI_RET_ALREADY_ON; |
85 | else |
86 | ret = PSCI_RET_INVALID_PARAMS; |
87 | |
88 | goto out_unlock; |
89 | } |
90 | |
91 | reset_state = &vcpu->arch.reset_state; |
92 | |
93 | reset_state->pc = smccc_get_arg2(vcpu: source_vcpu); |
94 | |
95 | /* Propagate caller endianness */ |
96 | reset_state->be = kvm_vcpu_is_be(source_vcpu); |
97 | |
98 | /* |
99 | * NOTE: We always update r0 (or x0) because for PSCI v0.1 |
100 | * the general purpose registers are undefined upon CPU_ON. |
101 | */ |
102 | reset_state->r0 = smccc_get_arg3(vcpu: source_vcpu); |
103 | |
104 | reset_state->reset = true; |
105 | kvm_make_request(req: KVM_REQ_VCPU_RESET, vcpu); |
106 | |
107 | /* |
108 | * Make sure the reset request is observed if the RUNNABLE mp_state is |
109 | * observed. |
110 | */ |
111 | smp_wmb(); |
112 | |
113 | WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_RUNNABLE); |
114 | kvm_vcpu_wake_up(vcpu); |
115 | |
116 | out_unlock: |
117 | spin_unlock(lock: &vcpu->arch.mp_state_lock); |
118 | return ret; |
119 | } |
120 | |
121 | static unsigned long kvm_psci_vcpu_affinity_info(struct kvm_vcpu *vcpu) |
122 | { |
123 | int matching_cpus = 0; |
124 | unsigned long i, mpidr; |
125 | unsigned long target_affinity; |
126 | unsigned long target_affinity_mask; |
127 | unsigned long lowest_affinity_level; |
128 | struct kvm *kvm = vcpu->kvm; |
129 | struct kvm_vcpu *tmp; |
130 | |
131 | target_affinity = smccc_get_arg1(vcpu); |
132 | lowest_affinity_level = smccc_get_arg2(vcpu); |
133 | |
134 | if (!kvm_psci_valid_affinity(vcpu, affinity: target_affinity)) |
135 | return PSCI_RET_INVALID_PARAMS; |
136 | |
137 | /* Determine target affinity mask */ |
138 | target_affinity_mask = psci_affinity_mask(affinity_level: lowest_affinity_level); |
139 | if (!target_affinity_mask) |
140 | return PSCI_RET_INVALID_PARAMS; |
141 | |
142 | /* Ignore other bits of target affinity */ |
143 | target_affinity &= target_affinity_mask; |
144 | |
145 | /* |
146 | * If one or more VCPU matching target affinity are running |
147 | * then ON else OFF |
148 | */ |
149 | kvm_for_each_vcpu(i, tmp, kvm) { |
150 | mpidr = kvm_vcpu_get_mpidr_aff(tmp); |
151 | if ((mpidr & target_affinity_mask) == target_affinity) { |
152 | matching_cpus++; |
153 | if (!kvm_arm_vcpu_stopped(tmp)) |
154 | return PSCI_0_2_AFFINITY_LEVEL_ON; |
155 | } |
156 | } |
157 | |
158 | if (!matching_cpus) |
159 | return PSCI_RET_INVALID_PARAMS; |
160 | |
161 | return PSCI_0_2_AFFINITY_LEVEL_OFF; |
162 | } |
163 | |
164 | static void kvm_prepare_system_event(struct kvm_vcpu *vcpu, u32 type, u64 flags) |
165 | { |
166 | unsigned long i; |
167 | struct kvm_vcpu *tmp; |
168 | |
169 | /* |
170 | * The KVM ABI specifies that a system event exit may call KVM_RUN |
171 | * again and may perform shutdown/reboot at a later time that when the |
172 | * actual request is made. Since we are implementing PSCI and a |
173 | * caller of PSCI reboot and shutdown expects that the system shuts |
174 | * down or reboots immediately, let's make sure that VCPUs are not run |
175 | * after this call is handled and before the VCPUs have been |
176 | * re-initialized. |
177 | */ |
178 | kvm_for_each_vcpu(i, tmp, vcpu->kvm) { |
179 | spin_lock(lock: &tmp->arch.mp_state_lock); |
180 | WRITE_ONCE(tmp->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED); |
181 | spin_unlock(lock: &tmp->arch.mp_state_lock); |
182 | } |
183 | kvm_make_all_cpus_request(kvm: vcpu->kvm, req: KVM_REQ_SLEEP); |
184 | |
185 | memset(&vcpu->run->system_event, 0, sizeof(vcpu->run->system_event)); |
186 | vcpu->run->system_event.type = type; |
187 | vcpu->run->system_event.ndata = 1; |
188 | vcpu->run->system_event.data[0] = flags; |
189 | vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT; |
190 | } |
191 | |
192 | static void kvm_psci_system_off(struct kvm_vcpu *vcpu) |
193 | { |
194 | kvm_prepare_system_event(vcpu, KVM_SYSTEM_EVENT_SHUTDOWN, flags: 0); |
195 | } |
196 | |
197 | static void kvm_psci_system_reset(struct kvm_vcpu *vcpu) |
198 | { |
199 | kvm_prepare_system_event(vcpu, KVM_SYSTEM_EVENT_RESET, flags: 0); |
200 | } |
201 | |
202 | static void kvm_psci_system_reset2(struct kvm_vcpu *vcpu) |
203 | { |
204 | kvm_prepare_system_event(vcpu, KVM_SYSTEM_EVENT_RESET, |
205 | flags: KVM_SYSTEM_EVENT_RESET_FLAG_PSCI_RESET2); |
206 | } |
207 | |
208 | static void kvm_psci_system_suspend(struct kvm_vcpu *vcpu) |
209 | { |
210 | struct kvm_run *run = vcpu->run; |
211 | |
212 | memset(&run->system_event, 0, sizeof(vcpu->run->system_event)); |
213 | run->system_event.type = KVM_SYSTEM_EVENT_SUSPEND; |
214 | run->exit_reason = KVM_EXIT_SYSTEM_EVENT; |
215 | } |
216 | |
217 | static void kvm_psci_narrow_to_32bit(struct kvm_vcpu *vcpu) |
218 | { |
219 | int i; |
220 | |
221 | /* |
222 | * Zero the input registers' upper 32 bits. They will be fully |
223 | * zeroed on exit, so we're fine changing them in place. |
224 | */ |
225 | for (i = 1; i < 4; i++) |
226 | vcpu_set_reg(vcpu, i, lower_32_bits(vcpu_get_reg(vcpu, i))); |
227 | } |
228 | |
229 | static unsigned long kvm_psci_check_allowed_function(struct kvm_vcpu *vcpu, u32 fn) |
230 | { |
231 | /* |
232 | * Prevent 32 bit guests from calling 64 bit PSCI functions. |
233 | */ |
234 | if ((fn & PSCI_0_2_64BIT) && vcpu_mode_is_32bit(vcpu)) |
235 | return PSCI_RET_NOT_SUPPORTED; |
236 | |
237 | return 0; |
238 | } |
239 | |
240 | static int kvm_psci_0_2_call(struct kvm_vcpu *vcpu) |
241 | { |
242 | u32 psci_fn = smccc_get_function(vcpu); |
243 | unsigned long val; |
244 | int ret = 1; |
245 | |
246 | switch (psci_fn) { |
247 | case PSCI_0_2_FN_PSCI_VERSION: |
248 | /* |
249 | * Bits[31:16] = Major Version = 0 |
250 | * Bits[15:0] = Minor Version = 2 |
251 | */ |
252 | val = KVM_ARM_PSCI_0_2; |
253 | break; |
254 | case PSCI_0_2_FN_CPU_SUSPEND: |
255 | case PSCI_0_2_FN64_CPU_SUSPEND: |
256 | val = kvm_psci_vcpu_suspend(vcpu); |
257 | break; |
258 | case PSCI_0_2_FN_CPU_OFF: |
259 | kvm_arm_vcpu_power_off(vcpu); |
260 | val = PSCI_RET_SUCCESS; |
261 | break; |
262 | case PSCI_0_2_FN_CPU_ON: |
263 | kvm_psci_narrow_to_32bit(vcpu); |
264 | fallthrough; |
265 | case PSCI_0_2_FN64_CPU_ON: |
266 | val = kvm_psci_vcpu_on(source_vcpu: vcpu); |
267 | break; |
268 | case PSCI_0_2_FN_AFFINITY_INFO: |
269 | kvm_psci_narrow_to_32bit(vcpu); |
270 | fallthrough; |
271 | case PSCI_0_2_FN64_AFFINITY_INFO: |
272 | val = kvm_psci_vcpu_affinity_info(vcpu); |
273 | break; |
274 | case PSCI_0_2_FN_MIGRATE_INFO_TYPE: |
275 | /* |
276 | * Trusted OS is MP hence does not require migration |
277 | * or |
278 | * Trusted OS is not present |
279 | */ |
280 | val = PSCI_0_2_TOS_MP; |
281 | break; |
282 | case PSCI_0_2_FN_SYSTEM_OFF: |
283 | kvm_psci_system_off(vcpu); |
284 | /* |
285 | * We shouldn't be going back to guest VCPU after |
286 | * receiving SYSTEM_OFF request. |
287 | * |
288 | * If user space accidentally/deliberately resumes |
289 | * guest VCPU after SYSTEM_OFF request then guest |
290 | * VCPU should see internal failure from PSCI return |
291 | * value. To achieve this, we preload r0 (or x0) with |
292 | * PSCI return value INTERNAL_FAILURE. |
293 | */ |
294 | val = PSCI_RET_INTERNAL_FAILURE; |
295 | ret = 0; |
296 | break; |
297 | case PSCI_0_2_FN_SYSTEM_RESET: |
298 | kvm_psci_system_reset(vcpu); |
299 | /* |
300 | * Same reason as SYSTEM_OFF for preloading r0 (or x0) |
301 | * with PSCI return value INTERNAL_FAILURE. |
302 | */ |
303 | val = PSCI_RET_INTERNAL_FAILURE; |
304 | ret = 0; |
305 | break; |
306 | default: |
307 | val = PSCI_RET_NOT_SUPPORTED; |
308 | break; |
309 | } |
310 | |
311 | smccc_set_retval(vcpu, a0: val, a1: 0, a2: 0, a3: 0); |
312 | return ret; |
313 | } |
314 | |
315 | static int kvm_psci_1_x_call(struct kvm_vcpu *vcpu, u32 minor) |
316 | { |
317 | unsigned long val = PSCI_RET_NOT_SUPPORTED; |
318 | u32 psci_fn = smccc_get_function(vcpu); |
319 | struct kvm *kvm = vcpu->kvm; |
320 | u32 arg; |
321 | int ret = 1; |
322 | |
323 | switch(psci_fn) { |
324 | case PSCI_0_2_FN_PSCI_VERSION: |
325 | val = minor == 0 ? KVM_ARM_PSCI_1_0 : KVM_ARM_PSCI_1_1; |
326 | break; |
327 | case PSCI_1_0_FN_PSCI_FEATURES: |
328 | arg = smccc_get_arg1(vcpu); |
329 | val = kvm_psci_check_allowed_function(vcpu, fn: arg); |
330 | if (val) |
331 | break; |
332 | |
333 | val = PSCI_RET_NOT_SUPPORTED; |
334 | |
335 | switch(arg) { |
336 | case PSCI_0_2_FN_PSCI_VERSION: |
337 | case PSCI_0_2_FN_CPU_SUSPEND: |
338 | case PSCI_0_2_FN64_CPU_SUSPEND: |
339 | case PSCI_0_2_FN_CPU_OFF: |
340 | case PSCI_0_2_FN_CPU_ON: |
341 | case PSCI_0_2_FN64_CPU_ON: |
342 | case PSCI_0_2_FN_AFFINITY_INFO: |
343 | case PSCI_0_2_FN64_AFFINITY_INFO: |
344 | case PSCI_0_2_FN_MIGRATE_INFO_TYPE: |
345 | case PSCI_0_2_FN_SYSTEM_OFF: |
346 | case PSCI_0_2_FN_SYSTEM_RESET: |
347 | case PSCI_1_0_FN_PSCI_FEATURES: |
348 | case ARM_SMCCC_VERSION_FUNC_ID: |
349 | val = 0; |
350 | break; |
351 | case PSCI_1_0_FN_SYSTEM_SUSPEND: |
352 | case PSCI_1_0_FN64_SYSTEM_SUSPEND: |
353 | if (test_bit(KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, &kvm->arch.flags)) |
354 | val = 0; |
355 | break; |
356 | case PSCI_1_1_FN_SYSTEM_RESET2: |
357 | case PSCI_1_1_FN64_SYSTEM_RESET2: |
358 | if (minor >= 1) |
359 | val = 0; |
360 | break; |
361 | } |
362 | break; |
363 | case PSCI_1_0_FN_SYSTEM_SUSPEND: |
364 | kvm_psci_narrow_to_32bit(vcpu); |
365 | fallthrough; |
366 | case PSCI_1_0_FN64_SYSTEM_SUSPEND: |
367 | /* |
368 | * Return directly to userspace without changing the vCPU's |
369 | * registers. Userspace depends on reading the SMCCC parameters |
370 | * to implement SYSTEM_SUSPEND. |
371 | */ |
372 | if (test_bit(KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, &kvm->arch.flags)) { |
373 | kvm_psci_system_suspend(vcpu); |
374 | return 0; |
375 | } |
376 | break; |
377 | case PSCI_1_1_FN_SYSTEM_RESET2: |
378 | kvm_psci_narrow_to_32bit(vcpu); |
379 | fallthrough; |
380 | case PSCI_1_1_FN64_SYSTEM_RESET2: |
381 | if (minor >= 1) { |
382 | arg = smccc_get_arg1(vcpu); |
383 | |
384 | if (arg <= PSCI_1_1_RESET_TYPE_SYSTEM_WARM_RESET || |
385 | arg >= PSCI_1_1_RESET_TYPE_VENDOR_START) { |
386 | kvm_psci_system_reset2(vcpu); |
387 | vcpu_set_reg(vcpu, 0, PSCI_RET_INTERNAL_FAILURE); |
388 | return 0; |
389 | } |
390 | |
391 | val = PSCI_RET_INVALID_PARAMS; |
392 | break; |
393 | } |
394 | break; |
395 | default: |
396 | return kvm_psci_0_2_call(vcpu); |
397 | } |
398 | |
399 | smccc_set_retval(vcpu, a0: val, a1: 0, a2: 0, a3: 0); |
400 | return ret; |
401 | } |
402 | |
403 | static int kvm_psci_0_1_call(struct kvm_vcpu *vcpu) |
404 | { |
405 | u32 psci_fn = smccc_get_function(vcpu); |
406 | unsigned long val; |
407 | |
408 | switch (psci_fn) { |
409 | case KVM_PSCI_FN_CPU_OFF: |
410 | kvm_arm_vcpu_power_off(vcpu); |
411 | val = PSCI_RET_SUCCESS; |
412 | break; |
413 | case KVM_PSCI_FN_CPU_ON: |
414 | val = kvm_psci_vcpu_on(source_vcpu: vcpu); |
415 | break; |
416 | default: |
417 | val = PSCI_RET_NOT_SUPPORTED; |
418 | break; |
419 | } |
420 | |
421 | smccc_set_retval(vcpu, a0: val, a1: 0, a2: 0, a3: 0); |
422 | return 1; |
423 | } |
424 | |
425 | /** |
426 | * kvm_psci_call - handle PSCI call if r0 value is in range |
427 | * @vcpu: Pointer to the VCPU struct |
428 | * |
429 | * Handle PSCI calls from guests through traps from HVC instructions. |
430 | * The calling convention is similar to SMC calls to the secure world |
431 | * where the function number is placed in r0. |
432 | * |
433 | * This function returns: > 0 (success), 0 (success but exit to user |
434 | * space), and < 0 (errors) |
435 | * |
436 | * Errors: |
437 | * -EINVAL: Unrecognized PSCI function |
438 | */ |
439 | int kvm_psci_call(struct kvm_vcpu *vcpu) |
440 | { |
441 | u32 psci_fn = smccc_get_function(vcpu); |
442 | int version = kvm_psci_version(vcpu); |
443 | unsigned long val; |
444 | |
445 | val = kvm_psci_check_allowed_function(vcpu, fn: psci_fn); |
446 | if (val) { |
447 | smccc_set_retval(vcpu, a0: val, a1: 0, a2: 0, a3: 0); |
448 | return 1; |
449 | } |
450 | |
451 | switch (version) { |
452 | case KVM_ARM_PSCI_1_1: |
453 | return kvm_psci_1_x_call(vcpu, minor: 1); |
454 | case KVM_ARM_PSCI_1_0: |
455 | return kvm_psci_1_x_call(vcpu, minor: 0); |
456 | case KVM_ARM_PSCI_0_2: |
457 | return kvm_psci_0_2_call(vcpu); |
458 | case KVM_ARM_PSCI_0_1: |
459 | return kvm_psci_0_1_call(vcpu); |
460 | default: |
461 | WARN_ONCE(1, "Unknown PSCI version %d" , version); |
462 | smccc_set_retval(vcpu, SMCCC_RET_NOT_SUPPORTED, a1: 0, a2: 0, a3: 0); |
463 | return 1; |
464 | } |
465 | } |
466 | |