1// SPDX-License-Identifier: GPL-2.0
2/*
3 * AMD Encrypted Register State Support
4 *
5 * Author: Joerg Roedel <jroedel@suse.de>
6 */
7
8/*
9 * misc.h needs to be first because it knows how to include the other kernel
10 * headers in the pre-decompression code in a way that does not break
11 * compilation.
12 */
13#include "misc.h"
14
15#include <asm/bootparam.h>
16#include <asm/pgtable_types.h>
17#include <asm/sev.h>
18#include <asm/trapnr.h>
19#include <asm/trap_pf.h>
20#include <asm/msr-index.h>
21#include <asm/fpu/xcr.h>
22#include <asm/ptrace.h>
23#include <asm/svm.h>
24#include <asm/cpuid/api.h>
25
26#include "error.h"
27#include "sev.h"
28
29static struct ghcb boot_ghcb_page __aligned(PAGE_SIZE);
30struct ghcb *boot_ghcb;
31
32#undef __init
33#define __init
34
35#undef __head
36#define __head
37
38#define __BOOT_COMPRESSED
39
40extern struct svsm_ca *boot_svsm_caa;
41extern u64 boot_svsm_caa_pa;
42
43struct svsm_ca *svsm_get_caa(void)
44{
45 return boot_svsm_caa;
46}
47
48u64 svsm_get_caa_pa(void)
49{
50 return boot_svsm_caa_pa;
51}
52
53int svsm_perform_call_protocol(struct svsm_call *call);
54
55u8 snp_vmpl;
56
57/* Include code for early handlers */
58#include "../../boot/startup/sev-shared.c"
59
60int svsm_perform_call_protocol(struct svsm_call *call)
61{
62 struct ghcb *ghcb;
63 int ret;
64
65 if (boot_ghcb)
66 ghcb = boot_ghcb;
67 else
68 ghcb = NULL;
69
70 do {
71 ret = ghcb ? svsm_perform_ghcb_protocol(ghcb, call)
72 : svsm_perform_msr_protocol(call);
73 } while (ret == -EAGAIN);
74
75 return ret;
76}
77
78static bool sev_snp_enabled(void)
79{
80 return sev_status & MSR_AMD64_SEV_SNP_ENABLED;
81}
82
83static void __page_state_change(unsigned long paddr, enum psc_op op)
84{
85 u64 val, msr;
86
87 /*
88 * If private -> shared then invalidate the page before requesting the
89 * state change in the RMP table.
90 */
91 if (op == SNP_PAGE_STATE_SHARED)
92 pvalidate_4k_page(vaddr: paddr, paddr, validate: false);
93
94 /* Save the current GHCB MSR value */
95 msr = sev_es_rd_ghcb_msr();
96
97 /* Issue VMGEXIT to change the page state in RMP table. */
98 sev_es_wr_ghcb_msr(GHCB_MSR_PSC_REQ_GFN(paddr >> PAGE_SHIFT, op));
99 VMGEXIT();
100
101 /* Read the response of the VMGEXIT. */
102 val = sev_es_rd_ghcb_msr();
103 if ((GHCB_RESP_CODE(val) != GHCB_MSR_PSC_RESP) || GHCB_MSR_PSC_RESP_VAL(val))
104 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
105
106 /* Restore the GHCB MSR value */
107 sev_es_wr_ghcb_msr(val: msr);
108
109 /*
110 * Now that page state is changed in the RMP table, validate it so that it is
111 * consistent with the RMP entry.
112 */
113 if (op == SNP_PAGE_STATE_PRIVATE)
114 pvalidate_4k_page(vaddr: paddr, paddr, validate: true);
115}
116
117void snp_set_page_private(unsigned long paddr)
118{
119 if (!sev_snp_enabled())
120 return;
121
122 __page_state_change(paddr, op: SNP_PAGE_STATE_PRIVATE);
123}
124
125void snp_set_page_shared(unsigned long paddr)
126{
127 if (!sev_snp_enabled())
128 return;
129
130 __page_state_change(paddr, op: SNP_PAGE_STATE_SHARED);
131}
132
133bool early_setup_ghcb(void)
134{
135 if (set_page_decrypted((unsigned long)&boot_ghcb_page))
136 return false;
137
138 /* Page is now mapped decrypted, clear it */
139 memset(s: &boot_ghcb_page, c: 0, n: sizeof(boot_ghcb_page));
140
141 boot_ghcb = &boot_ghcb_page;
142
143 /* Initialize lookup tables for the instruction decoder */
144 sev_insn_decode_init();
145
146 /* SNP guest requires the GHCB GPA must be registered */
147 if (sev_snp_enabled())
148 snp_register_ghcb_early(__pa(&boot_ghcb_page));
149
150 return true;
151}
152
153void snp_accept_memory(phys_addr_t start, phys_addr_t end)
154{
155 for (phys_addr_t pa = start; pa < end; pa += PAGE_SIZE)
156 __page_state_change(paddr: pa, op: SNP_PAGE_STATE_PRIVATE);
157}
158
159void sev_es_shutdown_ghcb(void)
160{
161 if (!boot_ghcb)
162 return;
163
164 if (!sev_es_check_cpu_features())
165 error(m: "SEV-ES CPU Features missing.");
166
167 /*
168 * This denotes whether to use the GHCB MSR protocol or the GHCB
169 * shared page to perform a GHCB request. Since the GHCB page is
170 * being changed to encrypted, it can't be used to perform GHCB
171 * requests. Clear the boot_ghcb variable so that the GHCB MSR
172 * protocol is used to change the GHCB page over to an encrypted
173 * page.
174 */
175 boot_ghcb = NULL;
176
177 /*
178 * GHCB Page must be flushed from the cache and mapped encrypted again.
179 * Otherwise the running kernel will see strange cache effects when
180 * trying to use that page.
181 */
182 if (set_page_encrypted((unsigned long)&boot_ghcb_page))
183 error(m: "Can't map GHCB page encrypted");
184
185 /*
186 * GHCB page is mapped encrypted again and flushed from the cache.
187 * Mark it non-present now to catch bugs when #VC exceptions trigger
188 * after this point.
189 */
190 if (set_page_non_present((unsigned long)&boot_ghcb_page))
191 error(m: "Can't unmap GHCB page");
192}
193
194static void __noreturn sev_es_ghcb_terminate(struct ghcb *ghcb, unsigned int set,
195 unsigned int reason, u64 exit_info_2)
196{
197 u64 exit_info_1 = SVM_VMGEXIT_TERM_REASON(set, reason);
198
199 vc_ghcb_invalidate(ghcb);
200 ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_TERM_REQUEST);
201 ghcb_set_sw_exit_info_1(ghcb, value: exit_info_1);
202 ghcb_set_sw_exit_info_2(ghcb, value: exit_info_2);
203
204 sev_es_wr_ghcb_msr(__pa(ghcb));
205 VMGEXIT();
206
207 while (true)
208 asm volatile("hlt\n" : : : "memory");
209}
210
211bool sev_es_check_ghcb_fault(unsigned long address)
212{
213 /* Check whether the fault was on the GHCB page */
214 return ((address & PAGE_MASK) == (unsigned long)&boot_ghcb_page);
215}
216
217/*
218 * SNP_FEATURES_IMPL_REQ is the mask of SNP features that will need
219 * guest side implementation for proper functioning of the guest. If any
220 * of these features are enabled in the hypervisor but are lacking guest
221 * side implementation, the behavior of the guest will be undefined. The
222 * guest could fail in non-obvious way making it difficult to debug.
223 *
224 * As the behavior of reserved feature bits is unknown to be on the
225 * safe side add them to the required features mask.
226 */
227#define SNP_FEATURES_IMPL_REQ (MSR_AMD64_SNP_VTOM | \
228 MSR_AMD64_SNP_REFLECT_VC | \
229 MSR_AMD64_SNP_RESTRICTED_INJ | \
230 MSR_AMD64_SNP_ALT_INJ | \
231 MSR_AMD64_SNP_DEBUG_SWAP | \
232 MSR_AMD64_SNP_VMPL_SSS | \
233 MSR_AMD64_SNP_SECURE_TSC | \
234 MSR_AMD64_SNP_VMGEXIT_PARAM | \
235 MSR_AMD64_SNP_VMSA_REG_PROT | \
236 MSR_AMD64_SNP_RESERVED_BIT13 | \
237 MSR_AMD64_SNP_RESERVED_BIT15 | \
238 MSR_AMD64_SNP_RESERVED_MASK)
239
240/*
241 * SNP_FEATURES_PRESENT is the mask of SNP features that are implemented
242 * by the guest kernel. As and when a new feature is implemented in the
243 * guest kernel, a corresponding bit should be added to the mask.
244 */
245#define SNP_FEATURES_PRESENT (MSR_AMD64_SNP_DEBUG_SWAP | \
246 MSR_AMD64_SNP_SECURE_TSC)
247
248u64 snp_get_unsupported_features(u64 status)
249{
250 if (!(status & MSR_AMD64_SEV_SNP_ENABLED))
251 return 0;
252
253 return status & SNP_FEATURES_IMPL_REQ & ~SNP_FEATURES_PRESENT;
254}
255
256void snp_check_features(void)
257{
258 u64 unsupported;
259
260 /*
261 * Terminate the boot if hypervisor has enabled any feature lacking
262 * guest side implementation. Pass on the unsupported features mask through
263 * EXIT_INFO_2 of the GHCB protocol so that those features can be reported
264 * as part of the guest boot failure.
265 */
266 unsupported = snp_get_unsupported_features(status: sev_status);
267 if (unsupported) {
268 if (ghcb_version < 2 || (!boot_ghcb && !early_setup_ghcb()))
269 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
270
271 sev_es_ghcb_terminate(ghcb: boot_ghcb, SEV_TERM_SET_GEN,
272 GHCB_SNP_UNSUPPORTED, exit_info_2: unsupported);
273 }
274}
275
276/* Search for Confidential Computing blob in the EFI config table. */
277static struct cc_blob_sev_info *find_cc_blob_efi(struct boot_params *bp)
278{
279 unsigned long cfg_table_pa;
280 unsigned int cfg_table_len;
281 int ret;
282
283 ret = efi_get_conf_table(bp, cfg_tbl_pa: &cfg_table_pa, cfg_tbl_len: &cfg_table_len);
284 if (ret)
285 return NULL;
286
287 return (struct cc_blob_sev_info *)efi_find_vendor_table(bp, cfg_tbl_pa: cfg_table_pa,
288 cfg_tbl_len: cfg_table_len,
289 EFI_CC_BLOB_GUID);
290}
291
292/*
293 * Initial set up of SNP relies on information provided by the
294 * Confidential Computing blob, which can be passed to the boot kernel
295 * by firmware/bootloader in the following ways:
296 *
297 * - via an entry in the EFI config table
298 * - via a setup_data structure, as defined by the Linux Boot Protocol
299 *
300 * Scan for the blob in that order.
301 */
302static struct cc_blob_sev_info *find_cc_blob(struct boot_params *bp)
303{
304 struct cc_blob_sev_info *cc_info;
305
306 cc_info = find_cc_blob_efi(bp);
307 if (cc_info)
308 goto found_cc_info;
309
310 cc_info = find_cc_blob_setup_data(bp);
311 if (!cc_info)
312 return NULL;
313
314found_cc_info:
315 if (cc_info->magic != CC_BLOB_SEV_HDR_MAGIC)
316 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
317
318 return cc_info;
319}
320
321/*
322 * Indicate SNP based on presence of SNP-specific CC blob. Subsequent checks
323 * will verify the SNP CPUID/MSR bits.
324 */
325static bool early_snp_init(struct boot_params *bp)
326{
327 struct cc_blob_sev_info *cc_info;
328
329 if (!bp)
330 return false;
331
332 cc_info = find_cc_blob(bp);
333 if (!cc_info)
334 return false;
335
336 /*
337 * If a SNP-specific Confidential Computing blob is present, then
338 * firmware/bootloader have indicated SNP support. Verifying this
339 * involves CPUID checks which will be more reliable if the SNP
340 * CPUID table is used. See comments over snp_setup_cpuid_table() for
341 * more details.
342 */
343 setup_cpuid_table(cc_info);
344
345 /*
346 * Record the SVSM Calling Area (CA) address if the guest is not
347 * running at VMPL0. The CA will be used to communicate with the
348 * SVSM and request its services.
349 */
350 svsm_setup_ca(cc_info);
351
352 /*
353 * Pass run-time kernel a pointer to CC info via boot_params so EFI
354 * config table doesn't need to be searched again during early startup
355 * phase.
356 */
357 bp->cc_blob_address = (u32)(unsigned long)cc_info;
358
359 return true;
360}
361
362/*
363 * sev_check_cpu_support - Check for SEV support in the CPU capabilities
364 *
365 * Returns < 0 if SEV is not supported, otherwise the position of the
366 * encryption bit in the page table descriptors.
367 */
368static int sev_check_cpu_support(void)
369{
370 unsigned int eax, ebx, ecx, edx;
371
372 /* Check for the SME/SEV support leaf */
373 eax = 0x80000000;
374 ecx = 0;
375 native_cpuid(eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
376 if (eax < 0x8000001f)
377 return -ENODEV;
378
379 /*
380 * Check for the SME/SEV feature:
381 * CPUID Fn8000_001F[EAX]
382 * - Bit 0 - Secure Memory Encryption support
383 * - Bit 1 - Secure Encrypted Virtualization support
384 * CPUID Fn8000_001F[EBX]
385 * - Bits 5:0 - Pagetable bit position used to indicate encryption
386 */
387 eax = 0x8000001f;
388 ecx = 0;
389 native_cpuid(eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
390 /* Check whether SEV is supported */
391 if (!(eax & BIT(1)))
392 return -ENODEV;
393
394 return ebx & 0x3f;
395}
396
397void sev_enable(struct boot_params *bp)
398{
399 struct msr m;
400 int bitpos;
401 bool snp;
402
403 /*
404 * bp->cc_blob_address should only be set by boot/compressed kernel.
405 * Initialize it to 0 to ensure that uninitialized values from
406 * buggy bootloaders aren't propagated.
407 */
408 if (bp)
409 bp->cc_blob_address = 0;
410
411 /*
412 * Do an initial SEV capability check before early_snp_init() which
413 * loads the CPUID page and the same checks afterwards are done
414 * without the hypervisor and are trustworthy.
415 *
416 * If the HV fakes SEV support, the guest will crash'n'burn
417 * which is good enough.
418 */
419
420 if (sev_check_cpu_support() < 0)
421 return;
422
423 /*
424 * Setup/preliminary detection of SNP. This will be sanity-checked
425 * against CPUID/MSR values later.
426 */
427 snp = early_snp_init(bp);
428
429 /* Now repeat the checks with the SNP CPUID table. */
430
431 bitpos = sev_check_cpu_support();
432 if (bitpos < 0) {
433 if (snp)
434 error(m: "SEV-SNP support indicated by CC blob, but not CPUID.");
435 return;
436 }
437
438 /* Set the SME mask if this is an SEV guest. */
439 boot_rdmsr(MSR_AMD64_SEV, m: &m);
440 sev_status = m.q;
441 if (!(sev_status & MSR_AMD64_SEV_ENABLED))
442 return;
443
444 /* Negotiate the GHCB protocol version. */
445 if (sev_status & MSR_AMD64_SEV_ES_ENABLED) {
446 if (!sev_es_negotiate_protocol())
447 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_PROT_UNSUPPORTED);
448 }
449
450 /*
451 * SNP is supported in v2 of the GHCB spec which mandates support for HV
452 * features.
453 */
454 if (sev_status & MSR_AMD64_SEV_SNP_ENABLED) {
455 u64 hv_features;
456 int ret;
457
458 hv_features = get_hv_features();
459 if (!(hv_features & GHCB_HV_FT_SNP))
460 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
461
462 /*
463 * Enforce running at VMPL0 or with an SVSM.
464 *
465 * Use RMPADJUST (see the rmpadjust() function for a description of
466 * what the instruction does) to update the VMPL1 permissions of a
467 * page. If the guest is running at VMPL0, this will succeed. If the
468 * guest is running at any other VMPL, this will fail. Linux SNP guests
469 * only ever run at a single VMPL level so permission mask changes of a
470 * lesser-privileged VMPL are a don't-care.
471 */
472 ret = rmpadjust(vaddr: (unsigned long)&boot_ghcb_page, RMP_PG_SIZE_4K, attrs: 1);
473
474 /*
475 * Running at VMPL0 is not required if an SVSM is present and the hypervisor
476 * supports the required SVSM GHCB events.
477 */
478 if (ret &&
479 !(snp_vmpl && (hv_features & GHCB_HV_FT_SNP_MULTI_VMPL)))
480 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_NOT_VMPL0);
481 }
482
483 if (snp && !(sev_status & MSR_AMD64_SEV_SNP_ENABLED))
484 error(m: "SEV-SNP supported indicated by CC blob, but not SEV status MSR.");
485
486 sme_me_mask = BIT_ULL(bitpos);
487}
488
489/*
490 * sev_get_status - Retrieve the SEV status mask
491 *
492 * Returns 0 if the CPU is not SEV capable, otherwise the value of the
493 * AMD64_SEV MSR.
494 */
495u64 sev_get_status(void)
496{
497 struct msr m;
498
499 if (sev_check_cpu_support() < 0)
500 return 0;
501
502 boot_rdmsr(MSR_AMD64_SEV, m: &m);
503 return m.q;
504}
505
506void sev_prep_identity_maps(unsigned long top_level_pgt)
507{
508 /*
509 * The Confidential Computing blob is used very early in uncompressed
510 * kernel to find the in-memory CPUID table to handle CPUID
511 * instructions. Make sure an identity-mapping exists so it can be
512 * accessed after switchover.
513 */
514 if (sev_snp_enabled()) {
515 unsigned long cc_info_pa = boot_params_ptr->cc_blob_address;
516 struct cc_blob_sev_info *cc_info;
517
518 kernel_add_identity_map(start: cc_info_pa, end: cc_info_pa + sizeof(*cc_info));
519
520 cc_info = (struct cc_blob_sev_info *)cc_info_pa;
521 kernel_add_identity_map(start: cc_info->cpuid_phys, end: cc_info->cpuid_phys + cc_info->cpuid_len);
522 }
523
524 sev_verify_cbit(cr3: top_level_pgt);
525}
526
527bool early_is_sevsnp_guest(void)
528{
529 static bool sevsnp;
530
531 if (sevsnp)
532 return true;
533
534 if (!(sev_get_status() & MSR_AMD64_SEV_SNP_ENABLED))
535 return false;
536
537 sevsnp = true;
538
539 if (!snp_vmpl) {
540 unsigned int eax, ebx, ecx, edx;
541
542 /*
543 * CPUID Fn8000_001F_EAX[28] - SVSM support
544 */
545 eax = 0x8000001f;
546 ecx = 0;
547 native_cpuid(eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
548 if (eax & BIT(28)) {
549 struct msr m;
550
551 /* Obtain the address of the calling area to use */
552 boot_rdmsr(MSR_SVSM_CAA, m: &m);
553 boot_svsm_caa = (void *)m.q;
554 boot_svsm_caa_pa = m.q;
555
556 /*
557 * The real VMPL level cannot be discovered, but the
558 * memory acceptance routines make no use of that so
559 * any non-zero value suffices here.
560 */
561 snp_vmpl = U8_MAX;
562 }
563 }
564 return true;
565}
566

source code of linux/arch/x86/boot/compressed/sev.c