1// SPDX-License-Identifier: GPL-2.0
2/*
3 * AMD Encrypted Register State Support
4 *
5 * Author: Joerg Roedel <jroedel@suse.de>
6 */
7
8/*
9 * misc.h needs to be first because it knows how to include the other kernel
10 * headers in the pre-decompression code in a way that does not break
11 * compilation.
12 */
13#include "misc.h"
14
15#include <asm/bootparam.h>
16#include <asm/pgtable_types.h>
17#include <asm/shared/msr.h>
18#include <asm/sev.h>
19#include <asm/trapnr.h>
20#include <asm/trap_pf.h>
21#include <asm/msr-index.h>
22#include <asm/fpu/xcr.h>
23#include <asm/ptrace.h>
24#include <asm/svm.h>
25#include <asm/cpuid/api.h>
26
27#include "error.h"
28#include "sev.h"
29
30static struct ghcb boot_ghcb_page __aligned(PAGE_SIZE);
31struct ghcb *boot_ghcb;
32
33#undef __init
34#define __init
35
36#define __BOOT_COMPRESSED
37
38u8 snp_vmpl;
39u16 ghcb_version;
40
41u64 boot_svsm_caa_pa;
42
43/* Include code for early handlers */
44#include "../../boot/startup/sev-shared.c"
45
46static bool sev_snp_enabled(void)
47{
48 return sev_status & MSR_AMD64_SEV_SNP_ENABLED;
49}
50
51void snp_set_page_private(unsigned long paddr)
52{
53 struct psc_desc d = {
54 SNP_PAGE_STATE_PRIVATE,
55 (struct svsm_ca *)boot_svsm_caa_pa,
56 boot_svsm_caa_pa
57 };
58
59 if (!sev_snp_enabled())
60 return;
61
62 __page_state_change(vaddr: paddr, paddr, desc: &d);
63}
64
65void snp_set_page_shared(unsigned long paddr)
66{
67 struct psc_desc d = {
68 SNP_PAGE_STATE_SHARED,
69 (struct svsm_ca *)boot_svsm_caa_pa,
70 boot_svsm_caa_pa
71 };
72
73 if (!sev_snp_enabled())
74 return;
75
76 __page_state_change(vaddr: paddr, paddr, desc: &d);
77}
78
79bool early_setup_ghcb(void)
80{
81 if (set_page_decrypted((unsigned long)&boot_ghcb_page))
82 return false;
83
84 /* Page is now mapped decrypted, clear it */
85 memset(s: &boot_ghcb_page, c: 0, n: sizeof(boot_ghcb_page));
86
87 boot_ghcb = &boot_ghcb_page;
88
89 /* Initialize lookup tables for the instruction decoder */
90 sev_insn_decode_init();
91
92 /* SNP guest requires the GHCB GPA must be registered */
93 if (sev_snp_enabled())
94 snp_register_ghcb_early(__pa(&boot_ghcb_page));
95
96 return true;
97}
98
99void snp_accept_memory(phys_addr_t start, phys_addr_t end)
100{
101 struct psc_desc d = {
102 SNP_PAGE_STATE_PRIVATE,
103 (struct svsm_ca *)boot_svsm_caa_pa,
104 boot_svsm_caa_pa
105 };
106
107 for (phys_addr_t pa = start; pa < end; pa += PAGE_SIZE)
108 __page_state_change(vaddr: pa, paddr: pa, desc: &d);
109}
110
111void sev_es_shutdown_ghcb(void)
112{
113 if (!boot_ghcb)
114 return;
115
116 if (!sev_es_check_cpu_features())
117 error(m: "SEV-ES CPU Features missing.");
118
119 /*
120 * This denotes whether to use the GHCB MSR protocol or the GHCB
121 * shared page to perform a GHCB request. Since the GHCB page is
122 * being changed to encrypted, it can't be used to perform GHCB
123 * requests. Clear the boot_ghcb variable so that the GHCB MSR
124 * protocol is used to change the GHCB page over to an encrypted
125 * page.
126 */
127 boot_ghcb = NULL;
128
129 /*
130 * GHCB Page must be flushed from the cache and mapped encrypted again.
131 * Otherwise the running kernel will see strange cache effects when
132 * trying to use that page.
133 */
134 if (set_page_encrypted((unsigned long)&boot_ghcb_page))
135 error(m: "Can't map GHCB page encrypted");
136
137 /*
138 * GHCB page is mapped encrypted again and flushed from the cache.
139 * Mark it non-present now to catch bugs when #VC exceptions trigger
140 * after this point.
141 */
142 if (set_page_non_present((unsigned long)&boot_ghcb_page))
143 error(m: "Can't unmap GHCB page");
144}
145
146static void __noreturn sev_es_ghcb_terminate(struct ghcb *ghcb, unsigned int set,
147 unsigned int reason, u64 exit_info_2)
148{
149 u64 exit_info_1 = SVM_VMGEXIT_TERM_REASON(set, reason);
150
151 vc_ghcb_invalidate(ghcb);
152 ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_TERM_REQUEST);
153 ghcb_set_sw_exit_info_1(ghcb, value: exit_info_1);
154 ghcb_set_sw_exit_info_2(ghcb, value: exit_info_2);
155
156 sev_es_wr_ghcb_msr(__pa(ghcb));
157 VMGEXIT();
158
159 while (true)
160 asm volatile("hlt\n" : : : "memory");
161}
162
163bool sev_es_check_ghcb_fault(unsigned long address)
164{
165 /* Check whether the fault was on the GHCB page */
166 return ((address & PAGE_MASK) == (unsigned long)&boot_ghcb_page);
167}
168
169/*
170 * SNP_FEATURES_IMPL_REQ is the mask of SNP features that will need
171 * guest side implementation for proper functioning of the guest. If any
172 * of these features are enabled in the hypervisor but are lacking guest
173 * side implementation, the behavior of the guest will be undefined. The
174 * guest could fail in non-obvious way making it difficult to debug.
175 *
176 * As the behavior of reserved feature bits is unknown to be on the
177 * safe side add them to the required features mask.
178 */
179#define SNP_FEATURES_IMPL_REQ (MSR_AMD64_SNP_VTOM | \
180 MSR_AMD64_SNP_REFLECT_VC | \
181 MSR_AMD64_SNP_RESTRICTED_INJ | \
182 MSR_AMD64_SNP_ALT_INJ | \
183 MSR_AMD64_SNP_DEBUG_SWAP | \
184 MSR_AMD64_SNP_VMPL_SSS | \
185 MSR_AMD64_SNP_SECURE_TSC | \
186 MSR_AMD64_SNP_VMGEXIT_PARAM | \
187 MSR_AMD64_SNP_VMSA_REG_PROT | \
188 MSR_AMD64_SNP_RESERVED_BIT13 | \
189 MSR_AMD64_SNP_RESERVED_BIT15 | \
190 MSR_AMD64_SNP_SECURE_AVIC | \
191 MSR_AMD64_SNP_RESERVED_MASK)
192
193#ifdef CONFIG_AMD_SECURE_AVIC
194#define SNP_FEATURE_SECURE_AVIC MSR_AMD64_SNP_SECURE_AVIC
195#else
196#define SNP_FEATURE_SECURE_AVIC 0
197#endif
198
199/*
200 * SNP_FEATURES_PRESENT is the mask of SNP features that are implemented
201 * by the guest kernel. As and when a new feature is implemented in the
202 * guest kernel, a corresponding bit should be added to the mask.
203 */
204#define SNP_FEATURES_PRESENT (MSR_AMD64_SNP_DEBUG_SWAP | \
205 MSR_AMD64_SNP_SECURE_TSC | \
206 SNP_FEATURE_SECURE_AVIC)
207
208u64 snp_get_unsupported_features(u64 status)
209{
210 if (!(status & MSR_AMD64_SEV_SNP_ENABLED))
211 return 0;
212
213 return status & SNP_FEATURES_IMPL_REQ & ~SNP_FEATURES_PRESENT;
214}
215
216void snp_check_features(void)
217{
218 u64 unsupported;
219
220 /*
221 * Terminate the boot if hypervisor has enabled any feature lacking
222 * guest side implementation. Pass on the unsupported features mask through
223 * EXIT_INFO_2 of the GHCB protocol so that those features can be reported
224 * as part of the guest boot failure.
225 */
226 unsupported = snp_get_unsupported_features(status: sev_status);
227 if (unsupported) {
228 if (ghcb_version < 2 || (!boot_ghcb && !early_setup_ghcb()))
229 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
230
231 sev_es_ghcb_terminate(ghcb: boot_ghcb, SEV_TERM_SET_GEN,
232 GHCB_SNP_UNSUPPORTED, exit_info_2: unsupported);
233 }
234}
235
236/* Search for Confidential Computing blob in the EFI config table. */
237static struct cc_blob_sev_info *find_cc_blob_efi(struct boot_params *bp)
238{
239 unsigned long cfg_table_pa;
240 unsigned int cfg_table_len;
241 int ret;
242
243 ret = efi_get_conf_table(bp, cfg_tbl_pa: &cfg_table_pa, cfg_tbl_len: &cfg_table_len);
244 if (ret)
245 return NULL;
246
247 return (struct cc_blob_sev_info *)efi_find_vendor_table(bp, cfg_tbl_pa: cfg_table_pa,
248 cfg_tbl_len: cfg_table_len,
249 EFI_CC_BLOB_GUID);
250}
251
252/*
253 * Initial set up of SNP relies on information provided by the
254 * Confidential Computing blob, which can be passed to the boot kernel
255 * by firmware/bootloader in the following ways:
256 *
257 * - via an entry in the EFI config table
258 * - via a setup_data structure, as defined by the Linux Boot Protocol
259 *
260 * Scan for the blob in that order.
261 */
262static struct cc_blob_sev_info *find_cc_blob(struct boot_params *bp)
263{
264 struct cc_blob_sev_info *cc_info;
265
266 cc_info = find_cc_blob_efi(bp);
267 if (cc_info)
268 goto found_cc_info;
269
270 cc_info = find_cc_blob_setup_data(bp);
271 if (!cc_info)
272 return NULL;
273
274found_cc_info:
275 if (cc_info->magic != CC_BLOB_SEV_HDR_MAGIC)
276 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
277
278 return cc_info;
279}
280
281/*
282 * Indicate SNP based on presence of SNP-specific CC blob. Subsequent checks
283 * will verify the SNP CPUID/MSR bits.
284 */
285static bool early_snp_init(struct boot_params *bp)
286{
287 struct cc_blob_sev_info *cc_info;
288
289 if (!bp)
290 return false;
291
292 cc_info = find_cc_blob(bp);
293 if (!cc_info)
294 return false;
295
296 /*
297 * If a SNP-specific Confidential Computing blob is present, then
298 * firmware/bootloader have indicated SNP support. Verifying this
299 * involves CPUID checks which will be more reliable if the SNP
300 * CPUID table is used. See comments over snp_setup_cpuid_table() for
301 * more details.
302 */
303 setup_cpuid_table(cc_info);
304
305 /*
306 * Record the SVSM Calling Area (CA) address if the guest is not
307 * running at VMPL0. The CA will be used to communicate with the
308 * SVSM and request its services.
309 */
310 svsm_setup_ca(cc_info, page: rip_rel_ptr(p: &boot_ghcb_page));
311
312 /*
313 * Pass run-time kernel a pointer to CC info via boot_params so EFI
314 * config table doesn't need to be searched again during early startup
315 * phase.
316 */
317 bp->cc_blob_address = (u32)(unsigned long)cc_info;
318
319 return true;
320}
321
322/*
323 * sev_check_cpu_support - Check for SEV support in the CPU capabilities
324 *
325 * Returns < 0 if SEV is not supported, otherwise the position of the
326 * encryption bit in the page table descriptors.
327 */
328static int sev_check_cpu_support(void)
329{
330 unsigned int eax, ebx, ecx, edx;
331
332 /* Check for the SME/SEV support leaf */
333 eax = 0x80000000;
334 ecx = 0;
335 native_cpuid(eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
336 if (eax < 0x8000001f)
337 return -ENODEV;
338
339 /*
340 * Check for the SME/SEV feature:
341 * CPUID Fn8000_001F[EAX]
342 * - Bit 0 - Secure Memory Encryption support
343 * - Bit 1 - Secure Encrypted Virtualization support
344 * CPUID Fn8000_001F[EBX]
345 * - Bits 5:0 - Pagetable bit position used to indicate encryption
346 */
347 eax = 0x8000001f;
348 ecx = 0;
349 native_cpuid(eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
350 /* Check whether SEV is supported */
351 if (!(eax & BIT(1)))
352 return -ENODEV;
353
354 sev_snp_needs_sfw = !(ebx & BIT(31));
355
356 return ebx & 0x3f;
357}
358
359void sev_enable(struct boot_params *bp)
360{
361 struct msr m;
362 int bitpos;
363 bool snp;
364
365 /*
366 * bp->cc_blob_address should only be set by boot/compressed kernel.
367 * Initialize it to 0 to ensure that uninitialized values from
368 * buggy bootloaders aren't propagated.
369 */
370 if (bp)
371 bp->cc_blob_address = 0;
372
373 /*
374 * Do an initial SEV capability check before early_snp_init() which
375 * loads the CPUID page and the same checks afterwards are done
376 * without the hypervisor and are trustworthy.
377 *
378 * If the HV fakes SEV support, the guest will crash'n'burn
379 * which is good enough.
380 */
381
382 if (sev_check_cpu_support() < 0)
383 return;
384
385 /*
386 * Setup/preliminary detection of SNP. This will be sanity-checked
387 * against CPUID/MSR values later.
388 */
389 snp = early_snp_init(bp);
390
391 /* Now repeat the checks with the SNP CPUID table. */
392
393 bitpos = sev_check_cpu_support();
394 if (bitpos < 0) {
395 if (snp)
396 error(m: "SEV-SNP support indicated by CC blob, but not CPUID.");
397 return;
398 }
399
400 /* Set the SME mask if this is an SEV guest. */
401 raw_rdmsr(MSR_AMD64_SEV, m: &m);
402 sev_status = m.q;
403 if (!(sev_status & MSR_AMD64_SEV_ENABLED))
404 return;
405
406 /* Negotiate the GHCB protocol version. */
407 if (sev_status & MSR_AMD64_SEV_ES_ENABLED) {
408 if (!sev_es_negotiate_protocol())
409 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_PROT_UNSUPPORTED);
410 }
411
412 /*
413 * SNP is supported in v2 of the GHCB spec which mandates support for HV
414 * features.
415 */
416 if (sev_status & MSR_AMD64_SEV_SNP_ENABLED) {
417 u64 hv_features;
418
419 hv_features = get_hv_features();
420 if (!(hv_features & GHCB_HV_FT_SNP))
421 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
422
423 /*
424 * Running at VMPL0 is required unless an SVSM is present and
425 * the hypervisor supports the required SVSM GHCB events.
426 */
427 if (snp_vmpl && !(hv_features & GHCB_HV_FT_SNP_MULTI_VMPL))
428 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_NOT_VMPL0);
429 }
430
431 if (snp && !(sev_status & MSR_AMD64_SEV_SNP_ENABLED))
432 error(m: "SEV-SNP supported indicated by CC blob, but not SEV status MSR.");
433
434 sme_me_mask = BIT_ULL(bitpos);
435}
436
437/*
438 * sev_get_status - Retrieve the SEV status mask
439 *
440 * Returns 0 if the CPU is not SEV capable, otherwise the value of the
441 * AMD64_SEV MSR.
442 */
443u64 sev_get_status(void)
444{
445 struct msr m;
446
447 if (sev_check_cpu_support() < 0)
448 return 0;
449
450 raw_rdmsr(MSR_AMD64_SEV, m: &m);
451 return m.q;
452}
453
454void sev_prep_identity_maps(unsigned long top_level_pgt)
455{
456 /*
457 * The Confidential Computing blob is used very early in uncompressed
458 * kernel to find the in-memory CPUID table to handle CPUID
459 * instructions. Make sure an identity-mapping exists so it can be
460 * accessed after switchover.
461 */
462 if (sev_snp_enabled()) {
463 unsigned long cc_info_pa = boot_params_ptr->cc_blob_address;
464 struct cc_blob_sev_info *cc_info;
465
466 kernel_add_identity_map(start: cc_info_pa, end: cc_info_pa + sizeof(*cc_info));
467
468 cc_info = (struct cc_blob_sev_info *)cc_info_pa;
469 kernel_add_identity_map(start: cc_info->cpuid_phys, end: cc_info->cpuid_phys + cc_info->cpuid_len);
470 }
471
472 sev_verify_cbit(cr3: top_level_pgt);
473}
474
475bool early_is_sevsnp_guest(void)
476{
477 static bool sevsnp;
478
479 if (sevsnp)
480 return true;
481
482 if (!(sev_get_status() & MSR_AMD64_SEV_SNP_ENABLED))
483 return false;
484
485 sevsnp = true;
486
487 if (!snp_vmpl) {
488 unsigned int eax, ebx, ecx, edx;
489
490 /*
491 * CPUID Fn8000_001F_EAX[28] - SVSM support
492 */
493 eax = 0x8000001f;
494 ecx = 0;
495 native_cpuid(eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
496 if (eax & BIT(28)) {
497 struct msr m;
498
499 /* Obtain the address of the calling area to use */
500 raw_rdmsr(MSR_SVSM_CAA, m: &m);
501 boot_svsm_caa_pa = m.q;
502
503 /*
504 * The real VMPL level cannot be discovered, but the
505 * memory acceptance routines make no use of that so
506 * any non-zero value suffices here.
507 */
508 snp_vmpl = U8_MAX;
509 }
510 }
511 return true;
512}
513

source code of linux/arch/x86/boot/compressed/sev.c