1// SPDX-License-Identifier: GPL-2.0
2/*
3 * AMD Encrypted Register State Support
4 *
5 * Author: Joerg Roedel <jroedel@suse.de>
6 */
7
8/*
9 * misc.h needs to be first because it knows how to include the other kernel
10 * headers in the pre-decompression code in a way that does not break
11 * compilation.
12 */
13#include "misc.h"
14
15#include <asm/bootparam.h>
16#include <asm/pgtable_types.h>
17#include <asm/sev.h>
18#include <asm/trapnr.h>
19#include <asm/trap_pf.h>
20#include <asm/msr-index.h>
21#include <asm/fpu/xcr.h>
22#include <asm/ptrace.h>
23#include <asm/svm.h>
24#include <asm/cpuid.h>
25
26#include "error.h"
27#include "../msr.h"
28
29static struct ghcb boot_ghcb_page __aligned(PAGE_SIZE);
30struct ghcb *boot_ghcb;
31
32/*
33 * Copy a version of this function here - insn-eval.c can't be used in
34 * pre-decompression code.
35 */
36static bool insn_has_rep_prefix(struct insn *insn)
37{
38 insn_byte_t p;
39 int i;
40
41 insn_get_prefixes(insn);
42
43 for_each_insn_prefix(insn, i, p) {
44 if (p == 0xf2 || p == 0xf3)
45 return true;
46 }
47
48 return false;
49}
50
51/*
52 * Only a dummy for insn_get_seg_base() - Early boot-code is 64bit only and
53 * doesn't use segments.
54 */
55static unsigned long insn_get_seg_base(struct pt_regs *regs, int seg_reg_idx)
56{
57 return 0UL;
58}
59
60static inline u64 sev_es_rd_ghcb_msr(void)
61{
62 struct msr m;
63
64 boot_rdmsr(MSR_AMD64_SEV_ES_GHCB, m: &m);
65
66 return m.q;
67}
68
69static inline void sev_es_wr_ghcb_msr(u64 val)
70{
71 struct msr m;
72
73 m.q = val;
74 boot_wrmsr(MSR_AMD64_SEV_ES_GHCB, m: &m);
75}
76
77static enum es_result vc_decode_insn(struct es_em_ctxt *ctxt)
78{
79 char buffer[MAX_INSN_SIZE];
80 int ret;
81
82 memcpy(to: buffer, from: (unsigned char *)ctxt->regs->ip, MAX_INSN_SIZE);
83
84 ret = insn_decode(insn: &ctxt->insn, kaddr: buffer, MAX_INSN_SIZE, m: INSN_MODE_64);
85 if (ret < 0)
86 return ES_DECODE_FAILED;
87
88 return ES_OK;
89}
90
91static enum es_result vc_write_mem(struct es_em_ctxt *ctxt,
92 void *dst, char *buf, size_t size)
93{
94 memcpy(to: dst, from: buf, len: size);
95
96 return ES_OK;
97}
98
99static enum es_result vc_read_mem(struct es_em_ctxt *ctxt,
100 void *src, char *buf, size_t size)
101{
102 memcpy(to: buf, from: src, len: size);
103
104 return ES_OK;
105}
106
107static enum es_result vc_ioio_check(struct es_em_ctxt *ctxt, u16 port, size_t size)
108{
109 return ES_OK;
110}
111
112static bool fault_in_kernel_space(unsigned long address)
113{
114 return false;
115}
116
117#undef __init
118#define __init
119
120#undef __head
121#define __head
122
123#define __BOOT_COMPRESSED
124
125/* Basic instruction decoding support needed */
126#include "../../lib/inat.c"
127#include "../../lib/insn.c"
128
129/* Include code for early handlers */
130#include "../../kernel/sev-shared.c"
131
132bool sev_snp_enabled(void)
133{
134 return sev_status & MSR_AMD64_SEV_SNP_ENABLED;
135}
136
137static void __page_state_change(unsigned long paddr, enum psc_op op)
138{
139 u64 val;
140
141 if (!sev_snp_enabled())
142 return;
143
144 /*
145 * If private -> shared then invalidate the page before requesting the
146 * state change in the RMP table.
147 */
148 if (op == SNP_PAGE_STATE_SHARED && pvalidate(vaddr: paddr, RMP_PG_SIZE_4K, validate: 0))
149 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE);
150
151 /* Issue VMGEXIT to change the page state in RMP table. */
152 sev_es_wr_ghcb_msr(GHCB_MSR_PSC_REQ_GFN(paddr >> PAGE_SHIFT, op));
153 VMGEXIT();
154
155 /* Read the response of the VMGEXIT. */
156 val = sev_es_rd_ghcb_msr();
157 if ((GHCB_RESP_CODE(val) != GHCB_MSR_PSC_RESP) || GHCB_MSR_PSC_RESP_VAL(val))
158 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
159
160 /*
161 * Now that page state is changed in the RMP table, validate it so that it is
162 * consistent with the RMP entry.
163 */
164 if (op == SNP_PAGE_STATE_PRIVATE && pvalidate(vaddr: paddr, RMP_PG_SIZE_4K, validate: 1))
165 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE);
166}
167
168void snp_set_page_private(unsigned long paddr)
169{
170 __page_state_change(paddr, op: SNP_PAGE_STATE_PRIVATE);
171}
172
173void snp_set_page_shared(unsigned long paddr)
174{
175 __page_state_change(paddr, op: SNP_PAGE_STATE_SHARED);
176}
177
178static bool early_setup_ghcb(void)
179{
180 if (set_page_decrypted((unsigned long)&boot_ghcb_page))
181 return false;
182
183 /* Page is now mapped decrypted, clear it */
184 memset(s: &boot_ghcb_page, c: 0, n: sizeof(boot_ghcb_page));
185
186 boot_ghcb = &boot_ghcb_page;
187
188 /* Initialize lookup tables for the instruction decoder */
189 inat_init_tables();
190
191 /* SNP guest requires the GHCB GPA must be registered */
192 if (sev_snp_enabled())
193 snp_register_ghcb_early(__pa(&boot_ghcb_page));
194
195 return true;
196}
197
198static phys_addr_t __snp_accept_memory(struct snp_psc_desc *desc,
199 phys_addr_t pa, phys_addr_t pa_end)
200{
201 struct psc_hdr *hdr;
202 struct psc_entry *e;
203 unsigned int i;
204
205 hdr = &desc->hdr;
206 memset(s: hdr, c: 0, n: sizeof(*hdr));
207
208 e = desc->entries;
209
210 i = 0;
211 while (pa < pa_end && i < VMGEXIT_PSC_MAX_ENTRY) {
212 hdr->end_entry = i;
213
214 e->gfn = pa >> PAGE_SHIFT;
215 e->operation = SNP_PAGE_STATE_PRIVATE;
216 if (IS_ALIGNED(pa, PMD_SIZE) && (pa_end - pa) >= PMD_SIZE) {
217 e->pagesize = RMP_PG_SIZE_2M;
218 pa += PMD_SIZE;
219 } else {
220 e->pagesize = RMP_PG_SIZE_4K;
221 pa += PAGE_SIZE;
222 }
223
224 e++;
225 i++;
226 }
227
228 if (vmgexit_psc(ghcb: boot_ghcb, desc))
229 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
230
231 pvalidate_pages(desc);
232
233 return pa;
234}
235
236void snp_accept_memory(phys_addr_t start, phys_addr_t end)
237{
238 struct snp_psc_desc desc = {};
239 unsigned int i;
240 phys_addr_t pa;
241
242 if (!boot_ghcb && !early_setup_ghcb())
243 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
244
245 pa = start;
246 while (pa < end)
247 pa = __snp_accept_memory(desc: &desc, pa, pa_end: end);
248}
249
250void sev_es_shutdown_ghcb(void)
251{
252 if (!boot_ghcb)
253 return;
254
255 if (!sev_es_check_cpu_features())
256 error(m: "SEV-ES CPU Features missing.");
257
258 /*
259 * GHCB Page must be flushed from the cache and mapped encrypted again.
260 * Otherwise the running kernel will see strange cache effects when
261 * trying to use that page.
262 */
263 if (set_page_encrypted((unsigned long)&boot_ghcb_page))
264 error(m: "Can't map GHCB page encrypted");
265
266 /*
267 * GHCB page is mapped encrypted again and flushed from the cache.
268 * Mark it non-present now to catch bugs when #VC exceptions trigger
269 * after this point.
270 */
271 if (set_page_non_present((unsigned long)&boot_ghcb_page))
272 error(m: "Can't unmap GHCB page");
273}
274
275static void __noreturn sev_es_ghcb_terminate(struct ghcb *ghcb, unsigned int set,
276 unsigned int reason, u64 exit_info_2)
277{
278 u64 exit_info_1 = SVM_VMGEXIT_TERM_REASON(set, reason);
279
280 vc_ghcb_invalidate(ghcb);
281 ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_TERM_REQUEST);
282 ghcb_set_sw_exit_info_1(ghcb, value: exit_info_1);
283 ghcb_set_sw_exit_info_2(ghcb, value: exit_info_2);
284
285 sev_es_wr_ghcb_msr(__pa(ghcb));
286 VMGEXIT();
287
288 while (true)
289 asm volatile("hlt\n" : : : "memory");
290}
291
292bool sev_es_check_ghcb_fault(unsigned long address)
293{
294 /* Check whether the fault was on the GHCB page */
295 return ((address & PAGE_MASK) == (unsigned long)&boot_ghcb_page);
296}
297
298void do_boot_stage2_vc(struct pt_regs *regs, unsigned long exit_code)
299{
300 struct es_em_ctxt ctxt;
301 enum es_result result;
302
303 if (!boot_ghcb && !early_setup_ghcb())
304 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
305
306 vc_ghcb_invalidate(ghcb: boot_ghcb);
307 result = vc_init_em_ctxt(ctxt: &ctxt, regs, exit_code);
308 if (result != ES_OK)
309 goto finish;
310
311 result = vc_check_opcode_bytes(ctxt: &ctxt, exit_code);
312 if (result != ES_OK)
313 goto finish;
314
315 switch (exit_code) {
316 case SVM_EXIT_RDTSC:
317 case SVM_EXIT_RDTSCP:
318 result = vc_handle_rdtsc(ghcb: boot_ghcb, ctxt: &ctxt, exit_code);
319 break;
320 case SVM_EXIT_IOIO:
321 result = vc_handle_ioio(ghcb: boot_ghcb, ctxt: &ctxt);
322 break;
323 case SVM_EXIT_CPUID:
324 result = vc_handle_cpuid(ghcb: boot_ghcb, ctxt: &ctxt);
325 break;
326 default:
327 result = ES_UNSUPPORTED;
328 break;
329 }
330
331finish:
332 if (result == ES_OK)
333 vc_finish_insn(ctxt: &ctxt);
334 else if (result != ES_RETRY)
335 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
336}
337
338static void enforce_vmpl0(void)
339{
340 u64 attrs;
341 int err;
342
343 /*
344 * RMPADJUST modifies RMP permissions of a lesser-privileged (numerically
345 * higher) privilege level. Here, clear the VMPL1 permission mask of the
346 * GHCB page. If the guest is not running at VMPL0, this will fail.
347 *
348 * If the guest is running at VMPL0, it will succeed. Even if that operation
349 * modifies permission bits, it is still ok to do so currently because Linux
350 * SNP guests are supported only on VMPL0 so VMPL1 or higher permission masks
351 * changing is a don't-care.
352 */
353 attrs = 1;
354 if (rmpadjust(vaddr: (unsigned long)&boot_ghcb_page, RMP_PG_SIZE_4K, attrs))
355 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_NOT_VMPL0);
356}
357
358/*
359 * SNP_FEATURES_IMPL_REQ is the mask of SNP features that will need
360 * guest side implementation for proper functioning of the guest. If any
361 * of these features are enabled in the hypervisor but are lacking guest
362 * side implementation, the behavior of the guest will be undefined. The
363 * guest could fail in non-obvious way making it difficult to debug.
364 *
365 * As the behavior of reserved feature bits is unknown to be on the
366 * safe side add them to the required features mask.
367 */
368#define SNP_FEATURES_IMPL_REQ (MSR_AMD64_SNP_VTOM | \
369 MSR_AMD64_SNP_REFLECT_VC | \
370 MSR_AMD64_SNP_RESTRICTED_INJ | \
371 MSR_AMD64_SNP_ALT_INJ | \
372 MSR_AMD64_SNP_DEBUG_SWAP | \
373 MSR_AMD64_SNP_VMPL_SSS | \
374 MSR_AMD64_SNP_SECURE_TSC | \
375 MSR_AMD64_SNP_VMGEXIT_PARAM | \
376 MSR_AMD64_SNP_VMSA_REG_PROT | \
377 MSR_AMD64_SNP_RESERVED_BIT13 | \
378 MSR_AMD64_SNP_RESERVED_BIT15 | \
379 MSR_AMD64_SNP_RESERVED_MASK)
380
381/*
382 * SNP_FEATURES_PRESENT is the mask of SNP features that are implemented
383 * by the guest kernel. As and when a new feature is implemented in the
384 * guest kernel, a corresponding bit should be added to the mask.
385 */
386#define SNP_FEATURES_PRESENT MSR_AMD64_SNP_DEBUG_SWAP
387
388u64 snp_get_unsupported_features(u64 status)
389{
390 if (!(status & MSR_AMD64_SEV_SNP_ENABLED))
391 return 0;
392
393 return status & SNP_FEATURES_IMPL_REQ & ~SNP_FEATURES_PRESENT;
394}
395
396void snp_check_features(void)
397{
398 u64 unsupported;
399
400 /*
401 * Terminate the boot if hypervisor has enabled any feature lacking
402 * guest side implementation. Pass on the unsupported features mask through
403 * EXIT_INFO_2 of the GHCB protocol so that those features can be reported
404 * as part of the guest boot failure.
405 */
406 unsupported = snp_get_unsupported_features(status: sev_status);
407 if (unsupported) {
408 if (ghcb_version < 2 || (!boot_ghcb && !early_setup_ghcb()))
409 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
410
411 sev_es_ghcb_terminate(ghcb: boot_ghcb, SEV_TERM_SET_GEN,
412 GHCB_SNP_UNSUPPORTED, exit_info_2: unsupported);
413 }
414}
415
416/*
417 * sev_check_cpu_support - Check for SEV support in the CPU capabilities
418 *
419 * Returns < 0 if SEV is not supported, otherwise the position of the
420 * encryption bit in the page table descriptors.
421 */
422static int sev_check_cpu_support(void)
423{
424 unsigned int eax, ebx, ecx, edx;
425
426 /* Check for the SME/SEV support leaf */
427 eax = 0x80000000;
428 ecx = 0;
429 native_cpuid(eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
430 if (eax < 0x8000001f)
431 return -ENODEV;
432
433 /*
434 * Check for the SME/SEV feature:
435 * CPUID Fn8000_001F[EAX]
436 * - Bit 0 - Secure Memory Encryption support
437 * - Bit 1 - Secure Encrypted Virtualization support
438 * CPUID Fn8000_001F[EBX]
439 * - Bits 5:0 - Pagetable bit position used to indicate encryption
440 */
441 eax = 0x8000001f;
442 ecx = 0;
443 native_cpuid(eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
444 /* Check whether SEV is supported */
445 if (!(eax & BIT(1)))
446 return -ENODEV;
447
448 return ebx & 0x3f;
449}
450
451void sev_enable(struct boot_params *bp)
452{
453 struct msr m;
454 int bitpos;
455 bool snp;
456
457 /*
458 * bp->cc_blob_address should only be set by boot/compressed kernel.
459 * Initialize it to 0 to ensure that uninitialized values from
460 * buggy bootloaders aren't propagated.
461 */
462 if (bp)
463 bp->cc_blob_address = 0;
464
465 /*
466 * Do an initial SEV capability check before snp_init() which
467 * loads the CPUID page and the same checks afterwards are done
468 * without the hypervisor and are trustworthy.
469 *
470 * If the HV fakes SEV support, the guest will crash'n'burn
471 * which is good enough.
472 */
473
474 if (sev_check_cpu_support() < 0)
475 return;
476
477 /*
478 * Setup/preliminary detection of SNP. This will be sanity-checked
479 * against CPUID/MSR values later.
480 */
481 snp = snp_init(bp);
482
483 /* Now repeat the checks with the SNP CPUID table. */
484
485 bitpos = sev_check_cpu_support();
486 if (bitpos < 0) {
487 if (snp)
488 error(m: "SEV-SNP support indicated by CC blob, but not CPUID.");
489 return;
490 }
491
492 /* Set the SME mask if this is an SEV guest. */
493 boot_rdmsr(MSR_AMD64_SEV, m: &m);
494 sev_status = m.q;
495 if (!(sev_status & MSR_AMD64_SEV_ENABLED))
496 return;
497
498 /* Negotiate the GHCB protocol version. */
499 if (sev_status & MSR_AMD64_SEV_ES_ENABLED) {
500 if (!sev_es_negotiate_protocol())
501 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_PROT_UNSUPPORTED);
502 }
503
504 /*
505 * SNP is supported in v2 of the GHCB spec which mandates support for HV
506 * features.
507 */
508 if (sev_status & MSR_AMD64_SEV_SNP_ENABLED) {
509 if (!(get_hv_features() & GHCB_HV_FT_SNP))
510 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
511
512 enforce_vmpl0();
513 }
514
515 if (snp && !(sev_status & MSR_AMD64_SEV_SNP_ENABLED))
516 error(m: "SEV-SNP supported indicated by CC blob, but not SEV status MSR.");
517
518 sme_me_mask = BIT_ULL(bitpos);
519}
520
521/*
522 * sev_get_status - Retrieve the SEV status mask
523 *
524 * Returns 0 if the CPU is not SEV capable, otherwise the value of the
525 * AMD64_SEV MSR.
526 */
527u64 sev_get_status(void)
528{
529 struct msr m;
530
531 if (sev_check_cpu_support() < 0)
532 return 0;
533
534 boot_rdmsr(MSR_AMD64_SEV, m: &m);
535 return m.q;
536}
537
538/* Search for Confidential Computing blob in the EFI config table. */
539static struct cc_blob_sev_info *find_cc_blob_efi(struct boot_params *bp)
540{
541 unsigned long cfg_table_pa;
542 unsigned int cfg_table_len;
543 int ret;
544
545 ret = efi_get_conf_table(bp, cfg_tbl_pa: &cfg_table_pa, cfg_tbl_len: &cfg_table_len);
546 if (ret)
547 return NULL;
548
549 return (struct cc_blob_sev_info *)efi_find_vendor_table(bp, cfg_tbl_pa: cfg_table_pa,
550 cfg_tbl_len: cfg_table_len,
551 EFI_CC_BLOB_GUID);
552}
553
554/*
555 * Initial set up of SNP relies on information provided by the
556 * Confidential Computing blob, which can be passed to the boot kernel
557 * by firmware/bootloader in the following ways:
558 *
559 * - via an entry in the EFI config table
560 * - via a setup_data structure, as defined by the Linux Boot Protocol
561 *
562 * Scan for the blob in that order.
563 */
564static struct cc_blob_sev_info *find_cc_blob(struct boot_params *bp)
565{
566 struct cc_blob_sev_info *cc_info;
567
568 cc_info = find_cc_blob_efi(bp);
569 if (cc_info)
570 goto found_cc_info;
571
572 cc_info = find_cc_blob_setup_data(bp);
573 if (!cc_info)
574 return NULL;
575
576found_cc_info:
577 if (cc_info->magic != CC_BLOB_SEV_HDR_MAGIC)
578 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
579
580 return cc_info;
581}
582
583/*
584 * Indicate SNP based on presence of SNP-specific CC blob. Subsequent checks
585 * will verify the SNP CPUID/MSR bits.
586 */
587bool snp_init(struct boot_params *bp)
588{
589 struct cc_blob_sev_info *cc_info;
590
591 if (!bp)
592 return false;
593
594 cc_info = find_cc_blob(bp);
595 if (!cc_info)
596 return false;
597
598 /*
599 * If a SNP-specific Confidential Computing blob is present, then
600 * firmware/bootloader have indicated SNP support. Verifying this
601 * involves CPUID checks which will be more reliable if the SNP
602 * CPUID table is used. See comments over snp_setup_cpuid_table() for
603 * more details.
604 */
605 setup_cpuid_table(cc_info);
606
607 /*
608 * Pass run-time kernel a pointer to CC info via boot_params so EFI
609 * config table doesn't need to be searched again during early startup
610 * phase.
611 */
612 bp->cc_blob_address = (u32)(unsigned long)cc_info;
613
614 return true;
615}
616
617void sev_prep_identity_maps(unsigned long top_level_pgt)
618{
619 /*
620 * The Confidential Computing blob is used very early in uncompressed
621 * kernel to find the in-memory CPUID table to handle CPUID
622 * instructions. Make sure an identity-mapping exists so it can be
623 * accessed after switchover.
624 */
625 if (sev_snp_enabled()) {
626 unsigned long cc_info_pa = boot_params_ptr->cc_blob_address;
627 struct cc_blob_sev_info *cc_info;
628
629 kernel_add_identity_map(start: cc_info_pa, end: cc_info_pa + sizeof(*cc_info));
630
631 cc_info = (struct cc_blob_sev_info *)cc_info_pa;
632 kernel_add_identity_map(start: cc_info->cpuid_phys, end: cc_info->cpuid_phys + cc_info->cpuid_len);
633 }
634
635 sev_verify_cbit(cr3: top_level_pgt);
636}
637

Provided by KDAB

Privacy Policy
Improve your Profiling and Debugging skills
Find out more

source code of linux/arch/x86/boot/compressed/sev.c