1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* Kernel-based Virtual Machine driver for Linux
4	*
5	* AMD SVM-SEV support
6	*
7	* Copyright 2010 Red Hat, Inc. and/or its affiliates.
8	*/
9	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
10
11	#include <linux/kvm_types.h>
12	#include <linux/kvm_host.h>
13	#include <linux/kernel.h>
14	#include <linux/highmem.h>
15	#include <linux/psp.h>
16	#include <linux/psp-sev.h>
17	#include <linux/pagemap.h>
18	#include <linux/swap.h>
19	#include <linux/misc_cgroup.h>
20	#include <linux/processor.h>
21	#include <linux/trace_events.h>
22	#include <uapi/linux/sev-guest.h>
23
24	#include <asm/pkru.h>
25	#include <asm/trapnr.h>
26	#include <asm/fpu/xcr.h>
27	#include <asm/fpu/xstate.h>
28	#include <asm/debugreg.h>
29	#include <asm/msr.h>
30	#include <asm/sev.h>
31
32	#include "mmu.h"
33	#include "x86.h"
34	#include "svm.h"
35	#include "svm_ops.h"
36	#include "cpuid.h"
37	#include "trace.h"
38
39	#define GHCB_VERSION_MAX 2ULL
40	#define GHCB_VERSION_MIN 1ULL
41
42	#define GHCB_HV_FT_SUPPORTED (GHCB_HV_FT_SNP | GHCB_HV_FT_SNP_AP_CREATION)
43
44	/* enable/disable SEV support */
45	static bool sev_enabled = true;
46	module_param_named(sev, sev_enabled, bool, 0444);
47
48	/* enable/disable SEV-ES support */
49	static bool sev_es_enabled = true;
50	module_param_named(sev_es, sev_es_enabled, bool, 0444);
51
52	/* enable/disable SEV-SNP support */
53	static bool sev_snp_enabled = true;
54	module_param_named(sev_snp, sev_snp_enabled, bool, 0444);
55
56	/* enable/disable SEV-ES DebugSwap support */
57	static bool sev_es_debug_swap_enabled = true;
58	module_param_named(debug_swap, sev_es_debug_swap_enabled, bool, 0444);
59	static u64 sev_supported_vmsa_features;
60
61	static unsigned int nr_ciphertext_hiding_asids;
62	module_param_named(ciphertext_hiding_asids, nr_ciphertext_hiding_asids, uint, 0444);
63
64	#define AP_RESET_HOLD_NONE 0
65	#define AP_RESET_HOLD_NAE_EVENT 1
66	#define AP_RESET_HOLD_MSR_PROTO 2
67
68	/*
69	* SEV-SNP policy bits that can be supported by KVM. These include policy bits
70	* that have implementation support within KVM or policy bits that do not
71	* require implementation support within KVM to enforce the policy.
72	*/
73	#define KVM_SNP_POLICY_MASK_VALID (SNP_POLICY_MASK_API_MINOR | \
74	SNP_POLICY_MASK_API_MAJOR | \
75	SNP_POLICY_MASK_SMT | \
76	SNP_POLICY_MASK_RSVD_MBO | \
77	SNP_POLICY_MASK_DEBUG | \
78	SNP_POLICY_MASK_SINGLE_SOCKET | \
79	SNP_POLICY_MASK_CXL_ALLOW | \
80	SNP_POLICY_MASK_MEM_AES_256_XTS | \
81	SNP_POLICY_MASK_RAPL_DIS | \
82	SNP_POLICY_MASK_CIPHERTEXT_HIDING_DRAM | \
83	SNP_POLICY_MASK_PAGE_SWAP_DISABLE)
84
85	static u64 snp_supported_policy_bits __ro_after_init;
86
87	#define INITIAL_VMSA_GPA 0xFFFFFFFFF000
88
89	static u8 sev_enc_bit;
90	static DECLARE_RWSEM(sev_deactivate_lock);
91	static DEFINE_MUTEX(sev_bitmap_lock);
92	unsigned int max_sev_asid;
93	static unsigned int min_sev_asid;
94	static unsigned int max_sev_es_asid;
95	static unsigned int min_sev_es_asid;
96	static unsigned int max_snp_asid;
97	static unsigned int min_snp_asid;
98	static unsigned long sev_me_mask;
99	static unsigned int nr_asids;
100	static unsigned long *sev_asid_bitmap;
101	static unsigned long *sev_reclaim_asid_bitmap;
102
103	static int snp_decommission_context(struct kvm *kvm);
104
105	struct enc_region {
106	struct list_head list;
107	unsigned long npages;
108	struct page **pages;
109	unsigned long uaddr;
110	unsigned long size;
111	};
112
113	/* Called with the sev_bitmap_lock held, or on shutdown */
114	static int sev_flush_asids(unsigned int min_asid, unsigned int max_asid)
115	{
116	int ret, error = 0;
117	unsigned int asid;
118
119	/* Check if there are any ASIDs to reclaim before performing a flush */
120	asid = find_next_bit(addr: sev_reclaim_asid_bitmap, size: nr_asids, offset: min_asid);
121	if (asid > max_asid)
122	return -EBUSY;
123
124	/*
125	* DEACTIVATE will clear the WBINVD indicator causing DF_FLUSH to fail,
126	* so it must be guarded.
127	*/
128	down_write(sem: &sev_deactivate_lock);
129
130	/* SNP firmware requires use of WBINVD for ASID recycling. */
131	wbinvd_on_all_cpus();
132
133	if (sev_snp_enabled)
134	ret = sev_do_cmd(cmd: SEV_CMD_SNP_DF_FLUSH, NULL, psp_ret: &error);
135	else
136	ret = sev_guest_df_flush(error: &error);
137
138	up_write(sem: &sev_deactivate_lock);
139
140	if (ret)
141	pr_err("SEV%s: DF_FLUSH failed, ret=%d, error=%#x\n",
142	sev_snp_enabled ? "-SNP" : "", ret, error);
143
144	return ret;
145	}
146
147	static inline bool is_mirroring_enc_context(struct kvm *kvm)
148	{
149	return !!to_kvm_sev_info(kvm)->enc_context_owner;
150	}
151
152	static bool sev_vcpu_has_debug_swap(struct vcpu_svm *svm)
153	{
154	struct kvm_vcpu *vcpu = &svm->vcpu;
155	struct kvm_sev_info *sev = to_kvm_sev_info(kvm: vcpu->kvm);
156
157	return sev->vmsa_features & SVM_SEV_FEAT_DEBUG_SWAP;
158	}
159
160	static bool snp_is_secure_tsc_enabled(struct kvm *kvm)
161	{
162	struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
163
164	return (sev->vmsa_features & SVM_SEV_FEAT_SECURE_TSC) &&
165	!WARN_ON_ONCE(!sev_snp_guest(kvm));
166	}
167
168	/* Must be called with the sev_bitmap_lock held */
169	static bool __sev_recycle_asids(unsigned int min_asid, unsigned int max_asid)
170	{
171	if (sev_flush_asids(min_asid, max_asid))
172	return false;
173
174	/* The flush process will flush all reclaimable SEV and SEV-ES ASIDs */
175	bitmap_xor(dst: sev_asid_bitmap, src1: sev_asid_bitmap, src2: sev_reclaim_asid_bitmap,
176	nbits: nr_asids);
177	bitmap_zero(dst: sev_reclaim_asid_bitmap, nbits: nr_asids);
178
179	return true;
180	}
181
182	static int sev_misc_cg_try_charge(struct kvm_sev_info *sev)
183	{
184	enum misc_res_type type = sev->es_active ? MISC_CG_RES_SEV_ES : MISC_CG_RES_SEV;
185	return misc_cg_try_charge(type, cg: sev->misc_cg, amount: 1);
186	}
187
188	static void sev_misc_cg_uncharge(struct kvm_sev_info *sev)
189	{
190	enum misc_res_type type = sev->es_active ? MISC_CG_RES_SEV_ES : MISC_CG_RES_SEV;
191	misc_cg_uncharge(type, cg: sev->misc_cg, amount: 1);
192	}
193
194	static int sev_asid_new(struct kvm_sev_info *sev, unsigned long vm_type)
195	{
196	/*
197	* SEV-enabled guests must use asid from min_sev_asid to max_sev_asid.
198	* SEV-ES-enabled guest can use from 1 to min_sev_asid - 1.
199	*/
200	unsigned int min_asid, max_asid, asid;
201	bool retry = true;
202	int ret;
203
204	if (vm_type == KVM_X86_SNP_VM) {
205	min_asid = min_snp_asid;
206	max_asid = max_snp_asid;
207	} else if (sev->es_active) {
208	min_asid = min_sev_es_asid;
209	max_asid = max_sev_es_asid;
210	} else {
211	min_asid = min_sev_asid;
212	max_asid = max_sev_asid;
213	}
214
215	/*
216	* The min ASID can end up larger than the max if basic SEV support is
217	* effectively disabled by disallowing use of ASIDs for SEV guests.
218	* Similarly for SEV-ES guests the min ASID can end up larger than the
219	* max when ciphertext hiding is enabled, effectively disabling SEV-ES
220	* support.
221	*/
222	if (min_asid > max_asid)
223	return -ENOTTY;
224
225	WARN_ON(sev->misc_cg);
226	sev->misc_cg = get_current_misc_cg();
227	ret = sev_misc_cg_try_charge(sev);
228	if (ret) {
229	put_misc_cg(cg: sev->misc_cg);
230	sev->misc_cg = NULL;
231	return ret;
232	}
233
234	mutex_lock(&sev_bitmap_lock);
235
236	again:
237	asid = find_next_zero_bit(addr: sev_asid_bitmap, size: max_asid + 1, offset: min_asid);
238	if (asid > max_asid) {
239	if (retry && __sev_recycle_asids(min_asid, max_asid)) {
240	retry = false;
241	goto again;
242	}
243	mutex_unlock(lock: &sev_bitmap_lock);
244	ret = -EBUSY;
245	goto e_uncharge;
246	}
247
248	__set_bit(asid, sev_asid_bitmap);
249
250	mutex_unlock(lock: &sev_bitmap_lock);
251
252	sev->asid = asid;
253	return 0;
254	e_uncharge:
255	sev_misc_cg_uncharge(sev);
256	put_misc_cg(cg: sev->misc_cg);
257	sev->misc_cg = NULL;
258	return ret;
259	}
260
261	static unsigned int sev_get_asid(struct kvm *kvm)
262	{
263	return to_kvm_sev_info(kvm)->asid;
264	}
265
266	static void sev_asid_free(struct kvm_sev_info *sev)
267	{
268	struct svm_cpu_data *sd;
269	int cpu;
270
271	mutex_lock(&sev_bitmap_lock);
272
273	__set_bit(sev->asid, sev_reclaim_asid_bitmap);
274
275	for_each_possible_cpu(cpu) {
276	sd = per_cpu_ptr(&svm_data, cpu);
277	sd->sev_vmcbs[sev->asid] = NULL;
278	}
279
280	mutex_unlock(lock: &sev_bitmap_lock);
281
282	sev_misc_cg_uncharge(sev);
283	put_misc_cg(cg: sev->misc_cg);
284	sev->misc_cg = NULL;
285	}
286
287	static void sev_decommission(unsigned int handle)
288	{
289	struct sev_data_decommission decommission;
290
291	if (!handle)
292	return;
293
294	decommission.handle = handle;
295	sev_guest_decommission(data: &decommission, NULL);
296	}
297
298	/*
299	* Transition a page to hypervisor-owned/shared state in the RMP table. This
300	* should not fail under normal conditions, but leak the page should that
301	* happen since it will no longer be usable by the host due to RMP protections.
302	*/
303	static int kvm_rmp_make_shared(struct kvm *kvm, u64 pfn, enum pg_level level)
304	{
305	if (KVM_BUG_ON(rmp_make_shared(pfn, level), kvm)) {
306	snp_leak_pages(pfn, pages: page_level_size(level) >> PAGE_SHIFT);
307	return -EIO;
308	}
309
310	return 0;
311	}
312
313	/*
314	* Certain page-states, such as Pre-Guest and Firmware pages (as documented
315	* in Chapter 5 of the SEV-SNP Firmware ABI under "Page States") cannot be
316	* directly transitioned back to normal/hypervisor-owned state via RMPUPDATE
317	* unless they are reclaimed first.
318	*
319	* Until they are reclaimed and subsequently transitioned via RMPUPDATE, they
320	* might not be usable by the host due to being set as immutable or still
321	* being associated with a guest ASID.
322	*
323	* Bug the VM and leak the page if reclaim fails, or if the RMP entry can't be
324	* converted back to shared, as the page is no longer usable due to RMP
325	* protections, and it's infeasible for the guest to continue on.
326	*/
327	static int snp_page_reclaim(struct kvm *kvm, u64 pfn)
328	{
329	struct sev_data_snp_page_reclaim data = {0};
330	int fw_err, rc;
331
332	data.paddr = __sme_set(pfn << PAGE_SHIFT);
333	rc = sev_do_cmd(cmd: SEV_CMD_SNP_PAGE_RECLAIM, data: &data, psp_ret: &fw_err);
334	if (KVM_BUG(rc, kvm, "Failed to reclaim PFN %llx, rc %d fw_err %d", pfn, rc, fw_err)) {
335	snp_leak_pages(pfn, pages: 1);
336	return -EIO;
337	}
338
339	if (kvm_rmp_make_shared(kvm, pfn, level: PG_LEVEL_4K))
340	return -EIO;
341
342	return rc;
343	}
344
345	static void sev_unbind_asid(struct kvm *kvm, unsigned int handle)
346	{
347	struct sev_data_deactivate deactivate;
348
349	if (!handle)
350	return;
351
352	deactivate.handle = handle;
353
354	/* Guard DEACTIVATE against WBINVD/DF_FLUSH used in ASID recycling */
355	down_read(sem: &sev_deactivate_lock);
356	sev_guest_deactivate(data: &deactivate, NULL);
357	up_read(sem: &sev_deactivate_lock);
358
359	sev_decommission(handle);
360	}
361
362	/*
363	* This sets up bounce buffers/firmware pages to handle SNP Guest Request
364	* messages (e.g. attestation requests). See "SNP Guest Request" in the GHCB
365	* 2.0 specification for more details.
366	*
367	* Technically, when an SNP Guest Request is issued, the guest will provide its
368	* own request/response pages, which could in theory be passed along directly
369	* to firmware rather than using bounce pages. However, these pages would need
370	* special care:
371	*
372	* - Both pages are from shared guest memory, so they need to be protected
373	* from migration/etc. occurring while firmware reads/writes to them. At a
374	* minimum, this requires elevating the ref counts and potentially needing
375	* an explicit pinning of the memory. This places additional restrictions
376	* on what type of memory backends userspace can use for shared guest
377	* memory since there is some reliance on using refcounted pages.
378	*
379	* - The response page needs to be switched to Firmware-owned[1] state
380	* before the firmware can write to it, which can lead to potential
381	* host RMP #PFs if the guest is misbehaved and hands the host a
382	* guest page that KVM might write to for other reasons (e.g. virtio
383	* buffers/etc.).
384	*
385	* Both of these issues can be avoided completely by using separately-allocated
386	* bounce pages for both the request/response pages and passing those to
387	* firmware instead. So that's what is being set up here.
388	*
389	* Guest requests rely on message sequence numbers to ensure requests are
390	* issued to firmware in the order the guest issues them, so concurrent guest
391	* requests generally shouldn't happen. But a misbehaved guest could issue
392	* concurrent guest requests in theory, so a mutex is used to serialize
393	* access to the bounce buffers.
394	*
395	* [1] See the "Page States" section of the SEV-SNP Firmware ABI for more
396	* details on Firmware-owned pages, along with "RMP and VMPL Access Checks"
397	* in the APM for details on the related RMP restrictions.
398	*/
399	static int snp_guest_req_init(struct kvm *kvm)
400	{
401	struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
402	struct page *req_page;
403
404	req_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
405	if (!req_page)
406	return -ENOMEM;
407
408	sev->guest_resp_buf = snp_alloc_firmware_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
409	if (!sev->guest_resp_buf) {
410	__free_page(req_page);
411	return -EIO;
412	}
413
414	sev->guest_req_buf = page_address(req_page);
415	mutex_init(&sev->guest_req_mutex);
416
417	return 0;
418	}
419
420	static void snp_guest_req_cleanup(struct kvm *kvm)
421	{
422	struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
423
424	if (sev->guest_resp_buf)
425	snp_free_firmware_page(addr: sev->guest_resp_buf);
426
427	if (sev->guest_req_buf)
428	__free_page(virt_to_page(sev->guest_req_buf));
429
430	sev->guest_req_buf = NULL;
431	sev->guest_resp_buf = NULL;
432	}
433
434	static int __sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp,
435	struct kvm_sev_init *data,
436	unsigned long vm_type)
437	{
438	struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
439	struct sev_platform_init_args init_args = {0};
440	bool es_active = vm_type != KVM_X86_SEV_VM;
441	bool snp_active = vm_type == KVM_X86_SNP_VM;
442	u64 valid_vmsa_features = es_active ? sev_supported_vmsa_features : 0;
443	int ret;
444
445	if (kvm->created_vcpus)
446	return -EINVAL;
447
448	if (data->flags)
449	return -EINVAL;
450
451	if (!snp_active)
452	valid_vmsa_features &= ~SVM_SEV_FEAT_SECURE_TSC;
453
454	if (data->vmsa_features & ~valid_vmsa_features)
455	return -EINVAL;
456
457	if (data->ghcb_version > GHCB_VERSION_MAX || (!es_active && data->ghcb_version))
458	return -EINVAL;
459
460	/*
461	* KVM supports the full range of mandatory features defined by version
462	* 2 of the GHCB protocol, so default to that for SEV-ES guests created
463	* via KVM_SEV_INIT2 (KVM_SEV_INIT forces version 1).
464	*/
465	if (es_active && !data->ghcb_version)
466	data->ghcb_version = 2;
467
468	if (snp_active && data->ghcb_version < 2)
469	return -EINVAL;
470
471	if (unlikely(sev->active))
472	return -EINVAL;
473
474	sev->active = true;
475	sev->es_active = es_active;
476	sev->vmsa_features = data->vmsa_features;
477	sev->ghcb_version = data->ghcb_version;
478
479	if (snp_active)
480	sev->vmsa_features |= SVM_SEV_FEAT_SNP_ACTIVE;
481
482	ret = sev_asid_new(sev, vm_type);
483	if (ret)
484	goto e_no_asid;
485
486	init_args.probe = false;
487	ret = sev_platform_init(args: &init_args);
488	if (ret)
489	goto e_free_asid;
490
491	if (!zalloc_cpumask_var(mask: &sev->have_run_cpus, GFP_KERNEL_ACCOUNT)) {
492	ret = -ENOMEM;
493	goto e_free_asid;
494	}
495
496	/* This needs to happen after SEV/SNP firmware initialization. */
497	if (snp_active) {
498	ret = snp_guest_req_init(kvm);
499	if (ret)
500	goto e_free;
501	}
502
503	INIT_LIST_HEAD(list: &sev->regions_list);
504	INIT_LIST_HEAD(list: &sev->mirror_vms);
505	sev->need_init = false;
506
507	kvm_set_apicv_inhibit(kvm, reason: APICV_INHIBIT_REASON_SEV);
508
509	return 0;
510
511	e_free:
512	free_cpumask_var(mask: sev->have_run_cpus);
513	e_free_asid:
514	argp->error = init_args.error;
515	sev_asid_free(sev);
516	sev->asid = 0;
517	e_no_asid:
518	sev->vmsa_features = 0;
519	sev->es_active = false;
520	sev->active = false;
521	return ret;
522	}
523
524	static int sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp)
525	{
526	struct kvm_sev_init data = {
527	.vmsa_features = 0,
528	.ghcb_version = 0,
529	};
530	unsigned long vm_type;
531
532	if (kvm->arch.vm_type != KVM_X86_DEFAULT_VM)
533	return -EINVAL;
534
535	vm_type = (argp->id == KVM_SEV_INIT ? KVM_X86_SEV_VM : KVM_X86_SEV_ES_VM);
536
537	/*
538	* KVM_SEV_ES_INIT has been deprecated by KVM_SEV_INIT2, so it will
539	* continue to only ever support the minimal GHCB protocol version.
540	*/
541	if (vm_type == KVM_X86_SEV_ES_VM)
542	data.ghcb_version = GHCB_VERSION_MIN;
543
544	return __sev_guest_init(kvm, argp, data: &data, vm_type);
545	}
546
547	static int sev_guest_init2(struct kvm *kvm, struct kvm_sev_cmd *argp)
548	{
549	struct kvm_sev_init data;
550
551	if (!to_kvm_sev_info(kvm)->need_init)
552	return -EINVAL;
553
554	if (kvm->arch.vm_type != KVM_X86_SEV_VM &&
555	kvm->arch.vm_type != KVM_X86_SEV_ES_VM &&
556	kvm->arch.vm_type != KVM_X86_SNP_VM)
557	return -EINVAL;
558
559	if (copy_from_user(to: &data, u64_to_user_ptr(argp->data), n: sizeof(data)))
560	return -EFAULT;
561
562	return __sev_guest_init(kvm, argp, data: &data, vm_type: kvm->arch.vm_type);
563	}
564
565	static int sev_bind_asid(struct kvm *kvm, unsigned int handle, int *error)
566	{
567	unsigned int asid = sev_get_asid(kvm);
568	struct sev_data_activate activate;
569	int ret;
570
571	/* activate ASID on the given handle */
572	activate.handle = handle;
573	activate.asid = asid;
574	ret = sev_guest_activate(data: &activate, error);
575
576	return ret;
577	}
578
579	static int __sev_issue_cmd(int fd, int id, void *data, int *error)
580	{
581	CLASS(fd, f)(fd);
582
583	if (fd_empty(f))
584	return -EBADF;
585
586	return sev_issue_cmd_external_user(fd_file(f), id, data, error);
587	}
588
589	static int sev_issue_cmd(struct kvm *kvm, int id, void *data, int *error)
590	{
591	struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
592
593	return __sev_issue_cmd(fd: sev->fd, id, data, error);
594	}
595
596	static int sev_launch_start(struct kvm *kvm, struct kvm_sev_cmd *argp)
597	{
598	struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
599	struct sev_data_launch_start start;
600	struct kvm_sev_launch_start params;
601	void *dh_blob, *session_blob;
602	int *error = &argp->error;
603	int ret;
604
605	if (!sev_guest(kvm))
606	return -ENOTTY;
607
608	if (copy_from_user(to: &params, u64_to_user_ptr(argp->data), n: sizeof(params)))
609	return -EFAULT;
610
611	memset(&start, 0, sizeof(start));
612
613	dh_blob = NULL;
614	if (params.dh_uaddr) {
615	dh_blob = psp_copy_user_blob(uaddr: params.dh_uaddr, len: params.dh_len);
616	if (IS_ERR(ptr: dh_blob))
617	return PTR_ERR(ptr: dh_blob);
618
619	start.dh_cert_address = __sme_set(__pa(dh_blob));
620	start.dh_cert_len = params.dh_len;
621	}
622
623	session_blob = NULL;
624	if (params.session_uaddr) {
625	session_blob = psp_copy_user_blob(uaddr: params.session_uaddr, len: params.session_len);
626	if (IS_ERR(ptr: session_blob)) {
627	ret = PTR_ERR(ptr: session_blob);
628	goto e_free_dh;
629	}
630
631	start.session_address = __sme_set(__pa(session_blob));
632	start.session_len = params.session_len;
633	}
634
635	start.handle = params.handle;
636	start.policy = params.policy;
637
638	/* create memory encryption context */
639	ret = __sev_issue_cmd(fd: argp->sev_fd, id: SEV_CMD_LAUNCH_START, data: &start, error);
640	if (ret)
641	goto e_free_session;
642
643	/* Bind ASID to this guest */
644	ret = sev_bind_asid(kvm, handle: start.handle, error);
645	if (ret) {
646	sev_decommission(handle: start.handle);
647	goto e_free_session;
648	}
649
650	/* return handle to userspace */
651	params.handle = start.handle;
652	if (copy_to_user(u64_to_user_ptr(argp->data), from: &params, n: sizeof(params))) {
653	sev_unbind_asid(kvm, handle: start.handle);
654	ret = -EFAULT;
655	goto e_free_session;
656	}
657
658	sev->policy = params.policy;
659	sev->handle = start.handle;
660	sev->fd = argp->sev_fd;
661
662	e_free_session:
663	kfree(objp: session_blob);
664	e_free_dh:
665	kfree(objp: dh_blob);
666	return ret;
667	}
668
669	static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr,
670	unsigned long ulen, unsigned long *n,
671	unsigned int flags)
672	{
673	struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
674	unsigned long npages, size;
675	int npinned;
676	unsigned long locked, lock_limit;
677	struct page **pages;
678	unsigned long first, last;
679	int ret;
680
681	lockdep_assert_held(&kvm->lock);
682
683	if (ulen == 0 || uaddr + ulen < uaddr)
684	return ERR_PTR(error: -EINVAL);
685
686	/* Calculate number of pages. */
687	first = (uaddr & PAGE_MASK) >> PAGE_SHIFT;
688	last = ((uaddr + ulen - 1) & PAGE_MASK) >> PAGE_SHIFT;
689	npages = (last - first + 1);
690
691	locked = sev->pages_locked + npages;
692	lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
693	if (locked > lock_limit && !capable(CAP_IPC_LOCK)) {
694	pr_err("SEV: %lu locked pages exceed the lock limit of %lu.\n", locked, lock_limit);
695	return ERR_PTR(error: -ENOMEM);
696	}
697
698	if (WARN_ON_ONCE(npages > INT_MAX))
699	return ERR_PTR(error: -EINVAL);
700
701	/* Avoid using vmalloc for smaller buffers. */
702	size = npages * sizeof(struct page *);
703	if (size > PAGE_SIZE)
704	pages = __vmalloc(size, GFP_KERNEL_ACCOUNT);
705	else
706	pages = kmalloc(size, GFP_KERNEL_ACCOUNT);
707
708	if (!pages)
709	return ERR_PTR(error: -ENOMEM);
710
711	/* Pin the user virtual address. */
712	npinned = pin_user_pages_fast(start: uaddr, nr_pages: npages, gup_flags: flags, pages);
713	if (npinned != npages) {
714	pr_err("SEV: Failure locking %lu pages.\n", npages);
715	ret = -ENOMEM;
716	goto err;
717	}
718
719	*n = npages;
720	sev->pages_locked = locked;
721
722	return pages;
723
724	err:
725	if (npinned > 0)
726	unpin_user_pages(pages, npages: npinned);
727
728	kvfree(addr: pages);
729	return ERR_PTR(error: ret);
730	}
731
732	static void sev_unpin_memory(struct kvm *kvm, struct page **pages,
733	unsigned long npages)
734	{
735	unpin_user_pages(pages, npages);
736	kvfree(addr: pages);
737	to_kvm_sev_info(kvm)->pages_locked -= npages;
738	}
739
740	static void sev_clflush_pages(struct page *pages[], unsigned long npages)
741	{
742	uint8_t *page_virtual;
743	unsigned long i;
744
745	if (this_cpu_has(X86_FEATURE_SME_COHERENT) || npages == 0 ||
746	pages == NULL)
747	return;
748
749	for (i = 0; i < npages; i++) {
750	page_virtual = kmap_local_page(page: pages[i]);
751	clflush_cache_range(addr: page_virtual, PAGE_SIZE);
752	kunmap_local(page_virtual);
753	cond_resched();
754	}
755	}
756
757	static void sev_writeback_caches(struct kvm *kvm)
758	{
759	/*
760	* Ensure that all dirty guest tagged cache entries are written back
761	* before releasing the pages back to the system for use. CLFLUSH will
762	* not do this without SME_COHERENT, and flushing many cache lines
763	* individually is slower than blasting WBINVD for large VMs, so issue
764	* WBNOINVD (or WBINVD if the "no invalidate" variant is unsupported)
765	* on CPUs that have done VMRUN, i.e. may have dirtied data using the
766	* VM's ASID.
767	*
768	* For simplicity, never remove CPUs from the bitmap. Ideally, KVM
769	* would clear the mask when flushing caches, but doing so requires
770	* serializing multiple calls and having responding CPUs (to the IPI)
771	* mark themselves as still running if they are running (or about to
772	* run) a vCPU for the VM.
773	*
774	* Note, the caller is responsible for ensuring correctness if the mask
775	* can be modified, e.g. if a CPU could be doing VMRUN.
776	*/
777	wbnoinvd_on_cpus_mask(cpus: to_kvm_sev_info(kvm)->have_run_cpus);
778	}
779
780	static unsigned long get_num_contig_pages(unsigned long idx,
781	struct page **inpages, unsigned long npages)
782	{
783	unsigned long paddr, next_paddr;
784	unsigned long i = idx + 1, pages = 1;
785
786	/* find the number of contiguous pages starting from idx */
787	paddr = __sme_page_pa(page: inpages[idx]);
788	while (i < npages) {
789	next_paddr = __sme_page_pa(page: inpages[i++]);
790	if ((paddr + PAGE_SIZE) == next_paddr) {
791	pages++;
792	paddr = next_paddr;
793	continue;
794	}
795	break;
796	}
797
798	return pages;
799	}
800
801	static int sev_launch_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
802	{
803	unsigned long vaddr, vaddr_end, next_vaddr, npages, pages, size, i;
804	struct kvm_sev_launch_update_data params;
805	struct sev_data_launch_update_data data;
806	struct page **inpages;
807	int ret;
808
809	if (!sev_guest(kvm))
810	return -ENOTTY;
811
812	if (copy_from_user(to: &params, u64_to_user_ptr(argp->data), n: sizeof(params)))
813	return -EFAULT;
814
815	vaddr = params.uaddr;
816	size = params.len;
817	vaddr_end = vaddr + size;
818
819	/* Lock the user memory. */
820	inpages = sev_pin_memory(kvm, uaddr: vaddr, ulen: size, n: &npages, flags: FOLL_WRITE);
821	if (IS_ERR(ptr: inpages))
822	return PTR_ERR(ptr: inpages);
823
824	/*
825	* Flush (on non-coherent CPUs) before LAUNCH_UPDATE encrypts pages in
826	* place; the cache may contain the data that was written unencrypted.
827	*/
828	sev_clflush_pages(pages: inpages, npages);
829
830	data.reserved = 0;
831	data.handle = to_kvm_sev_info(kvm)->handle;
832
833	for (i = 0; vaddr < vaddr_end; vaddr = next_vaddr, i += pages) {
834	int offset, len;
835
836	/*
837	* If the user buffer is not page-aligned, calculate the offset
838	* within the page.
839	*/
840	offset = vaddr & (PAGE_SIZE - 1);
841
842	/* Calculate the number of pages that can be encrypted in one go. */
843	pages = get_num_contig_pages(idx: i, inpages, npages);
844
845	len = min_t(size_t, ((pages * PAGE_SIZE) - offset), size);
846
847	data.len = len;
848	data.address = __sme_page_pa(page: inpages[i]) + offset;
849	ret = sev_issue_cmd(kvm, id: SEV_CMD_LAUNCH_UPDATE_DATA, data: &data, error: &argp->error);
850	if (ret)
851	goto e_unpin;
852
853	size -= len;
854	next_vaddr = vaddr + len;
855	}
856
857	e_unpin:
858	/* content of memory is updated, mark pages dirty */
859	for (i = 0; i < npages; i++) {
860	set_page_dirty_lock(inpages[i]);
861	mark_page_accessed(inpages[i]);
862	}
863	/* unlock the user pages */
864	sev_unpin_memory(kvm, pages: inpages, npages);
865	return ret;
866	}
867
868	static int sev_es_sync_vmsa(struct vcpu_svm *svm)
869	{
870	struct kvm_vcpu *vcpu = &svm->vcpu;
871	struct kvm_sev_info *sev = to_kvm_sev_info(kvm: vcpu->kvm);
872	struct sev_es_save_area *save = svm->sev_es.vmsa;
873	struct xregs_state *xsave;
874	const u8 *s;
875	u8 *d;
876	int i;
877
878	/* Check some debug related fields before encrypting the VMSA */
879	if (svm->vcpu.guest_debug || (svm->vmcb->save.dr7 & ~DR7_FIXED_1))
880	return -EINVAL;
881
882	/*
883	* SEV-ES will use a VMSA that is pointed to by the VMCB, not
884	* the traditional VMSA that is part of the VMCB. Copy the
885	* traditional VMSA as it has been built so far (in prep
886	* for LAUNCH_UPDATE_VMSA) to be the initial SEV-ES state.
887	*/
888	memcpy(save, &svm->vmcb->save, sizeof(svm->vmcb->save));
889
890	/* Sync registgers */
891	save->rax = svm->vcpu.arch.regs[VCPU_REGS_RAX];
892	save->rbx = svm->vcpu.arch.regs[VCPU_REGS_RBX];
893	save->rcx = svm->vcpu.arch.regs[VCPU_REGS_RCX];
894	save->rdx = svm->vcpu.arch.regs[VCPU_REGS_RDX];
895	save->rsp = svm->vcpu.arch.regs[VCPU_REGS_RSP];
896	save->rbp = svm->vcpu.arch.regs[VCPU_REGS_RBP];
897	save->rsi = svm->vcpu.arch.regs[VCPU_REGS_RSI];
898	save->rdi = svm->vcpu.arch.regs[VCPU_REGS_RDI];
899	#ifdef CONFIG_X86_64
900	save->r8 = svm->vcpu.arch.regs[VCPU_REGS_R8];
901	save->r9 = svm->vcpu.arch.regs[VCPU_REGS_R9];
902	save->r10 = svm->vcpu.arch.regs[VCPU_REGS_R10];
903	save->r11 = svm->vcpu.arch.regs[VCPU_REGS_R11];
904	save->r12 = svm->vcpu.arch.regs[VCPU_REGS_R12];
905	save->r13 = svm->vcpu.arch.regs[VCPU_REGS_R13];
906	save->r14 = svm->vcpu.arch.regs[VCPU_REGS_R14];
907	save->r15 = svm->vcpu.arch.regs[VCPU_REGS_R15];
908	#endif
909	save->rip = svm->vcpu.arch.regs[VCPU_REGS_RIP];
910
911	/* Sync some non-GPR registers before encrypting */
912	save->xcr0 = svm->vcpu.arch.xcr0;
913	save->pkru = svm->vcpu.arch.pkru;
914	save->xss = svm->vcpu.arch.ia32_xss;
915	save->dr6 = svm->vcpu.arch.dr6;
916
917	save->sev_features = sev->vmsa_features;
918
919	/*
920	* Skip FPU and AVX setup with KVM_SEV_ES_INIT to avoid
921	* breaking older measurements.
922	*/
923	if (vcpu->kvm->arch.vm_type != KVM_X86_DEFAULT_VM) {
924	xsave = &vcpu->arch.guest_fpu.fpstate->regs.xsave;
925	save->x87_dp = xsave->i387.rdp;
926	save->mxcsr = xsave->i387.mxcsr;
927	save->x87_ftw = xsave->i387.twd;
928	save->x87_fsw = xsave->i387.swd;
929	save->x87_fcw = xsave->i387.cwd;
930	save->x87_fop = xsave->i387.fop;
931	save->x87_ds = 0;
932	save->x87_cs = 0;
933	save->x87_rip = xsave->i387.rip;
934
935	for (i = 0; i < 8; i++) {
936	/*
937	* The format of the x87 save area is undocumented and
938	* definitely not what you would expect. It consists of
939	* an 8*8 bytes area with bytes 0-7, and an 8*2 bytes
940	* area with bytes 8-9 of each register.
941	*/
942	d = save->fpreg_x87 + i * 8;
943	s = ((u8 *)xsave->i387.st_space) + i * 16;
944	memcpy(d, s, 8);
945	save->fpreg_x87[64 + i * 2] = s[8];
946	save->fpreg_x87[64 + i * 2 + 1] = s[9];
947	}
948	memcpy(save->fpreg_xmm, xsave->i387.xmm_space, 256);
949
950	s = get_xsave_addr(xsave, xfeature_nr: XFEATURE_YMM);
951	if (s)
952	memcpy(save->fpreg_ymm, s, 256);
953	else
954	memset(save->fpreg_ymm, 0, 256);
955	}
956
957	pr_debug("Virtual Machine Save Area (VMSA):\n");
958	print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1, save, sizeof(*save), false);
959
960	return 0;
961	}
962
963	static int __sev_launch_update_vmsa(struct kvm *kvm, struct kvm_vcpu *vcpu,
964	int *error)
965	{
966	struct sev_data_launch_update_vmsa vmsa;
967	struct vcpu_svm *svm = to_svm(vcpu);
968	int ret;
969
970	if (vcpu->guest_debug) {
971	pr_warn_once("KVM_SET_GUEST_DEBUG for SEV-ES guest is not supported");
972	return -EINVAL;
973	}
974
975	/* Perform some pre-encryption checks against the VMSA */
976	ret = sev_es_sync_vmsa(svm);
977	if (ret)
978	return ret;
979
980	/*
981	* The LAUNCH_UPDATE_VMSA command will perform in-place encryption of
982	* the VMSA memory content (i.e it will write the same memory region
983	* with the guest's key), so invalidate it first.
984	*/
985	clflush_cache_range(addr: svm->sev_es.vmsa, PAGE_SIZE);
986
987	vmsa.reserved = 0;
988	vmsa.handle = to_kvm_sev_info(kvm)->handle;
989	vmsa.address = __sme_pa(svm->sev_es.vmsa);
990	vmsa.len = PAGE_SIZE;
991	ret = sev_issue_cmd(kvm, id: SEV_CMD_LAUNCH_UPDATE_VMSA, data: &vmsa, error);
992	if (ret)
993	return ret;
994
995	/*
996	* SEV-ES guests maintain an encrypted version of their FPU
997	* state which is restored and saved on VMRUN and VMEXIT.
998	* Mark vcpu->arch.guest_fpu->fpstate as scratch so it won't
999	* do xsave/xrstor on it.
1000	*/
1001	fpstate_set_confidential(gfpu: &vcpu->arch.guest_fpu);
1002	vcpu->arch.guest_state_protected = true;
1003
1004	/*
1005	* SEV-ES guest mandates LBR Virtualization to be _always_ ON. Enable it
1006	* only after setting guest_state_protected because KVM_SET_MSRS allows
1007	* dynamic toggling of LBRV (for performance reason) on write access to
1008	* MSR_IA32_DEBUGCTLMSR when guest_state_protected is not set.
1009	*/
1010	svm_enable_lbrv(vcpu);
1011	return 0;
1012	}
1013
1014	static int sev_launch_update_vmsa(struct kvm *kvm, struct kvm_sev_cmd *argp)
1015	{
1016	struct kvm_vcpu *vcpu;
1017	unsigned long i;
1018	int ret;
1019
1020	if (!sev_es_guest(kvm))
1021	return -ENOTTY;
1022
1023	kvm_for_each_vcpu(i, vcpu, kvm) {
1024	ret = mutex_lock_killable(&vcpu->mutex);
1025	if (ret)
1026	return ret;
1027
1028	ret = __sev_launch_update_vmsa(kvm, vcpu, error: &argp->error);
1029
1030	mutex_unlock(lock: &vcpu->mutex);
1031	if (ret)
1032	return ret;
1033	}
1034
1035	return 0;
1036	}
1037
1038	static int sev_launch_measure(struct kvm *kvm, struct kvm_sev_cmd *argp)
1039	{
1040	void __user *measure = u64_to_user_ptr(argp->data);
1041	struct sev_data_launch_measure data;
1042	struct kvm_sev_launch_measure params;
1043	void __user *p = NULL;
1044	void *blob = NULL;
1045	int ret;
1046
1047	if (!sev_guest(kvm))
1048	return -ENOTTY;
1049
1050	if (copy_from_user(to: &params, from: measure, n: sizeof(params)))
1051	return -EFAULT;
1052
1053	memset(&data, 0, sizeof(data));
1054
1055	/* User wants to query the blob length */
1056	if (!params.len)
1057	goto cmd;
1058
1059	p = u64_to_user_ptr(params.uaddr);
1060	if (p) {
1061	if (params.len > SEV_FW_BLOB_MAX_SIZE)
1062	return -EINVAL;
1063
1064	blob = kzalloc(params.len, GFP_KERNEL_ACCOUNT);
1065	if (!blob)
1066	return -ENOMEM;
1067
1068	data.address = __psp_pa(blob);
1069	data.len = params.len;
1070	}
1071
1072	cmd:
1073	data.handle = to_kvm_sev_info(kvm)->handle;
1074	ret = sev_issue_cmd(kvm, id: SEV_CMD_LAUNCH_MEASURE, data: &data, error: &argp->error);
1075
1076	/*
1077	* If we query the session length, FW responded with expected data.
1078	*/
1079	if (!params.len)
1080	goto done;
1081
1082	if (ret)
1083	goto e_free_blob;
1084
1085	if (blob) {
1086	if (copy_to_user(to: p, from: blob, n: params.len))
1087	ret = -EFAULT;
1088	}
1089
1090	done:
1091	params.len = data.len;
1092	if (copy_to_user(to: measure, from: &params, n: sizeof(params)))
1093	ret = -EFAULT;
1094	e_free_blob:
1095	kfree(objp: blob);
1096	return ret;
1097	}
1098
1099	static int sev_launch_finish(struct kvm *kvm, struct kvm_sev_cmd *argp)
1100	{
1101	struct sev_data_launch_finish data;
1102
1103	if (!sev_guest(kvm))
1104	return -ENOTTY;
1105
1106	data.handle = to_kvm_sev_info(kvm)->handle;
1107	return sev_issue_cmd(kvm, id: SEV_CMD_LAUNCH_FINISH, data: &data, error: &argp->error);
1108	}
1109
1110	static int sev_guest_status(struct kvm *kvm, struct kvm_sev_cmd *argp)
1111	{
1112	struct kvm_sev_guest_status params;
1113	struct sev_data_guest_status data;
1114	int ret;
1115
1116	if (!sev_guest(kvm))
1117	return -ENOTTY;
1118
1119	memset(&data, 0, sizeof(data));
1120
1121	data.handle = to_kvm_sev_info(kvm)->handle;
1122	ret = sev_issue_cmd(kvm, id: SEV_CMD_GUEST_STATUS, data: &data, error: &argp->error);
1123	if (ret)
1124	return ret;
1125
1126	params.policy = data.policy;
1127	params.state = data.state;
1128	params.handle = data.handle;
1129
1130	if (copy_to_user(u64_to_user_ptr(argp->data), from: &params, n: sizeof(params)))
1131	ret = -EFAULT;
1132
1133	return ret;
1134	}
1135
1136	static int __sev_issue_dbg_cmd(struct kvm *kvm, unsigned long src,
1137	unsigned long dst, int size,
1138	int *error, bool enc)
1139	{
1140	struct sev_data_dbg data;
1141
1142	data.reserved = 0;
1143	data.handle = to_kvm_sev_info(kvm)->handle;
1144	data.dst_addr = dst;
1145	data.src_addr = src;
1146	data.len = size;
1147
1148	return sev_issue_cmd(kvm,
1149	id: enc ? SEV_CMD_DBG_ENCRYPT : SEV_CMD_DBG_DECRYPT,
1150	data: &data, error);
1151	}
1152
1153	static int __sev_dbg_decrypt(struct kvm *kvm, unsigned long src_paddr,
1154	unsigned long dst_paddr, int sz, int *err)
1155	{
1156	int offset;
1157
1158	/*
1159	* Its safe to read more than we are asked, caller should ensure that
1160	* destination has enough space.
1161	*/
1162	offset = src_paddr & 15;
1163	src_paddr = round_down(src_paddr, 16);
1164	sz = round_up(sz + offset, 16);
1165
1166	return __sev_issue_dbg_cmd(kvm, src: src_paddr, dst: dst_paddr, size: sz, error: err, enc: false);
1167	}
1168
1169	static int __sev_dbg_decrypt_user(struct kvm *kvm, unsigned long paddr,
1170	void __user *dst_uaddr,
1171	unsigned long dst_paddr,
1172	int size, int *err)
1173	{
1174	struct page *tpage = NULL;
1175	int ret, offset;
1176
1177	/* if inputs are not 16-byte then use intermediate buffer */
1178	if (!IS_ALIGNED(dst_paddr, 16) ||
1179	!IS_ALIGNED(paddr, 16) ||
1180	!IS_ALIGNED(size, 16)) {
1181	tpage = (void *)alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
1182	if (!tpage)
1183	return -ENOMEM;
1184
1185	dst_paddr = __sme_page_pa(page: tpage);
1186	}
1187
1188	ret = __sev_dbg_decrypt(kvm, src_paddr: paddr, dst_paddr, sz: size, err);
1189	if (ret)
1190	goto e_free;
1191
1192	if (tpage) {
1193	offset = paddr & 15;
1194	if (copy_to_user(to: dst_uaddr, page_address(tpage) + offset, n: size))
1195	ret = -EFAULT;
1196	}
1197
1198	e_free:
1199	if (tpage)
1200	__free_page(tpage);
1201
1202	return ret;
1203	}
1204
1205	static int __sev_dbg_encrypt_user(struct kvm *kvm, unsigned long paddr,
1206	void __user *vaddr,
1207	unsigned long dst_paddr,
1208	void __user *dst_vaddr,
1209	int size, int *error)
1210	{
1211	struct page *src_tpage = NULL;
1212	struct page *dst_tpage = NULL;
1213	int ret, len = size;
1214
1215	/* If source buffer is not aligned then use an intermediate buffer */
1216	if (!IS_ALIGNED((unsigned long)vaddr, 16)) {
1217	src_tpage = alloc_page(GFP_KERNEL_ACCOUNT);
1218	if (!src_tpage)
1219	return -ENOMEM;
1220
1221	if (copy_from_user(page_address(src_tpage), from: vaddr, n: size)) {
1222	__free_page(src_tpage);
1223	return -EFAULT;
1224	}
1225
1226	paddr = __sme_page_pa(page: src_tpage);
1227	}
1228
1229	/*
1230	* If destination buffer or length is not aligned then do read-modify-write:
1231	* - decrypt destination in an intermediate buffer
1232	* - copy the source buffer in an intermediate buffer
1233	* - use the intermediate buffer as source buffer
1234	*/
1235	if (!IS_ALIGNED((unsigned long)dst_vaddr, 16) || !IS_ALIGNED(size, 16)) {
1236	int dst_offset;
1237
1238	dst_tpage = alloc_page(GFP_KERNEL_ACCOUNT);
1239	if (!dst_tpage) {
1240	ret = -ENOMEM;
1241	goto e_free;
1242	}
1243
1244	ret = __sev_dbg_decrypt(kvm, src_paddr: dst_paddr,
1245	dst_paddr: __sme_page_pa(page: dst_tpage), sz: size, err: error);
1246	if (ret)
1247	goto e_free;
1248
1249	/*
1250	* If source is kernel buffer then use memcpy() otherwise
1251	* copy_from_user().
1252	*/
1253	dst_offset = dst_paddr & 15;
1254
1255	if (src_tpage)
1256	memcpy(page_address(dst_tpage) + dst_offset,
1257	page_address(src_tpage), size);
1258	else {
1259	if (copy_from_user(page_address(dst_tpage) + dst_offset,
1260	from: vaddr, n: size)) {
1261	ret = -EFAULT;
1262	goto e_free;
1263	}
1264	}
1265
1266	paddr = __sme_page_pa(page: dst_tpage);
1267	dst_paddr = round_down(dst_paddr, 16);
1268	len = round_up(size, 16);
1269	}
1270
1271	ret = __sev_issue_dbg_cmd(kvm, src: paddr, dst: dst_paddr, size: len, error, enc: true);
1272
1273	e_free:
1274	if (src_tpage)
1275	__free_page(src_tpage);
1276	if (dst_tpage)
1277	__free_page(dst_tpage);
1278	return ret;
1279	}
1280
1281	static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, bool dec)
1282	{
1283	unsigned long vaddr, vaddr_end, next_vaddr;
1284	unsigned long dst_vaddr;
1285	struct page **src_p, **dst_p;
1286	struct kvm_sev_dbg debug;
1287	unsigned long n;
1288	unsigned int size;
1289	int ret;
1290
1291	if (!sev_guest(kvm))
1292	return -ENOTTY;
1293
1294	if (copy_from_user(to: &debug, u64_to_user_ptr(argp->data), n: sizeof(debug)))
1295	return -EFAULT;
1296
1297	if (!debug.len || debug.src_uaddr + debug.len < debug.src_uaddr)
1298	return -EINVAL;
1299	if (!debug.dst_uaddr)
1300	return -EINVAL;
1301
1302	vaddr = debug.src_uaddr;
1303	size = debug.len;
1304	vaddr_end = vaddr + size;
1305	dst_vaddr = debug.dst_uaddr;
1306
1307	for (; vaddr < vaddr_end; vaddr = next_vaddr) {
1308	int len, s_off, d_off;
1309
1310	/* lock userspace source and destination page */
1311	src_p = sev_pin_memory(kvm, uaddr: vaddr & PAGE_MASK, PAGE_SIZE, n: &n, flags: 0);
1312	if (IS_ERR(ptr: src_p))
1313	return PTR_ERR(ptr: src_p);
1314
1315	dst_p = sev_pin_memory(kvm, uaddr: dst_vaddr & PAGE_MASK, PAGE_SIZE, n: &n, flags: FOLL_WRITE);
1316	if (IS_ERR(ptr: dst_p)) {
1317	sev_unpin_memory(kvm, pages: src_p, npages: n);
1318	return PTR_ERR(ptr: dst_p);
1319	}
1320
1321	/*
1322	* Flush (on non-coherent CPUs) before DBG_{DE,EN}CRYPT read or modify
1323	* the pages; flush the destination too so that future accesses do not
1324	* see stale data.
1325	*/
1326	sev_clflush_pages(pages: src_p, npages: 1);
1327	sev_clflush_pages(pages: dst_p, npages: 1);
1328
1329	/*
1330	* Since user buffer may not be page aligned, calculate the
1331	* offset within the page.
1332	*/
1333	s_off = vaddr & ~PAGE_MASK;
1334	d_off = dst_vaddr & ~PAGE_MASK;
1335	len = min_t(size_t, (PAGE_SIZE - s_off), size);
1336
1337	if (dec)
1338	ret = __sev_dbg_decrypt_user(kvm,
1339	paddr: __sme_page_pa(page: src_p[0]) + s_off,
1340	dst_uaddr: (void __user *)dst_vaddr,
1341	dst_paddr: __sme_page_pa(page: dst_p[0]) + d_off,
1342	size: len, err: &argp->error);
1343	else
1344	ret = __sev_dbg_encrypt_user(kvm,
1345	paddr: __sme_page_pa(page: src_p[0]) + s_off,
1346	vaddr: (void __user *)vaddr,
1347	dst_paddr: __sme_page_pa(page: dst_p[0]) + d_off,
1348	dst_vaddr: (void __user *)dst_vaddr,
1349	size: len, error: &argp->error);
1350
1351	sev_unpin_memory(kvm, pages: src_p, npages: n);
1352	sev_unpin_memory(kvm, pages: dst_p, npages: n);
1353
1354	if (ret)
1355	goto err;
1356
1357	next_vaddr = vaddr + len;
1358	dst_vaddr = dst_vaddr + len;
1359	size -= len;
1360	}
1361	err:
1362	return ret;
1363	}
1364
1365	static int sev_launch_secret(struct kvm *kvm, struct kvm_sev_cmd *argp)
1366	{
1367	struct sev_data_launch_secret data;
1368	struct kvm_sev_launch_secret params;
1369	struct page **pages;
1370	void *blob, *hdr;
1371	unsigned long n, i;
1372	int ret, offset;
1373
1374	if (!sev_guest(kvm))
1375	return -ENOTTY;
1376
1377	if (copy_from_user(to: &params, u64_to_user_ptr(argp->data), n: sizeof(params)))
1378	return -EFAULT;
1379
1380	pages = sev_pin_memory(kvm, uaddr: params.guest_uaddr, ulen: params.guest_len, n: &n, flags: FOLL_WRITE);
1381	if (IS_ERR(ptr: pages))
1382	return PTR_ERR(ptr: pages);
1383
1384	/*
1385	* Flush (on non-coherent CPUs) before LAUNCH_SECRET encrypts pages in
1386	* place; the cache may contain the data that was written unencrypted.
1387	*/
1388	sev_clflush_pages(pages, npages: n);
1389
1390	/*
1391	* The secret must be copied into contiguous memory region, lets verify
1392	* that userspace memory pages are contiguous before we issue command.
1393	*/
1394	if (get_num_contig_pages(idx: 0, inpages: pages, npages: n) != n) {
1395	ret = -EINVAL;
1396	goto e_unpin_memory;
1397	}
1398
1399	memset(&data, 0, sizeof(data));
1400
1401	offset = params.guest_uaddr & (PAGE_SIZE - 1);
1402	data.guest_address = __sme_page_pa(page: pages[0]) + offset;
1403	data.guest_len = params.guest_len;
1404
1405	blob = psp_copy_user_blob(uaddr: params.trans_uaddr, len: params.trans_len);
1406	if (IS_ERR(ptr: blob)) {
1407	ret = PTR_ERR(ptr: blob);
1408	goto e_unpin_memory;
1409	}
1410
1411	data.trans_address = __psp_pa(blob);
1412	data.trans_len = params.trans_len;
1413
1414	hdr = psp_copy_user_blob(uaddr: params.hdr_uaddr, len: params.hdr_len);
1415	if (IS_ERR(ptr: hdr)) {
1416	ret = PTR_ERR(ptr: hdr);
1417	goto e_free_blob;
1418	}
1419	data.hdr_address = __psp_pa(hdr);
1420	data.hdr_len = params.hdr_len;
1421
1422	data.handle = to_kvm_sev_info(kvm)->handle;
1423	ret = sev_issue_cmd(kvm, id: SEV_CMD_LAUNCH_UPDATE_SECRET, data: &data, error: &argp->error);
1424
1425	kfree(objp: hdr);
1426
1427	e_free_blob:
1428	kfree(objp: blob);
1429	e_unpin_memory:
1430	/* content of memory is updated, mark pages dirty */
1431	for (i = 0; i < n; i++) {
1432	set_page_dirty_lock(pages[i]);
1433	mark_page_accessed(pages[i]);
1434	}
1435	sev_unpin_memory(kvm, pages, npages: n);
1436	return ret;
1437	}
1438
1439	static int sev_get_attestation_report(struct kvm *kvm, struct kvm_sev_cmd *argp)
1440	{
1441	void __user *report = u64_to_user_ptr(argp->data);
1442	struct sev_data_attestation_report data;
1443	struct kvm_sev_attestation_report params;
1444	void __user *p;
1445	void *blob = NULL;
1446	int ret;
1447
1448	if (!sev_guest(kvm))
1449	return -ENOTTY;
1450
1451	if (copy_from_user(to: &params, u64_to_user_ptr(argp->data), n: sizeof(params)))
1452	return -EFAULT;
1453
1454	memset(&data, 0, sizeof(data));
1455
1456	/* User wants to query the blob length */
1457	if (!params.len)
1458	goto cmd;
1459
1460	p = u64_to_user_ptr(params.uaddr);
1461	if (p) {
1462	if (params.len > SEV_FW_BLOB_MAX_SIZE)
1463	return -EINVAL;
1464
1465	blob = kzalloc(params.len, GFP_KERNEL_ACCOUNT);
1466	if (!blob)
1467	return -ENOMEM;
1468
1469	data.address = __psp_pa(blob);
1470	data.len = params.len;
1471	memcpy(data.mnonce, params.mnonce, sizeof(params.mnonce));
1472	}
1473	cmd:
1474	data.handle = to_kvm_sev_info(kvm)->handle;
1475	ret = sev_issue_cmd(kvm, id: SEV_CMD_ATTESTATION_REPORT, data: &data, error: &argp->error);
1476	/*
1477	* If we query the session length, FW responded with expected data.
1478	*/
1479	if (!params.len)
1480	goto done;
1481
1482	if (ret)
1483	goto e_free_blob;
1484
1485	if (blob) {
1486	if (copy_to_user(to: p, from: blob, n: params.len))
1487	ret = -EFAULT;
1488	}
1489
1490	done:
1491	params.len = data.len;
1492	if (copy_to_user(to: report, from: &params, n: sizeof(params)))
1493	ret = -EFAULT;
1494	e_free_blob:
1495	kfree(objp: blob);
1496	return ret;
1497	}
1498
1499	/* Userspace wants to query session length. */
1500	static int
1501	__sev_send_start_query_session_length(struct kvm *kvm, struct kvm_sev_cmd *argp,
1502	struct kvm_sev_send_start *params)
1503	{
1504	struct sev_data_send_start data;
1505	int ret;
1506
1507	memset(&data, 0, sizeof(data));
1508	data.handle = to_kvm_sev_info(kvm)->handle;
1509	ret = sev_issue_cmd(kvm, id: SEV_CMD_SEND_START, data: &data, error: &argp->error);
1510
1511	params->session_len = data.session_len;
1512	if (copy_to_user(u64_to_user_ptr(argp->data), from: params,
1513	n: sizeof(struct kvm_sev_send_start)))
1514	ret = -EFAULT;
1515
1516	return ret;
1517	}
1518
1519	static int sev_send_start(struct kvm *kvm, struct kvm_sev_cmd *argp)
1520	{
1521	struct sev_data_send_start data;
1522	struct kvm_sev_send_start params;
1523	void *amd_certs, *session_data;
1524	void *pdh_cert, *plat_certs;
1525	int ret;
1526
1527	if (!sev_guest(kvm))
1528	return -ENOTTY;
1529
1530	if (copy_from_user(to: &params, u64_to_user_ptr(argp->data),
1531	n: sizeof(struct kvm_sev_send_start)))
1532	return -EFAULT;
1533
1534	/* if session_len is zero, userspace wants to query the session length */
1535	if (!params.session_len)
1536	return __sev_send_start_query_session_length(kvm, argp,
1537	params: &params);
1538
1539	/* some sanity checks */
1540	if (!params.pdh_cert_uaddr || !params.pdh_cert_len ||
1541	!params.session_uaddr || params.session_len > SEV_FW_BLOB_MAX_SIZE)
1542	return -EINVAL;
1543
1544	/* allocate the memory to hold the session data blob */
1545	session_data = kzalloc(params.session_len, GFP_KERNEL_ACCOUNT);
1546	if (!session_data)
1547	return -ENOMEM;
1548
1549	/* copy the certificate blobs from userspace */
1550	pdh_cert = psp_copy_user_blob(uaddr: params.pdh_cert_uaddr,
1551	len: params.pdh_cert_len);
1552	if (IS_ERR(ptr: pdh_cert)) {
1553	ret = PTR_ERR(ptr: pdh_cert);
1554	goto e_free_session;
1555	}
1556
1557	plat_certs = psp_copy_user_blob(uaddr: params.plat_certs_uaddr,
1558	len: params.plat_certs_len);
1559	if (IS_ERR(ptr: plat_certs)) {
1560	ret = PTR_ERR(ptr: plat_certs);
1561	goto e_free_pdh;
1562	}
1563
1564	amd_certs = psp_copy_user_blob(uaddr: params.amd_certs_uaddr,
1565	len: params.amd_certs_len);
1566	if (IS_ERR(ptr: amd_certs)) {
1567	ret = PTR_ERR(ptr: amd_certs);
1568	goto e_free_plat_cert;
1569	}
1570
1571	/* populate the FW SEND_START field with system physical address */
1572	memset(&data, 0, sizeof(data));
1573	data.pdh_cert_address = __psp_pa(pdh_cert);
1574	data.pdh_cert_len = params.pdh_cert_len;
1575	data.plat_certs_address = __psp_pa(plat_certs);
1576	data.plat_certs_len = params.plat_certs_len;
1577	data.amd_certs_address = __psp_pa(amd_certs);
1578	data.amd_certs_len = params.amd_certs_len;
1579	data.session_address = __psp_pa(session_data);
1580	data.session_len = params.session_len;
1581	data.handle = to_kvm_sev_info(kvm)->handle;
1582
1583	ret = sev_issue_cmd(kvm, id: SEV_CMD_SEND_START, data: &data, error: &argp->error);
1584
1585	if (!ret && copy_to_user(u64_to_user_ptr(params.session_uaddr),
1586	from: session_data, n: params.session_len)) {
1587	ret = -EFAULT;
1588	goto e_free_amd_cert;
1589	}
1590
1591	params.policy = data.policy;
1592	params.session_len = data.session_len;
1593	if (copy_to_user(u64_to_user_ptr(argp->data), from: &params,
1594	n: sizeof(struct kvm_sev_send_start)))
1595	ret = -EFAULT;
1596
1597	e_free_amd_cert:
1598	kfree(objp: amd_certs);
1599	e_free_plat_cert:
1600	kfree(objp: plat_certs);
1601	e_free_pdh:
1602	kfree(objp: pdh_cert);
1603	e_free_session:
1604	kfree(objp: session_data);
1605	return ret;
1606	}
1607
1608	/* Userspace wants to query either header or trans length. */
1609	static int
1610	__sev_send_update_data_query_lengths(struct kvm *kvm, struct kvm_sev_cmd *argp,
1611	struct kvm_sev_send_update_data *params)
1612	{
1613	struct sev_data_send_update_data data;
1614	int ret;
1615
1616	memset(&data, 0, sizeof(data));
1617	data.handle = to_kvm_sev_info(kvm)->handle;
1618	ret = sev_issue_cmd(kvm, id: SEV_CMD_SEND_UPDATE_DATA, data: &data, error: &argp->error);
1619
1620	params->hdr_len = data.hdr_len;
1621	params->trans_len = data.trans_len;
1622
1623	if (copy_to_user(u64_to_user_ptr(argp->data), from: params,
1624	n: sizeof(struct kvm_sev_send_update_data)))
1625	ret = -EFAULT;
1626
1627	return ret;
1628	}
1629
1630	static int sev_send_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
1631	{
1632	struct sev_data_send_update_data data;
1633	struct kvm_sev_send_update_data params;
1634	void *hdr, *trans_data;
1635	struct page **guest_page;
1636	unsigned long n;
1637	int ret, offset;
1638
1639	if (!sev_guest(kvm))
1640	return -ENOTTY;
1641
1642	if (copy_from_user(to: &params, u64_to_user_ptr(argp->data),
1643	n: sizeof(struct kvm_sev_send_update_data)))
1644	return -EFAULT;
1645
1646	/* userspace wants to query either header or trans length */
1647	if (!params.trans_len || !params.hdr_len)
1648	return __sev_send_update_data_query_lengths(kvm, argp, params: &params);
1649
1650	if (!params.trans_uaddr || !params.guest_uaddr ||
1651	!params.guest_len || !params.hdr_uaddr)
1652	return -EINVAL;
1653
1654	/* Check if we are crossing the page boundary */
1655	offset = params.guest_uaddr & (PAGE_SIZE - 1);
1656	if (params.guest_len > PAGE_SIZE || (params.guest_len + offset) > PAGE_SIZE)
1657	return -EINVAL;
1658
1659	/* Pin guest memory */
1660	guest_page = sev_pin_memory(kvm, uaddr: params.guest_uaddr & PAGE_MASK,
1661	PAGE_SIZE, n: &n, flags: 0);
1662	if (IS_ERR(ptr: guest_page))
1663	return PTR_ERR(ptr: guest_page);
1664
1665	/* allocate memory for header and transport buffer */
1666	ret = -ENOMEM;
1667	hdr = kzalloc(params.hdr_len, GFP_KERNEL);
1668	if (!hdr)
1669	goto e_unpin;
1670
1671	trans_data = kzalloc(params.trans_len, GFP_KERNEL);
1672	if (!trans_data)
1673	goto e_free_hdr;
1674
1675	memset(&data, 0, sizeof(data));
1676	data.hdr_address = __psp_pa(hdr);
1677	data.hdr_len = params.hdr_len;
1678	data.trans_address = __psp_pa(trans_data);
1679	data.trans_len = params.trans_len;
1680
1681	/* The SEND_UPDATE_DATA command requires C-bit to be always set. */
1682	data.guest_address = (page_to_pfn(guest_page[0]) << PAGE_SHIFT) + offset;
1683	data.guest_address |= sev_me_mask;
1684	data.guest_len = params.guest_len;
1685	data.handle = to_kvm_sev_info(kvm)->handle;
1686
1687	ret = sev_issue_cmd(kvm, id: SEV_CMD_SEND_UPDATE_DATA, data: &data, error: &argp->error);
1688
1689	if (ret)
1690	goto e_free_trans_data;
1691
1692	/* copy transport buffer to user space */
1693	if (copy_to_user(u64_to_user_ptr(params.trans_uaddr),
1694	from: trans_data, n: params.trans_len)) {
1695	ret = -EFAULT;
1696	goto e_free_trans_data;
1697	}
1698
1699	/* Copy packet header to userspace. */
1700	if (copy_to_user(u64_to_user_ptr(params.hdr_uaddr), from: hdr,
1701	n: params.hdr_len))
1702	ret = -EFAULT;
1703
1704	e_free_trans_data:
1705	kfree(objp: trans_data);
1706	e_free_hdr:
1707	kfree(objp: hdr);
1708	e_unpin:
1709	sev_unpin_memory(kvm, pages: guest_page, npages: n);
1710
1711	return ret;
1712	}
1713
1714	static int sev_send_finish(struct kvm *kvm, struct kvm_sev_cmd *argp)
1715	{
1716	struct sev_data_send_finish data;
1717
1718	if (!sev_guest(kvm))
1719	return -ENOTTY;
1720
1721	data.handle = to_kvm_sev_info(kvm)->handle;
1722	return sev_issue_cmd(kvm, id: SEV_CMD_SEND_FINISH, data: &data, error: &argp->error);
1723	}
1724
1725	static int sev_send_cancel(struct kvm *kvm, struct kvm_sev_cmd *argp)
1726	{
1727	struct sev_data_send_cancel data;
1728
1729	if (!sev_guest(kvm))
1730	return -ENOTTY;
1731
1732	data.handle = to_kvm_sev_info(kvm)->handle;
1733	return sev_issue_cmd(kvm, id: SEV_CMD_SEND_CANCEL, data: &data, error: &argp->error);
1734	}
1735
1736	static int sev_receive_start(struct kvm *kvm, struct kvm_sev_cmd *argp)
1737	{
1738	struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
1739	struct sev_data_receive_start start;
1740	struct kvm_sev_receive_start params;
1741	int *error = &argp->error;
1742	void *session_data;
1743	void *pdh_data;
1744	int ret;
1745
1746	if (!sev_guest(kvm))
1747	return -ENOTTY;
1748
1749	/* Get parameter from the userspace */
1750	if (copy_from_user(to: &params, u64_to_user_ptr(argp->data),
1751	n: sizeof(struct kvm_sev_receive_start)))
1752	return -EFAULT;
1753
1754	/* some sanity checks */
1755	if (!params.pdh_uaddr || !params.pdh_len ||
1756	!params.session_uaddr || !params.session_len)
1757	return -EINVAL;
1758
1759	pdh_data = psp_copy_user_blob(uaddr: params.pdh_uaddr, len: params.pdh_len);
1760	if (IS_ERR(ptr: pdh_data))
1761	return PTR_ERR(ptr: pdh_data);
1762
1763	session_data = psp_copy_user_blob(uaddr: params.session_uaddr,
1764	len: params.session_len);
1765	if (IS_ERR(ptr: session_data)) {
1766	ret = PTR_ERR(ptr: session_data);
1767	goto e_free_pdh;
1768	}
1769
1770	memset(&start, 0, sizeof(start));
1771	start.handle = params.handle;
1772	start.policy = params.policy;
1773	start.pdh_cert_address = __psp_pa(pdh_data);
1774	start.pdh_cert_len = params.pdh_len;
1775	start.session_address = __psp_pa(session_data);
1776	start.session_len = params.session_len;
1777
1778	/* create memory encryption context */
1779	ret = __sev_issue_cmd(fd: argp->sev_fd, id: SEV_CMD_RECEIVE_START, data: &start,
1780	error);
1781	if (ret)
1782	goto e_free_session;
1783
1784	/* Bind ASID to this guest */
1785	ret = sev_bind_asid(kvm, handle: start.handle, error);
1786	if (ret) {
1787	sev_decommission(handle: start.handle);
1788	goto e_free_session;
1789	}
1790
1791	params.handle = start.handle;
1792	if (copy_to_user(u64_to_user_ptr(argp->data),
1793	from: &params, n: sizeof(struct kvm_sev_receive_start))) {
1794	ret = -EFAULT;
1795	sev_unbind_asid(kvm, handle: start.handle);
1796	goto e_free_session;
1797	}
1798
1799	sev->handle = start.handle;
1800	sev->fd = argp->sev_fd;
1801
1802	e_free_session:
1803	kfree(objp: session_data);
1804	e_free_pdh:
1805	kfree(objp: pdh_data);
1806
1807	return ret;
1808	}
1809
1810	static int sev_receive_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
1811	{
1812	struct kvm_sev_receive_update_data params;
1813	struct sev_data_receive_update_data data;
1814	void *hdr = NULL, *trans = NULL;
1815	struct page **guest_page;
1816	unsigned long n;
1817	int ret, offset;
1818
1819	if (!sev_guest(kvm))
1820	return -EINVAL;
1821
1822	if (copy_from_user(to: &params, u64_to_user_ptr(argp->data),
1823	n: sizeof(struct kvm_sev_receive_update_data)))
1824	return -EFAULT;
1825
1826	if (!params.hdr_uaddr || !params.hdr_len ||
1827	!params.guest_uaddr || !params.guest_len ||
1828	!params.trans_uaddr || !params.trans_len)
1829	return -EINVAL;
1830
1831	/* Check if we are crossing the page boundary */
1832	offset = params.guest_uaddr & (PAGE_SIZE - 1);
1833	if (params.guest_len > PAGE_SIZE || (params.guest_len + offset) > PAGE_SIZE)
1834	return -EINVAL;
1835
1836	hdr = psp_copy_user_blob(uaddr: params.hdr_uaddr, len: params.hdr_len);
1837	if (IS_ERR(ptr: hdr))
1838	return PTR_ERR(ptr: hdr);
1839
1840	trans = psp_copy_user_blob(uaddr: params.trans_uaddr, len: params.trans_len);
1841	if (IS_ERR(ptr: trans)) {
1842	ret = PTR_ERR(ptr: trans);
1843	goto e_free_hdr;
1844	}
1845
1846	memset(&data, 0, sizeof(data));
1847	data.hdr_address = __psp_pa(hdr);
1848	data.hdr_len = params.hdr_len;
1849	data.trans_address = __psp_pa(trans);
1850	data.trans_len = params.trans_len;
1851
1852	/* Pin guest memory */
1853	guest_page = sev_pin_memory(kvm, uaddr: params.guest_uaddr & PAGE_MASK,
1854	PAGE_SIZE, n: &n, flags: FOLL_WRITE);
1855	if (IS_ERR(ptr: guest_page)) {
1856	ret = PTR_ERR(ptr: guest_page);
1857	goto e_free_trans;
1858	}
1859
1860	/*
1861	* Flush (on non-coherent CPUs) before RECEIVE_UPDATE_DATA, the PSP
1862	* encrypts the written data with the guest's key, and the cache may
1863	* contain dirty, unencrypted data.
1864	*/
1865	sev_clflush_pages(pages: guest_page, npages: n);
1866
1867	/* The RECEIVE_UPDATE_DATA command requires C-bit to be always set. */
1868	data.guest_address = (page_to_pfn(guest_page[0]) << PAGE_SHIFT) + offset;
1869	data.guest_address |= sev_me_mask;
1870	data.guest_len = params.guest_len;
1871	data.handle = to_kvm_sev_info(kvm)->handle;
1872
1873	ret = sev_issue_cmd(kvm, id: SEV_CMD_RECEIVE_UPDATE_DATA, data: &data,
1874	error: &argp->error);
1875
1876	sev_unpin_memory(kvm, pages: guest_page, npages: n);
1877
1878	e_free_trans:
1879	kfree(objp: trans);
1880	e_free_hdr:
1881	kfree(objp: hdr);
1882
1883	return ret;
1884	}
1885
1886	static int sev_receive_finish(struct kvm *kvm, struct kvm_sev_cmd *argp)
1887	{
1888	struct sev_data_receive_finish data;
1889
1890	if (!sev_guest(kvm))
1891	return -ENOTTY;
1892
1893	data.handle = to_kvm_sev_info(kvm)->handle;
1894	return sev_issue_cmd(kvm, id: SEV_CMD_RECEIVE_FINISH, data: &data, error: &argp->error);
1895	}
1896
1897	static bool is_cmd_allowed_from_mirror(u32 cmd_id)
1898	{
1899	/*
1900	* Allow mirrors VM to call KVM_SEV_LAUNCH_UPDATE_VMSA to enable SEV-ES
1901	* active mirror VMs. Also allow the debugging and status commands.
1902	*/
1903	if (cmd_id == KVM_SEV_LAUNCH_UPDATE_VMSA ||
1904	cmd_id == KVM_SEV_GUEST_STATUS || cmd_id == KVM_SEV_DBG_DECRYPT ||
1905	cmd_id == KVM_SEV_DBG_ENCRYPT)
1906	return true;
1907
1908	return false;
1909	}
1910
1911	static int sev_lock_two_vms(struct kvm *dst_kvm, struct kvm *src_kvm)
1912	{
1913	struct kvm_sev_info *dst_sev = to_kvm_sev_info(kvm: dst_kvm);
1914	struct kvm_sev_info *src_sev = to_kvm_sev_info(kvm: src_kvm);
1915	int r = -EBUSY;
1916
1917	if (dst_kvm == src_kvm)
1918	return -EINVAL;
1919
1920	/*
1921	* Bail if these VMs are already involved in a migration to avoid
1922	* deadlock between two VMs trying to migrate to/from each other.
1923	*/
1924	if (atomic_cmpxchg_acquire(v: &dst_sev->migration_in_progress, old: 0, new: 1))
1925	return -EBUSY;
1926
1927	if (atomic_cmpxchg_acquire(v: &src_sev->migration_in_progress, old: 0, new: 1))
1928	goto release_dst;
1929
1930	r = -EINTR;
1931	if (mutex_lock_killable(&dst_kvm->lock))
1932	goto release_src;
1933	if (mutex_lock_killable_nested(&src_kvm->lock, SINGLE_DEPTH_NESTING))
1934	goto unlock_dst;
1935	return 0;
1936
1937	unlock_dst:
1938	mutex_unlock(lock: &dst_kvm->lock);
1939	release_src:
1940	atomic_set_release(v: &src_sev->migration_in_progress, i: 0);
1941	release_dst:
1942	atomic_set_release(v: &dst_sev->migration_in_progress, i: 0);
1943	return r;
1944	}
1945
1946	static void sev_unlock_two_vms(struct kvm *dst_kvm, struct kvm *src_kvm)
1947	{
1948	struct kvm_sev_info *dst_sev = to_kvm_sev_info(kvm: dst_kvm);
1949	struct kvm_sev_info *src_sev = to_kvm_sev_info(kvm: src_kvm);
1950
1951	mutex_unlock(lock: &dst_kvm->lock);
1952	mutex_unlock(lock: &src_kvm->lock);
1953	atomic_set_release(v: &dst_sev->migration_in_progress, i: 0);
1954	atomic_set_release(v: &src_sev->migration_in_progress, i: 0);
1955	}
1956
1957	static void sev_migrate_from(struct kvm *dst_kvm, struct kvm *src_kvm)
1958	{
1959	struct kvm_sev_info *dst = to_kvm_sev_info(kvm: dst_kvm);
1960	struct kvm_sev_info *src = to_kvm_sev_info(kvm: src_kvm);
1961	struct kvm_vcpu *dst_vcpu, *src_vcpu;
1962	struct vcpu_svm *dst_svm, *src_svm;
1963	struct kvm_sev_info *mirror;
1964	unsigned long i;
1965
1966	dst->active = true;
1967	dst->asid = src->asid;
1968	dst->handle = src->handle;
1969	dst->pages_locked = src->pages_locked;
1970	dst->enc_context_owner = src->enc_context_owner;
1971	dst->es_active = src->es_active;
1972	dst->vmsa_features = src->vmsa_features;
1973
1974	src->asid = 0;
1975	src->active = false;
1976	src->handle = 0;
1977	src->pages_locked = 0;
1978	src->enc_context_owner = NULL;
1979	src->es_active = false;
1980
1981	list_cut_before(list: &dst->regions_list, head: &src->regions_list, entry: &src->regions_list);
1982
1983	/*
1984	* If this VM has mirrors, "transfer" each mirror's refcount of the
1985	* source to the destination (this KVM). The caller holds a reference
1986	* to the source, so there's no danger of use-after-free.
1987	*/
1988	list_cut_before(list: &dst->mirror_vms, head: &src->mirror_vms, entry: &src->mirror_vms);
1989	list_for_each_entry(mirror, &dst->mirror_vms, mirror_entry) {
1990	kvm_get_kvm(kvm: dst_kvm);
1991	kvm_put_kvm(kvm: src_kvm);
1992	mirror->enc_context_owner = dst_kvm;
1993	}
1994
1995	/*
1996	* If this VM is a mirror, remove the old mirror from the owners list
1997	* and add the new mirror to the list.
1998	*/
1999	if (is_mirroring_enc_context(kvm: dst_kvm)) {
2000	struct kvm_sev_info *owner_sev_info = to_kvm_sev_info(kvm: dst->enc_context_owner);
2001
2002	list_del(entry: &src->mirror_entry);
2003	list_add_tail(new: &dst->mirror_entry, head: &owner_sev_info->mirror_vms);
2004	}
2005
2006	kvm_for_each_vcpu(i, dst_vcpu, dst_kvm) {
2007	dst_svm = to_svm(vcpu: dst_vcpu);
2008
2009	sev_init_vmcb(svm: dst_svm, init_event: false);
2010
2011	if (!dst->es_active)
2012	continue;
2013
2014	/*
2015	* Note, the source is not required to have the same number of
2016	* vCPUs as the destination when migrating a vanilla SEV VM.
2017	*/
2018	src_vcpu = kvm_get_vcpu(kvm: src_kvm, i);
2019	src_svm = to_svm(vcpu: src_vcpu);
2020
2021	/*
2022	* Transfer VMSA and GHCB state to the destination. Nullify and
2023	* clear source fields as appropriate, the state now belongs to
2024	* the destination.
2025	*/
2026	memcpy(&dst_svm->sev_es, &src_svm->sev_es, sizeof(src_svm->sev_es));
2027	dst_svm->vmcb->control.ghcb_gpa = src_svm->vmcb->control.ghcb_gpa;
2028	dst_svm->vmcb->control.vmsa_pa = src_svm->vmcb->control.vmsa_pa;
2029	dst_vcpu->arch.guest_state_protected = true;
2030
2031	memset(&src_svm->sev_es, 0, sizeof(src_svm->sev_es));
2032	src_svm->vmcb->control.ghcb_gpa = INVALID_PAGE;
2033	src_svm->vmcb->control.vmsa_pa = INVALID_PAGE;
2034	src_vcpu->arch.guest_state_protected = false;
2035	}
2036	}
2037
2038	static int sev_check_source_vcpus(struct kvm *dst, struct kvm *src)
2039	{
2040	struct kvm_vcpu *src_vcpu;
2041	unsigned long i;
2042
2043	if (src->created_vcpus != atomic_read(v: &src->online_vcpus) ||
2044	dst->created_vcpus != atomic_read(v: &dst->online_vcpus))
2045	return -EBUSY;
2046
2047	if (!sev_es_guest(kvm: src))
2048	return 0;
2049
2050	if (atomic_read(v: &src->online_vcpus) != atomic_read(v: &dst->online_vcpus))
2051	return -EINVAL;
2052
2053	kvm_for_each_vcpu(i, src_vcpu, src) {
2054	if (!src_vcpu->arch.guest_state_protected)
2055	return -EINVAL;
2056	}
2057
2058	return 0;
2059	}
2060
2061	int sev_vm_move_enc_context_from(struct kvm *kvm, unsigned int source_fd)
2062	{
2063	struct kvm_sev_info *dst_sev = to_kvm_sev_info(kvm);
2064	struct kvm_sev_info *src_sev, *cg_cleanup_sev;
2065	CLASS(fd, f)(fd: source_fd);
2066	struct kvm *source_kvm;
2067	bool charged = false;
2068	int ret;
2069
2070	if (fd_empty(f))
2071	return -EBADF;
2072
2073	if (!file_is_kvm(fd_file(f)))
2074	return -EBADF;
2075
2076	source_kvm = fd_file(f)->private_data;
2077	ret = sev_lock_two_vms(dst_kvm: kvm, src_kvm: source_kvm);
2078	if (ret)
2079	return ret;
2080
2081	if (kvm->arch.vm_type != source_kvm->arch.vm_type ||
2082	sev_guest(kvm) || !sev_guest(kvm: source_kvm)) {
2083	ret = -EINVAL;
2084	goto out_unlock;
2085	}
2086
2087	src_sev = to_kvm_sev_info(kvm: source_kvm);
2088
2089	dst_sev->misc_cg = get_current_misc_cg();
2090	cg_cleanup_sev = dst_sev;
2091	if (dst_sev->misc_cg != src_sev->misc_cg) {
2092	ret = sev_misc_cg_try_charge(sev: dst_sev);
2093	if (ret)
2094	goto out_dst_cgroup;
2095	charged = true;
2096	}
2097
2098	ret = kvm_lock_all_vcpus(kvm);
2099	if (ret)
2100	goto out_dst_cgroup;
2101	ret = kvm_lock_all_vcpus(kvm: source_kvm);
2102	if (ret)
2103	goto out_dst_vcpu;
2104
2105	ret = sev_check_source_vcpus(dst: kvm, src: source_kvm);
2106	if (ret)
2107	goto out_source_vcpu;
2108
2109	/*
2110	* Allocate a new have_run_cpus for the destination, i.e. don't copy
2111	* the set of CPUs from the source. If a CPU was used to run a vCPU in
2112	* the source VM but is never used for the destination VM, then the CPU
2113	* can only have cached memory that was accessible to the source VM.
2114	*/
2115	if (!zalloc_cpumask_var(mask: &dst_sev->have_run_cpus, GFP_KERNEL_ACCOUNT)) {
2116	ret = -ENOMEM;
2117	goto out_source_vcpu;
2118	}
2119
2120	sev_migrate_from(dst_kvm: kvm, src_kvm: source_kvm);
2121	kvm_vm_dead(kvm: source_kvm);
2122	cg_cleanup_sev = src_sev;
2123	ret = 0;
2124
2125	out_source_vcpu:
2126	kvm_unlock_all_vcpus(kvm: source_kvm);
2127	out_dst_vcpu:
2128	kvm_unlock_all_vcpus(kvm);
2129	out_dst_cgroup:
2130	/* Operates on the source on success, on the destination on failure. */
2131	if (charged)
2132	sev_misc_cg_uncharge(sev: cg_cleanup_sev);
2133	put_misc_cg(cg: cg_cleanup_sev->misc_cg);
2134	cg_cleanup_sev->misc_cg = NULL;
2135	out_unlock:
2136	sev_unlock_two_vms(dst_kvm: kvm, src_kvm: source_kvm);
2137	return ret;
2138	}
2139
2140	int sev_dev_get_attr(u32 group, u64 attr, u64 *val)
2141	{
2142	if (group != KVM_X86_GRP_SEV)
2143	return -ENXIO;
2144
2145	switch (attr) {
2146	case KVM_X86_SEV_VMSA_FEATURES:
2147	*val = sev_supported_vmsa_features;
2148	return 0;
2149
2150	case KVM_X86_SNP_POLICY_BITS:
2151	*val = snp_supported_policy_bits;
2152	return 0;
2153
2154	default:
2155	return -ENXIO;
2156	}
2157	}
2158
2159	/*
2160	* The guest context contains all the information, keys and metadata
2161	* associated with the guest that the firmware tracks to implement SEV
2162	* and SNP features. The firmware stores the guest context in hypervisor
2163	* provide page via the SNP_GCTX_CREATE command.
2164	*/
2165	static void *snp_context_create(struct kvm *kvm, struct kvm_sev_cmd *argp)
2166	{
2167	struct sev_data_snp_addr data = {};
2168	void *context;
2169	int rc;
2170
2171	/* Allocate memory for context page */
2172	context = snp_alloc_firmware_page(GFP_KERNEL_ACCOUNT);
2173	if (!context)
2174	return NULL;
2175
2176	data.address = __psp_pa(context);
2177	rc = __sev_issue_cmd(fd: argp->sev_fd, id: SEV_CMD_SNP_GCTX_CREATE, data: &data, error: &argp->error);
2178	if (rc) {
2179	pr_warn("Failed to create SEV-SNP context, rc %d fw_error %d",
2180	rc, argp->error);
2181	snp_free_firmware_page(addr: context);
2182	return NULL;
2183	}
2184
2185	return context;
2186	}
2187
2188	static int snp_bind_asid(struct kvm *kvm, int *error)
2189	{
2190	struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
2191	struct sev_data_snp_activate data = {0};
2192
2193	data.gctx_paddr = __psp_pa(sev->snp_context);
2194	data.asid = sev_get_asid(kvm);
2195	return sev_issue_cmd(kvm, id: SEV_CMD_SNP_ACTIVATE, data: &data, error);
2196	}
2197
2198	static int snp_launch_start(struct kvm *kvm, struct kvm_sev_cmd *argp)
2199	{
2200	struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
2201	struct sev_data_snp_launch_start start = {0};
2202	struct kvm_sev_snp_launch_start params;
2203	int rc;
2204
2205	if (!sev_snp_guest(kvm))
2206	return -ENOTTY;
2207
2208	if (copy_from_user(to: &params, u64_to_user_ptr(argp->data), n: sizeof(params)))
2209	return -EFAULT;
2210
2211	/* Don't allow userspace to allocate memory for more than 1 SNP context. */
2212	if (sev->snp_context)
2213	return -EINVAL;
2214
2215	if (params.flags)
2216	return -EINVAL;
2217
2218	if (params.policy & ~snp_supported_policy_bits)
2219	return -EINVAL;
2220
2221	/* Check for policy bits that must be set */
2222	if (!(params.policy & SNP_POLICY_MASK_RSVD_MBO))
2223	return -EINVAL;
2224
2225	if (snp_is_secure_tsc_enabled(kvm)) {
2226	if (WARN_ON_ONCE(!kvm->arch.default_tsc_khz))
2227	return -EINVAL;
2228
2229	start.desired_tsc_khz = kvm->arch.default_tsc_khz;
2230	}
2231
2232	sev->snp_context = snp_context_create(kvm, argp);
2233	if (!sev->snp_context)
2234	return -ENOTTY;
2235
2236	start.gctx_paddr = __psp_pa(sev->snp_context);
2237	start.policy = params.policy;
2238
2239	memcpy(start.gosvw, params.gosvw, sizeof(params.gosvw));
2240	rc = __sev_issue_cmd(fd: argp->sev_fd, id: SEV_CMD_SNP_LAUNCH_START, data: &start, error: &argp->error);
2241	if (rc) {
2242	pr_debug("%s: SEV_CMD_SNP_LAUNCH_START firmware command failed, rc %d\n",
2243	__func__, rc);
2244	goto e_free_context;
2245	}
2246
2247	sev->policy = params.policy;
2248	sev->fd = argp->sev_fd;
2249	rc = snp_bind_asid(kvm, error: &argp->error);
2250	if (rc) {
2251	pr_debug("%s: Failed to bind ASID to SEV-SNP context, rc %d\n",
2252	__func__, rc);
2253	goto e_free_context;
2254	}
2255
2256	return 0;
2257
2258	e_free_context:
2259	snp_decommission_context(kvm);
2260
2261	return rc;
2262	}
2263
2264	struct sev_gmem_populate_args {
2265	__u8 type;
2266	int sev_fd;
2267	int fw_error;
2268	};
2269
2270	static int sev_gmem_post_populate(struct kvm *kvm, gfn_t gfn_start, kvm_pfn_t pfn,
2271	void __user *src, int order, void *opaque)
2272	{
2273	struct sev_gmem_populate_args *sev_populate_args = opaque;
2274	struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
2275	int n_private = 0, ret, i;
2276	int npages = (1 << order);
2277	gfn_t gfn;
2278
2279	if (WARN_ON_ONCE(sev_populate_args->type != KVM_SEV_SNP_PAGE_TYPE_ZERO && !src))
2280	return -EINVAL;
2281
2282	for (gfn = gfn_start, i = 0; gfn < gfn_start + npages; gfn++, i++) {
2283	struct sev_data_snp_launch_update fw_args = {0};
2284	bool assigned = false;
2285	int level;
2286
2287	ret = snp_lookup_rmpentry(pfn: (u64)pfn + i, assigned: &assigned, level: &level);
2288	if (ret || assigned) {
2289	pr_debug("%s: Failed to ensure GFN 0x%llx RMP entry is initial shared state, ret: %d assigned: %d\n",
2290	__func__, gfn, ret, assigned);
2291	ret = ret ? -EINVAL : -EEXIST;
2292	goto err;
2293	}
2294
2295	if (src) {
2296	void *vaddr = kmap_local_pfn(pfn: pfn + i);
2297
2298	if (copy_from_user(to: vaddr, from: src + i * PAGE_SIZE, PAGE_SIZE)) {
2299	ret = -EFAULT;
2300	goto err;
2301	}
2302	kunmap_local(vaddr);
2303	}
2304
2305	ret = rmp_make_private(pfn: pfn + i, gpa: gfn << PAGE_SHIFT, level: PG_LEVEL_4K,
2306	asid: sev_get_asid(kvm), immutable: true);
2307	if (ret)
2308	goto err;
2309
2310	n_private++;
2311
2312	fw_args.gctx_paddr = __psp_pa(sev->snp_context);
2313	fw_args.address = __sme_set(pfn_to_hpa(pfn + i));
2314	fw_args.page_size = PG_LEVEL_TO_RMP(PG_LEVEL_4K);
2315	fw_args.page_type = sev_populate_args->type;
2316
2317	ret = __sev_issue_cmd(fd: sev_populate_args->sev_fd, id: SEV_CMD_SNP_LAUNCH_UPDATE,
2318	data: &fw_args, error: &sev_populate_args->fw_error);
2319	if (ret)
2320	goto fw_err;
2321	}
2322
2323	return 0;
2324
2325	fw_err:
2326	/*
2327	* If the firmware command failed handle the reclaim and cleanup of that
2328	* PFN specially vs. prior pages which can be cleaned up below without
2329	* needing to reclaim in advance.
2330	*
2331	* Additionally, when invalid CPUID function entries are detected,
2332	* firmware writes the expected values into the page and leaves it
2333	* unencrypted so it can be used for debugging and error-reporting.
2334	*
2335	* Copy this page back into the source buffer so userspace can use this
2336	* information to provide information on which CPUID leaves/fields
2337	* failed CPUID validation.
2338	*/
2339	if (!snp_page_reclaim(kvm, pfn: pfn + i) &&
2340	sev_populate_args->type == KVM_SEV_SNP_PAGE_TYPE_CPUID &&
2341	sev_populate_args->fw_error == SEV_RET_INVALID_PARAM) {
2342	void *vaddr = kmap_local_pfn(pfn: pfn + i);
2343
2344	if (copy_to_user(to: src + i * PAGE_SIZE, from: vaddr, PAGE_SIZE))
2345	pr_debug("Failed to write CPUID page back to userspace\n");
2346
2347	kunmap_local(vaddr);
2348	}
2349
2350	/* pfn + i is hypervisor-owned now, so skip below cleanup for it. */
2351	n_private--;
2352
2353	err:
2354	pr_debug("%s: exiting with error ret %d (fw_error %d), restoring %d gmem PFNs to shared.\n",
2355	__func__, ret, sev_populate_args->fw_error, n_private);
2356	for (i = 0; i < n_private; i++)
2357	kvm_rmp_make_shared(kvm, pfn: pfn + i, level: PG_LEVEL_4K);
2358
2359	return ret;
2360	}
2361
2362	static int snp_launch_update(struct kvm *kvm, struct kvm_sev_cmd *argp)
2363	{
2364	struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
2365	struct sev_gmem_populate_args sev_populate_args = {0};
2366	struct kvm_sev_snp_launch_update params;
2367	struct kvm_memory_slot *memslot;
2368	long npages, count;
2369	void __user *src;
2370	int ret = 0;
2371
2372	if (!sev_snp_guest(kvm) || !sev->snp_context)
2373	return -EINVAL;
2374
2375	if (copy_from_user(to: &params, u64_to_user_ptr(argp->data), n: sizeof(params)))
2376	return -EFAULT;
2377
2378	pr_debug("%s: GFN start 0x%llx length 0x%llx type %d flags %d\n", __func__,
2379	params.gfn_start, params.len, params.type, params.flags);
2380
2381	if (!params.len || !PAGE_ALIGNED(params.len) || params.flags ||
2382	(params.type != KVM_SEV_SNP_PAGE_TYPE_NORMAL &&
2383	params.type != KVM_SEV_SNP_PAGE_TYPE_ZERO &&
2384	params.type != KVM_SEV_SNP_PAGE_TYPE_UNMEASURED &&
2385	params.type != KVM_SEV_SNP_PAGE_TYPE_SECRETS &&
2386	params.type != KVM_SEV_SNP_PAGE_TYPE_CPUID))
2387	return -EINVAL;
2388
2389	npages = params.len / PAGE_SIZE;
2390
2391	/*
2392	* For each GFN that's being prepared as part of the initial guest
2393	* state, the following pre-conditions are verified:
2394	*
2395	* 1) The backing memslot is a valid private memslot.
2396	* 2) The GFN has been set to private via KVM_SET_MEMORY_ATTRIBUTES
2397	* beforehand.
2398	* 3) The PFN of the guest_memfd has not already been set to private
2399	* in the RMP table.
2400	*
2401	* The KVM MMU relies on kvm->mmu_invalidate_seq to retry nested page
2402	* faults if there's a race between a fault and an attribute update via
2403	* KVM_SET_MEMORY_ATTRIBUTES, and a similar approach could be utilized
2404	* here. However, kvm->slots_lock guards against both this as well as
2405	* concurrent memslot updates occurring while these checks are being
2406	* performed, so use that here to make it easier to reason about the
2407	* initial expected state and better guard against unexpected
2408	* situations.
2409	*/
2410	mutex_lock(&kvm->slots_lock);
2411
2412	memslot = gfn_to_memslot(kvm, gfn: params.gfn_start);
2413	if (!kvm_slot_has_gmem(slot: memslot)) {
2414	ret = -EINVAL;
2415	goto out;
2416	}
2417
2418	sev_populate_args.sev_fd = argp->sev_fd;
2419	sev_populate_args.type = params.type;
2420	src = params.type == KVM_SEV_SNP_PAGE_TYPE_ZERO ? NULL : u64_to_user_ptr(params.uaddr);
2421
2422	count = kvm_gmem_populate(kvm, gfn: params.gfn_start, src, npages,
2423	post_populate: sev_gmem_post_populate, opaque: &sev_populate_args);
2424	if (count < 0) {
2425	argp->error = sev_populate_args.fw_error;
2426	pr_debug("%s: kvm_gmem_populate failed, ret %ld (fw_error %d)\n",
2427	__func__, count, argp->error);
2428	ret = -EIO;
2429	} else {
2430	params.gfn_start += count;
2431	params.len -= count * PAGE_SIZE;
2432	if (params.type != KVM_SEV_SNP_PAGE_TYPE_ZERO)
2433	params.uaddr += count * PAGE_SIZE;
2434
2435	ret = 0;
2436	if (copy_to_user(u64_to_user_ptr(argp->data), from: &params, n: sizeof(params)))
2437	ret = -EFAULT;
2438	}
2439
2440	out:
2441	mutex_unlock(lock: &kvm->slots_lock);
2442
2443	return ret;
2444	}
2445
2446	static int snp_launch_update_vmsa(struct kvm *kvm, struct kvm_sev_cmd *argp)
2447	{
2448	struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
2449	struct sev_data_snp_launch_update data = {};
2450	struct kvm_vcpu *vcpu;
2451	unsigned long i;
2452	int ret;
2453
2454	data.gctx_paddr = __psp_pa(sev->snp_context);
2455	data.page_type = SNP_PAGE_TYPE_VMSA;
2456
2457	kvm_for_each_vcpu(i, vcpu, kvm) {
2458	struct vcpu_svm *svm = to_svm(vcpu);
2459	u64 pfn = __pa(svm->sev_es.vmsa) >> PAGE_SHIFT;
2460
2461	ret = sev_es_sync_vmsa(svm);
2462	if (ret)
2463	return ret;
2464
2465	/* Transition the VMSA page to a firmware state. */
2466	ret = rmp_make_private(pfn, INITIAL_VMSA_GPA, level: PG_LEVEL_4K, asid: sev->asid, immutable: true);
2467	if (ret)
2468	return ret;
2469
2470	/* Issue the SNP command to encrypt the VMSA */
2471	data.address = __sme_pa(svm->sev_es.vmsa);
2472	ret = __sev_issue_cmd(fd: argp->sev_fd, id: SEV_CMD_SNP_LAUNCH_UPDATE,
2473	data: &data, error: &argp->error);
2474	if (ret) {
2475	snp_page_reclaim(kvm, pfn);
2476
2477	return ret;
2478	}
2479
2480	svm->vcpu.arch.guest_state_protected = true;
2481	/*
2482	* SEV-ES (and thus SNP) guest mandates LBR Virtualization to
2483	* be _always_ ON. Enable it only after setting
2484	* guest_state_protected because KVM_SET_MSRS allows dynamic
2485	* toggling of LBRV (for performance reason) on write access to
2486	* MSR_IA32_DEBUGCTLMSR when guest_state_protected is not set.
2487	*/
2488	svm_enable_lbrv(vcpu);
2489	}
2490
2491	return 0;
2492	}
2493
2494	static int snp_launch_finish(struct kvm *kvm, struct kvm_sev_cmd *argp)
2495	{
2496	struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
2497	struct kvm_sev_snp_launch_finish params;
2498	struct sev_data_snp_launch_finish *data;
2499	void *id_block = NULL, *id_auth = NULL;
2500	int ret;
2501
2502	if (!sev_snp_guest(kvm))
2503	return -ENOTTY;
2504
2505	if (!sev->snp_context)
2506	return -EINVAL;
2507
2508	if (copy_from_user(to: &params, u64_to_user_ptr(argp->data), n: sizeof(params)))
2509	return -EFAULT;
2510
2511	if (params.flags)
2512	return -EINVAL;
2513
2514	/* Measure all vCPUs using LAUNCH_UPDATE before finalizing the launch flow. */
2515	ret = snp_launch_update_vmsa(kvm, argp);
2516	if (ret)
2517	return ret;
2518
2519	data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
2520	if (!data)
2521	return -ENOMEM;
2522
2523	if (params.id_block_en) {
2524	id_block = psp_copy_user_blob(uaddr: params.id_block_uaddr, KVM_SEV_SNP_ID_BLOCK_SIZE);
2525	if (IS_ERR(ptr: id_block)) {
2526	ret = PTR_ERR(ptr: id_block);
2527	goto e_free;
2528	}
2529
2530	data->id_block_en = 1;
2531	data->id_block_paddr = __sme_pa(id_block);
2532
2533	id_auth = psp_copy_user_blob(uaddr: params.id_auth_uaddr, KVM_SEV_SNP_ID_AUTH_SIZE);
2534	if (IS_ERR(ptr: id_auth)) {
2535	ret = PTR_ERR(ptr: id_auth);
2536	goto e_free_id_block;
2537	}
2538
2539	data->id_auth_paddr = __sme_pa(id_auth);
2540
2541	if (params.auth_key_en)
2542	data->auth_key_en = 1;
2543	}
2544
2545	data->vcek_disabled = params.vcek_disabled;
2546
2547	memcpy(data->host_data, params.host_data, KVM_SEV_SNP_FINISH_DATA_SIZE);
2548	data->gctx_paddr = __psp_pa(sev->snp_context);
2549	ret = sev_issue_cmd(kvm, id: SEV_CMD_SNP_LAUNCH_FINISH, data, error: &argp->error);
2550
2551	/*
2552	* Now that there will be no more SNP_LAUNCH_UPDATE ioctls, private pages
2553	* can be given to the guest simply by marking the RMP entry as private.
2554	* This can happen on first access and also with KVM_PRE_FAULT_MEMORY.
2555	*/
2556	if (!ret)
2557	kvm->arch.pre_fault_allowed = true;
2558
2559	kfree(objp: id_auth);
2560
2561	e_free_id_block:
2562	kfree(objp: id_block);
2563
2564	e_free:
2565	kfree(objp: data);
2566
2567	return ret;
2568	}
2569
2570	int sev_mem_enc_ioctl(struct kvm *kvm, void __user *argp)
2571	{
2572	struct kvm_sev_cmd sev_cmd;
2573	int r;
2574
2575	if (!sev_enabled)
2576	return -ENOTTY;
2577
2578	if (!argp)
2579	return 0;
2580
2581	if (copy_from_user(to: &sev_cmd, from: argp, n: sizeof(struct kvm_sev_cmd)))
2582	return -EFAULT;
2583
2584	mutex_lock(&kvm->lock);
2585
2586	/* Only the enc_context_owner handles some memory enc operations. */
2587	if (is_mirroring_enc_context(kvm) &&
2588	!is_cmd_allowed_from_mirror(cmd_id: sev_cmd.id)) {
2589	r = -EINVAL;
2590	goto out;
2591	}
2592
2593	/*
2594	* Once KVM_SEV_INIT2 initializes a KVM instance as an SNP guest, only
2595	* allow the use of SNP-specific commands.
2596	*/
2597	if (sev_snp_guest(kvm) && sev_cmd.id < KVM_SEV_SNP_LAUNCH_START) {
2598	r = -EPERM;
2599	goto out;
2600	}
2601
2602	switch (sev_cmd.id) {
2603	case KVM_SEV_ES_INIT:
2604	if (!sev_es_enabled) {
2605	r = -ENOTTY;
2606	goto out;
2607	}
2608	fallthrough;
2609	case KVM_SEV_INIT:
2610	r = sev_guest_init(kvm, argp: &sev_cmd);
2611	break;
2612	case KVM_SEV_INIT2:
2613	r = sev_guest_init2(kvm, argp: &sev_cmd);
2614	break;
2615	case KVM_SEV_LAUNCH_START:
2616	r = sev_launch_start(kvm, argp: &sev_cmd);
2617	break;
2618	case KVM_SEV_LAUNCH_UPDATE_DATA:
2619	r = sev_launch_update_data(kvm, argp: &sev_cmd);
2620	break;
2621	case KVM_SEV_LAUNCH_UPDATE_VMSA:
2622	r = sev_launch_update_vmsa(kvm, argp: &sev_cmd);
2623	break;
2624	case KVM_SEV_LAUNCH_MEASURE:
2625	r = sev_launch_measure(kvm, argp: &sev_cmd);
2626	break;
2627	case KVM_SEV_LAUNCH_FINISH:
2628	r = sev_launch_finish(kvm, argp: &sev_cmd);
2629	break;
2630	case KVM_SEV_GUEST_STATUS:
2631	r = sev_guest_status(kvm, argp: &sev_cmd);
2632	break;
2633	case KVM_SEV_DBG_DECRYPT:
2634	r = sev_dbg_crypt(kvm, argp: &sev_cmd, dec: true);
2635	break;
2636	case KVM_SEV_DBG_ENCRYPT:
2637	r = sev_dbg_crypt(kvm, argp: &sev_cmd, dec: false);
2638	break;
2639	case KVM_SEV_LAUNCH_SECRET:
2640	r = sev_launch_secret(kvm, argp: &sev_cmd);
2641	break;
2642	case KVM_SEV_GET_ATTESTATION_REPORT:
2643	r = sev_get_attestation_report(kvm, argp: &sev_cmd);
2644	break;
2645	case KVM_SEV_SEND_START:
2646	r = sev_send_start(kvm, argp: &sev_cmd);
2647	break;
2648	case KVM_SEV_SEND_UPDATE_DATA:
2649	r = sev_send_update_data(kvm, argp: &sev_cmd);
2650	break;
2651	case KVM_SEV_SEND_FINISH:
2652	r = sev_send_finish(kvm, argp: &sev_cmd);
2653	break;
2654	case KVM_SEV_SEND_CANCEL:
2655	r = sev_send_cancel(kvm, argp: &sev_cmd);
2656	break;
2657	case KVM_SEV_RECEIVE_START:
2658	r = sev_receive_start(kvm, argp: &sev_cmd);
2659	break;
2660	case KVM_SEV_RECEIVE_UPDATE_DATA:
2661	r = sev_receive_update_data(kvm, argp: &sev_cmd);
2662	break;
2663	case KVM_SEV_RECEIVE_FINISH:
2664	r = sev_receive_finish(kvm, argp: &sev_cmd);
2665	break;
2666	case KVM_SEV_SNP_LAUNCH_START:
2667	r = snp_launch_start(kvm, argp: &sev_cmd);
2668	break;
2669	case KVM_SEV_SNP_LAUNCH_UPDATE:
2670	r = snp_launch_update(kvm, argp: &sev_cmd);
2671	break;
2672	case KVM_SEV_SNP_LAUNCH_FINISH:
2673	r = snp_launch_finish(kvm, argp: &sev_cmd);
2674	break;
2675	default:
2676	r = -EINVAL;
2677	goto out;
2678	}
2679
2680	if (copy_to_user(to: argp, from: &sev_cmd, n: sizeof(struct kvm_sev_cmd)))
2681	r = -EFAULT;
2682
2683	out:
2684	mutex_unlock(lock: &kvm->lock);
2685	return r;
2686	}
2687
2688	int sev_mem_enc_register_region(struct kvm *kvm,
2689	struct kvm_enc_region *range)
2690	{
2691	struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
2692	struct enc_region *region;
2693	int ret = 0;
2694
2695	if (!sev_guest(kvm))
2696	return -ENOTTY;
2697
2698	/* If kvm is mirroring encryption context it isn't responsible for it */
2699	if (is_mirroring_enc_context(kvm))
2700	return -EINVAL;
2701
2702	if (range->addr > ULONG_MAX || range->size > ULONG_MAX)
2703	return -EINVAL;
2704
2705	region = kzalloc(sizeof(*region), GFP_KERNEL_ACCOUNT);
2706	if (!region)
2707	return -ENOMEM;
2708
2709	mutex_lock(&kvm->lock);
2710	region->pages = sev_pin_memory(kvm, uaddr: range->addr, ulen: range->size, n: &region->npages,
2711	flags: FOLL_WRITE | FOLL_LONGTERM);
2712	if (IS_ERR(ptr: region->pages)) {
2713	ret = PTR_ERR(ptr: region->pages);
2714	mutex_unlock(lock: &kvm->lock);
2715	goto e_free;
2716	}
2717
2718	/*
2719	* The guest may change the memory encryption attribute from C=0 -> C=1
2720	* or vice versa for this memory range. Lets make sure caches are
2721	* flushed to ensure that guest data gets written into memory with
2722	* correct C-bit. Note, this must be done before dropping kvm->lock,
2723	* as region and its array of pages can be freed by a different task
2724	* once kvm->lock is released.
2725	*/
2726	sev_clflush_pages(pages: region->pages, npages: region->npages);
2727
2728	region->uaddr = range->addr;
2729	region->size = range->size;
2730
2731	list_add_tail(new: &region->list, head: &sev->regions_list);
2732	mutex_unlock(lock: &kvm->lock);
2733
2734	return ret;
2735
2736	e_free:
2737	kfree(objp: region);
2738	return ret;
2739	}
2740
2741	static struct enc_region *
2742	find_enc_region(struct kvm *kvm, struct kvm_enc_region *range)
2743	{
2744	struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
2745	struct list_head *head = &sev->regions_list;
2746	struct enc_region *i;
2747
2748	list_for_each_entry(i, head, list) {
2749	if (i->uaddr == range->addr &&
2750	i->size == range->size)
2751	return i;
2752	}
2753
2754	return NULL;
2755	}
2756
2757	static void __unregister_enc_region_locked(struct kvm *kvm,
2758	struct enc_region *region)
2759	{
2760	sev_unpin_memory(kvm, pages: region->pages, npages: region->npages);
2761	list_del(entry: &region->list);
2762	kfree(objp: region);
2763	}
2764
2765	int sev_mem_enc_unregister_region(struct kvm *kvm,
2766	struct kvm_enc_region *range)
2767	{
2768	struct enc_region *region;
2769	int ret;
2770
2771	/* If kvm is mirroring encryption context it isn't responsible for it */
2772	if (is_mirroring_enc_context(kvm))
2773	return -EINVAL;
2774
2775	mutex_lock(&kvm->lock);
2776
2777	if (!sev_guest(kvm)) {
2778	ret = -ENOTTY;
2779	goto failed;
2780	}
2781
2782	region = find_enc_region(kvm, range);
2783	if (!region) {
2784	ret = -EINVAL;
2785	goto failed;
2786	}
2787
2788	sev_writeback_caches(kvm);
2789
2790	__unregister_enc_region_locked(kvm, region);
2791
2792	mutex_unlock(lock: &kvm->lock);
2793	return 0;
2794
2795	failed:
2796	mutex_unlock(lock: &kvm->lock);
2797	return ret;
2798	}
2799
2800	int sev_vm_copy_enc_context_from(struct kvm *kvm, unsigned int source_fd)
2801	{
2802	CLASS(fd, f)(fd: source_fd);
2803	struct kvm *source_kvm;
2804	struct kvm_sev_info *source_sev, *mirror_sev;
2805	int ret;
2806
2807	if (fd_empty(f))
2808	return -EBADF;
2809
2810	if (!file_is_kvm(fd_file(f)))
2811	return -EBADF;
2812
2813	source_kvm = fd_file(f)->private_data;
2814	ret = sev_lock_two_vms(dst_kvm: kvm, src_kvm: source_kvm);
2815	if (ret)
2816	return ret;
2817
2818	/*
2819	* Mirrors of mirrors should work, but let's not get silly. Also
2820	* disallow out-of-band SEV/SEV-ES init if the target is already an
2821	* SEV guest, or if vCPUs have been created. KVM relies on vCPUs being
2822	* created after SEV/SEV-ES initialization, e.g. to init intercepts.
2823	*/
2824	if (sev_guest(kvm) || !sev_guest(kvm: source_kvm) ||
2825	is_mirroring_enc_context(kvm: source_kvm) || kvm->created_vcpus) {
2826	ret = -EINVAL;
2827	goto e_unlock;
2828	}
2829
2830	mirror_sev = to_kvm_sev_info(kvm);
2831	if (!zalloc_cpumask_var(mask: &mirror_sev->have_run_cpus, GFP_KERNEL_ACCOUNT)) {
2832	ret = -ENOMEM;
2833	goto e_unlock;
2834	}
2835
2836	/*
2837	* The mirror kvm holds an enc_context_owner ref so its asid can't
2838	* disappear until we're done with it
2839	*/
2840	source_sev = to_kvm_sev_info(kvm: source_kvm);
2841	kvm_get_kvm(kvm: source_kvm);
2842	list_add_tail(new: &mirror_sev->mirror_entry, head: &source_sev->mirror_vms);
2843
2844	/* Set enc_context_owner and copy its encryption context over */
2845	mirror_sev->enc_context_owner = source_kvm;
2846	mirror_sev->active = true;
2847	mirror_sev->asid = source_sev->asid;
2848	mirror_sev->fd = source_sev->fd;
2849	mirror_sev->es_active = source_sev->es_active;
2850	mirror_sev->need_init = false;
2851	mirror_sev->handle = source_sev->handle;
2852	INIT_LIST_HEAD(list: &mirror_sev->regions_list);
2853	INIT_LIST_HEAD(list: &mirror_sev->mirror_vms);
2854	ret = 0;
2855
2856	/*
2857	* Do not copy ap_jump_table. Since the mirror does not share the same
2858	* KVM contexts as the original, and they may have different
2859	* memory-views.
2860	*/
2861
2862	e_unlock:
2863	sev_unlock_two_vms(dst_kvm: kvm, src_kvm: source_kvm);
2864	return ret;
2865	}
2866
2867	static int snp_decommission_context(struct kvm *kvm)
2868	{
2869	struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
2870	struct sev_data_snp_addr data = {};
2871	int ret;
2872
2873	/* If context is not created then do nothing */
2874	if (!sev->snp_context)
2875	return 0;
2876
2877	/* Do the decommision, which will unbind the ASID from the SNP context */
2878	data.address = __sme_pa(sev->snp_context);
2879	down_write(sem: &sev_deactivate_lock);
2880	ret = sev_do_cmd(cmd: SEV_CMD_SNP_DECOMMISSION, data: &data, NULL);
2881	up_write(sem: &sev_deactivate_lock);
2882
2883	if (WARN_ONCE(ret, "Failed to release guest context, ret %d", ret))
2884	return ret;
2885
2886	snp_free_firmware_page(addr: sev->snp_context);
2887	sev->snp_context = NULL;
2888
2889	return 0;
2890	}
2891
2892	void sev_vm_destroy(struct kvm *kvm)
2893	{
2894	struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
2895	struct list_head *head = &sev->regions_list;
2896	struct list_head *pos, *q;
2897
2898	if (!sev_guest(kvm))
2899	return;
2900
2901	WARN_ON(!list_empty(&sev->mirror_vms));
2902
2903	free_cpumask_var(mask: sev->have_run_cpus);
2904
2905	/*
2906	* If this is a mirror VM, remove it from the owner's list of a mirrors
2907	* and skip ASID cleanup (the ASID is tied to the lifetime of the owner).
2908	* Note, mirror VMs don't support registering encrypted regions.
2909	*/
2910	if (is_mirroring_enc_context(kvm)) {
2911	struct kvm *owner_kvm = sev->enc_context_owner;
2912
2913	mutex_lock(&owner_kvm->lock);
2914	list_del(entry: &sev->mirror_entry);
2915	mutex_unlock(lock: &owner_kvm->lock);
2916	kvm_put_kvm(kvm: owner_kvm);
2917	return;
2918	}
2919
2920
2921	/*
2922	* if userspace was terminated before unregistering the memory regions
2923	* then lets unpin all the registered memory.
2924	*/
2925	if (!list_empty(head)) {
2926	list_for_each_safe(pos, q, head) {
2927	__unregister_enc_region_locked(kvm,
2928	list_entry(pos, struct enc_region, list));
2929	cond_resched();
2930	}
2931	}
2932
2933	if (sev_snp_guest(kvm)) {
2934	snp_guest_req_cleanup(kvm);
2935
2936	/*
2937	* Decomission handles unbinding of the ASID. If it fails for
2938	* some unexpected reason, just leak the ASID.
2939	*/
2940	if (snp_decommission_context(kvm))
2941	return;
2942	} else {
2943	sev_unbind_asid(kvm, handle: sev->handle);
2944	}
2945
2946	sev_asid_free(sev);
2947	}
2948
2949	void __init sev_set_cpu_caps(void)
2950	{
2951	if (sev_enabled) {
2952	kvm_cpu_cap_set(X86_FEATURE_SEV);
2953	kvm_caps.supported_vm_types |= BIT(KVM_X86_SEV_VM);
2954	}
2955	if (sev_es_enabled) {
2956	kvm_cpu_cap_set(X86_FEATURE_SEV_ES);
2957	kvm_caps.supported_vm_types |= BIT(KVM_X86_SEV_ES_VM);
2958	}
2959	if (sev_snp_enabled) {
2960	kvm_cpu_cap_set(X86_FEATURE_SEV_SNP);
2961	kvm_caps.supported_vm_types |= BIT(KVM_X86_SNP_VM);
2962	}
2963	}
2964
2965	static bool is_sev_snp_initialized(void)
2966	{
2967	struct sev_user_data_snp_status *status;
2968	struct sev_data_snp_addr buf;
2969	bool initialized = false;
2970	int ret, error = 0;
2971
2972	status = snp_alloc_firmware_page(GFP_KERNEL | __GFP_ZERO);
2973	if (!status)
2974	return false;
2975
2976	buf.address = __psp_pa(status);
2977	ret = sev_do_cmd(cmd: SEV_CMD_SNP_PLATFORM_STATUS, data: &buf, psp_ret: &error);
2978	if (ret) {
2979	pr_err("SEV: SNP_PLATFORM_STATUS failed ret=%d, fw_error=%d (%#x)\n",
2980	ret, error, error);
2981	goto out;
2982	}
2983
2984	initialized = !!status->state;
2985
2986	out:
2987	snp_free_firmware_page(addr: status);
2988
2989	return initialized;
2990	}
2991
2992	void __init sev_hardware_setup(void)
2993	{
2994	unsigned int eax, ebx, ecx, edx, sev_asid_count, sev_es_asid_count;
2995	struct sev_platform_init_args init_args = {0};
2996	bool sev_snp_supported = false;
2997	bool sev_es_supported = false;
2998	bool sev_supported = false;
2999
3000	if (!sev_enabled || !npt_enabled || !nrips)
3001	goto out;
3002
3003	/*
3004	* SEV must obviously be supported in hardware. Sanity check that the
3005	* CPU supports decode assists, which is mandatory for SEV guests to
3006	* support instruction emulation. Ditto for flushing by ASID, as SEV
3007	* guests are bound to a single ASID, i.e. KVM can't rotate to a new
3008	* ASID to effect a TLB flush.
3009	*/
3010	if (!boot_cpu_has(X86_FEATURE_SEV) ||
3011	WARN_ON_ONCE(!boot_cpu_has(X86_FEATURE_DECODEASSISTS)) ||
3012	WARN_ON_ONCE(!boot_cpu_has(X86_FEATURE_FLUSHBYASID)))
3013	goto out;
3014
3015	/*
3016	* The kernel's initcall infrastructure lacks the ability to express
3017	* dependencies between initcalls, whereas the modules infrastructure
3018	* automatically handles dependencies via symbol loading. Ensure the
3019	* PSP SEV driver is initialized before proceeding if KVM is built-in,
3020	* as the dependency isn't handled by the initcall infrastructure.
3021	*/
3022	if (IS_BUILTIN(CONFIG_KVM_AMD) && sev_module_init())
3023	goto out;
3024
3025	/* Retrieve SEV CPUID information */
3026	cpuid(op: 0x8000001f, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
3027
3028	/* Set encryption bit location for SEV-ES guests */
3029	sev_enc_bit = ebx & 0x3f;
3030
3031	/* Maximum number of encrypted guests supported simultaneously */
3032	max_sev_asid = ecx;
3033	if (!max_sev_asid)
3034	goto out;
3035
3036	/* Minimum ASID value that should be used for SEV guest */
3037	min_sev_asid = edx;
3038	sev_me_mask = 1UL << (ebx & 0x3f);
3039
3040	/*
3041	* Initialize SEV ASID bitmaps. Allocate space for ASID 0 in the bitmap,
3042	* even though it's never used, so that the bitmap is indexed by the
3043	* actual ASID.
3044	*/
3045	nr_asids = max_sev_asid + 1;
3046	sev_asid_bitmap = bitmap_zalloc(nbits: nr_asids, GFP_KERNEL);
3047	if (!sev_asid_bitmap)
3048	goto out;
3049
3050	sev_reclaim_asid_bitmap = bitmap_zalloc(nbits: nr_asids, GFP_KERNEL);
3051	if (!sev_reclaim_asid_bitmap) {
3052	bitmap_free(bitmap: sev_asid_bitmap);
3053	sev_asid_bitmap = NULL;
3054	goto out;
3055	}
3056
3057	if (min_sev_asid <= max_sev_asid) {
3058	sev_asid_count = max_sev_asid - min_sev_asid + 1;
3059	WARN_ON_ONCE(misc_cg_set_capacity(MISC_CG_RES_SEV, sev_asid_count));
3060	}
3061	sev_supported = true;
3062
3063	/* SEV-ES support requested? */
3064	if (!sev_es_enabled)
3065	goto out;
3066
3067	/*
3068	* SEV-ES requires MMIO caching as KVM doesn't have access to the guest
3069	* instruction stream, i.e. can't emulate in response to a #NPF and
3070	* instead relies on #NPF(RSVD) being reflected into the guest as #VC
3071	* (the guest can then do a #VMGEXIT to request MMIO emulation).
3072	*/
3073	if (!enable_mmio_caching)
3074	goto out;
3075
3076	/* Does the CPU support SEV-ES? */
3077	if (!boot_cpu_has(X86_FEATURE_SEV_ES))
3078	goto out;
3079
3080	if (!lbrv) {
3081	WARN_ONCE(!boot_cpu_has(X86_FEATURE_LBRV),
3082	"LBRV must be present for SEV-ES support");
3083	goto out;
3084	}
3085
3086	/* Has the system been allocated ASIDs for SEV-ES? */
3087	if (min_sev_asid == 1)
3088	goto out;
3089
3090	min_sev_es_asid = min_snp_asid = 1;
3091	max_sev_es_asid = max_snp_asid = min_sev_asid - 1;
3092
3093	sev_es_asid_count = min_sev_asid - 1;
3094	WARN_ON_ONCE(misc_cg_set_capacity(MISC_CG_RES_SEV_ES, sev_es_asid_count));
3095	sev_es_supported = true;
3096	sev_snp_supported = sev_snp_enabled && cc_platform_has(attr: CC_ATTR_HOST_SEV_SNP);
3097
3098	out:
3099	if (sev_enabled) {
3100	init_args.probe = true;
3101
3102	if (sev_is_snp_ciphertext_hiding_supported())
3103	init_args.max_snp_asid = min(nr_ciphertext_hiding_asids,
3104	min_sev_asid - 1);
3105
3106	if (sev_platform_init(args: &init_args))
3107	sev_supported = sev_es_supported = sev_snp_supported = false;
3108	else if (sev_snp_supported)
3109	sev_snp_supported = is_sev_snp_initialized();
3110
3111	if (sev_snp_supported) {
3112	snp_supported_policy_bits = sev_get_snp_policy_bits() &
3113	KVM_SNP_POLICY_MASK_VALID;
3114	nr_ciphertext_hiding_asids = init_args.max_snp_asid;
3115	}
3116
3117	/*
3118	* If ciphertext hiding is enabled, the joint SEV-ES/SEV-SNP
3119	* ASID range is partitioned into separate SEV-ES and SEV-SNP
3120	* ASID ranges, with the SEV-SNP range being [1..max_snp_asid]
3121	* and the SEV-ES range being (max_snp_asid..max_sev_es_asid].
3122	* Note, SEV-ES may effectively be disabled if all ASIDs from
3123	* the joint range are assigned to SEV-SNP.
3124	*/
3125	if (nr_ciphertext_hiding_asids) {
3126	max_snp_asid = nr_ciphertext_hiding_asids;
3127	min_sev_es_asid = max_snp_asid + 1;
3128	pr_info("SEV-SNP ciphertext hiding enabled\n");
3129	}
3130	}
3131
3132	if (boot_cpu_has(X86_FEATURE_SEV))
3133	pr_info("SEV %s (ASIDs %u - %u)\n",
3134	sev_supported ? min_sev_asid <= max_sev_asid ? "enabled" :
3135	"unusable" :
3136	"disabled",
3137	min_sev_asid, max_sev_asid);
3138	if (boot_cpu_has(X86_FEATURE_SEV_ES))
3139	pr_info("SEV-ES %s (ASIDs %u - %u)\n",
3140	sev_es_supported ? min_sev_es_asid <= max_sev_es_asid ? "enabled" :
3141	"unusable" :
3142	"disabled",
3143	min_sev_es_asid, max_sev_es_asid);
3144	if (boot_cpu_has(X86_FEATURE_SEV_SNP))
3145	pr_info("SEV-SNP %s (ASIDs %u - %u)\n",
3146	str_enabled_disabled(sev_snp_supported),
3147	min_snp_asid, max_snp_asid);
3148
3149	sev_enabled = sev_supported;
3150	sev_es_enabled = sev_es_supported;
3151	sev_snp_enabled = sev_snp_supported;
3152
3153	if (!sev_es_enabled || !cpu_feature_enabled(X86_FEATURE_DEBUG_SWAP) ||
3154	!cpu_feature_enabled(X86_FEATURE_NO_NESTED_DATA_BP))
3155	sev_es_debug_swap_enabled = false;
3156
3157	sev_supported_vmsa_features = 0;
3158	if (sev_es_debug_swap_enabled)
3159	sev_supported_vmsa_features |= SVM_SEV_FEAT_DEBUG_SWAP;
3160
3161	if (sev_snp_enabled && tsc_khz && cpu_feature_enabled(X86_FEATURE_SNP_SECURE_TSC))
3162	sev_supported_vmsa_features |= SVM_SEV_FEAT_SECURE_TSC;
3163	}
3164
3165	void sev_hardware_unsetup(void)
3166	{
3167	if (!sev_enabled)
3168	return;
3169
3170	/* No need to take sev_bitmap_lock, all VMs have been destroyed. */
3171	sev_flush_asids(min_asid: 1, max_asid: max_sev_asid);
3172
3173	bitmap_free(bitmap: sev_asid_bitmap);
3174	bitmap_free(bitmap: sev_reclaim_asid_bitmap);
3175
3176	misc_cg_set_capacity(type: MISC_CG_RES_SEV, capacity: 0);
3177	misc_cg_set_capacity(type: MISC_CG_RES_SEV_ES, capacity: 0);
3178
3179	sev_platform_shutdown();
3180	}
3181
3182	int sev_cpu_init(struct svm_cpu_data *sd)
3183	{
3184	if (!sev_enabled)
3185	return 0;
3186
3187	sd->sev_vmcbs = kcalloc(nr_asids, sizeof(void *), GFP_KERNEL);
3188	if (!sd->sev_vmcbs)
3189	return -ENOMEM;
3190
3191	return 0;
3192	}
3193
3194	/*
3195	* Pages used by hardware to hold guest encrypted state must be flushed before
3196	* returning them to the system.
3197	*/
3198	static void sev_flush_encrypted_page(struct kvm_vcpu *vcpu, void *va)
3199	{
3200	unsigned int asid = sev_get_asid(kvm: vcpu->kvm);
3201
3202	/*
3203	* Note! The address must be a kernel address, as regular page walk
3204	* checks are performed by VM_PAGE_FLUSH, i.e. operating on a user
3205	* address is non-deterministic and unsafe. This function deliberately
3206	* takes a pointer to deter passing in a user address.
3207	*/
3208	unsigned long addr = (unsigned long)va;
3209
3210	/*
3211	* If CPU enforced cache coherency for encrypted mappings of the
3212	* same physical page is supported, use CLFLUSHOPT instead. NOTE: cache
3213	* flush is still needed in order to work properly with DMA devices.
3214	*/
3215	if (boot_cpu_has(X86_FEATURE_SME_COHERENT)) {
3216	clflush_cache_range(addr: va, PAGE_SIZE);
3217	return;
3218	}
3219
3220	/*
3221	* VM Page Flush takes a host virtual address and a guest ASID. Fall
3222	* back to full writeback of caches if this faults so as not to make
3223	* any problems worse by leaving stale encrypted data in the cache.
3224	*/
3225	if (WARN_ON_ONCE(wrmsrq_safe(MSR_AMD64_VM_PAGE_FLUSH, addr | asid)))
3226	goto do_sev_writeback_caches;
3227
3228	return;
3229
3230	do_sev_writeback_caches:
3231	sev_writeback_caches(kvm: vcpu->kvm);
3232	}
3233
3234	void sev_guest_memory_reclaimed(struct kvm *kvm)
3235	{
3236	/*
3237	* With SNP+gmem, private/encrypted memory is unreachable via the
3238	* hva-based mmu notifiers, i.e. these events are explicitly scoped to
3239	* shared pages, where there's no need to flush caches.
3240	*/
3241	if (!sev_guest(kvm) || sev_snp_guest(kvm))
3242	return;
3243
3244	sev_writeback_caches(kvm);
3245	}
3246
3247	void sev_free_vcpu(struct kvm_vcpu *vcpu)
3248	{
3249	struct vcpu_svm *svm;
3250
3251	if (!sev_es_guest(kvm: vcpu->kvm))
3252	return;
3253
3254	svm = to_svm(vcpu);
3255
3256	/*
3257	* If it's an SNP guest, then the VMSA was marked in the RMP table as
3258	* a guest-owned page. Transition the page to hypervisor state before
3259	* releasing it back to the system.
3260	*/
3261	if (sev_snp_guest(kvm: vcpu->kvm)) {
3262	u64 pfn = __pa(svm->sev_es.vmsa) >> PAGE_SHIFT;
3263
3264	if (kvm_rmp_make_shared(kvm: vcpu->kvm, pfn, level: PG_LEVEL_4K))
3265	goto skip_vmsa_free;
3266	}
3267
3268	if (vcpu->arch.guest_state_protected)
3269	sev_flush_encrypted_page(vcpu, va: svm->sev_es.vmsa);
3270
3271	__free_page(virt_to_page(svm->sev_es.vmsa));
3272
3273	skip_vmsa_free:
3274	if (svm->sev_es.ghcb_sa_free)
3275	kvfree(addr: svm->sev_es.ghcb_sa);
3276	}
3277
3278	static u64 kvm_get_cached_sw_exit_code(struct vmcb_control_area *control)
3279	{
3280	return (((u64)control->exit_code_hi) << 32) | control->exit_code;
3281	}
3282
3283	static void dump_ghcb(struct vcpu_svm *svm)
3284	{
3285	struct vmcb_control_area *control = &svm->vmcb->control;
3286	unsigned int nbits;
3287
3288	/* Re-use the dump_invalid_vmcb module parameter */
3289	if (!dump_invalid_vmcb) {
3290	pr_warn_ratelimited("set kvm_amd.dump_invalid_vmcb=1 to dump internal KVM state.\n");
3291	return;
3292	}
3293
3294	nbits = sizeof(svm->sev_es.valid_bitmap) * 8;
3295
3296	/*
3297	* Print KVM's snapshot of the GHCB values that were (unsuccessfully)
3298	* used to handle the exit. If the guest has since modified the GHCB
3299	* itself, dumping the raw GHCB won't help debug why KVM was unable to
3300	* handle the VMGEXIT that KVM observed.
3301	*/
3302	pr_err("GHCB (GPA=%016llx) snapshot:\n", svm->vmcb->control.ghcb_gpa);
3303	pr_err("%-20s%016llx is_valid: %u\n", "sw_exit_code",
3304	kvm_get_cached_sw_exit_code(control), kvm_ghcb_sw_exit_code_is_valid(svm));
3305	pr_err("%-20s%016llx is_valid: %u\n", "sw_exit_info_1",
3306	control->exit_info_1, kvm_ghcb_sw_exit_info_1_is_valid(svm));
3307	pr_err("%-20s%016llx is_valid: %u\n", "sw_exit_info_2",
3308	control->exit_info_2, kvm_ghcb_sw_exit_info_2_is_valid(svm));
3309	pr_err("%-20s%016llx is_valid: %u\n", "sw_scratch",
3310	svm->sev_es.sw_scratch, kvm_ghcb_sw_scratch_is_valid(svm));
3311	pr_err("%-20s%*pb\n", "valid_bitmap", nbits, svm->sev_es.valid_bitmap);
3312	}
3313
3314	static void sev_es_sync_to_ghcb(struct vcpu_svm *svm)
3315	{
3316	struct kvm_vcpu *vcpu = &svm->vcpu;
3317	struct ghcb *ghcb = svm->sev_es.ghcb;
3318
3319	/*
3320	* The GHCB protocol so far allows for the following data
3321	* to be returned:
3322	* GPRs RAX, RBX, RCX, RDX
3323	*
3324	* Copy their values, even if they may not have been written during the
3325	* VM-Exit. It's the guest's responsibility to not consume random data.
3326	*/
3327	ghcb_set_rax(ghcb, value: vcpu->arch.regs[VCPU_REGS_RAX]);
3328	ghcb_set_rbx(ghcb, value: vcpu->arch.regs[VCPU_REGS_RBX]);
3329	ghcb_set_rcx(ghcb, value: vcpu->arch.regs[VCPU_REGS_RCX]);
3330	ghcb_set_rdx(ghcb, value: vcpu->arch.regs[VCPU_REGS_RDX]);
3331	}
3332
3333	static void sev_es_sync_from_ghcb(struct vcpu_svm *svm)
3334	{
3335	struct vmcb_control_area *control = &svm->vmcb->control;
3336	struct kvm_vcpu *vcpu = &svm->vcpu;
3337	struct ghcb *ghcb = svm->sev_es.ghcb;
3338	u64 exit_code;
3339
3340	/*
3341	* The GHCB protocol so far allows for the following data
3342	* to be supplied:
3343	* GPRs RAX, RBX, RCX, RDX
3344	* XCR0
3345	* CPL
3346	*
3347	* VMMCALL allows the guest to provide extra registers. KVM also
3348	* expects RSI for hypercalls, so include that, too.
3349	*
3350	* Copy their values to the appropriate location if supplied.
3351	*/
3352	memset(vcpu->arch.regs, 0, sizeof(vcpu->arch.regs));
3353
3354	BUILD_BUG_ON(sizeof(svm->sev_es.valid_bitmap) != sizeof(ghcb->save.valid_bitmap));
3355	memcpy(&svm->sev_es.valid_bitmap, &ghcb->save.valid_bitmap, sizeof(ghcb->save.valid_bitmap));
3356
3357	vcpu->arch.regs[VCPU_REGS_RAX] = kvm_ghcb_get_rax_if_valid(svm);
3358	vcpu->arch.regs[VCPU_REGS_RBX] = kvm_ghcb_get_rbx_if_valid(svm);
3359	vcpu->arch.regs[VCPU_REGS_RCX] = kvm_ghcb_get_rcx_if_valid(svm);
3360	vcpu->arch.regs[VCPU_REGS_RDX] = kvm_ghcb_get_rdx_if_valid(svm);
3361	vcpu->arch.regs[VCPU_REGS_RSI] = kvm_ghcb_get_rsi_if_valid(svm);
3362
3363	svm->vmcb->save.cpl = kvm_ghcb_get_cpl_if_valid(svm);
3364
3365	if (kvm_ghcb_xcr0_is_valid(svm))
3366	__kvm_set_xcr(vcpu, index: 0, xcr: kvm_ghcb_get_xcr0(svm));
3367
3368	if (kvm_ghcb_xss_is_valid(svm))
3369	__kvm_emulate_msr_write(vcpu, MSR_IA32_XSS, data: kvm_ghcb_get_xss(svm));
3370
3371	/* Copy the GHCB exit information into the VMCB fields */
3372	exit_code = kvm_ghcb_get_sw_exit_code(svm);
3373	control->exit_code = lower_32_bits(exit_code);
3374	control->exit_code_hi = upper_32_bits(exit_code);
3375	control->exit_info_1 = kvm_ghcb_get_sw_exit_info_1(svm);
3376	control->exit_info_2 = kvm_ghcb_get_sw_exit_info_2(svm);
3377	svm->sev_es.sw_scratch = kvm_ghcb_get_sw_scratch_if_valid(svm);
3378
3379	/* Clear the valid entries fields */
3380	memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap));
3381	}
3382
3383	static int sev_es_validate_vmgexit(struct vcpu_svm *svm)
3384	{
3385	struct vmcb_control_area *control = &svm->vmcb->control;
3386	struct kvm_vcpu *vcpu = &svm->vcpu;
3387	u64 exit_code;
3388	u64 reason;
3389
3390	/*
3391	* Retrieve the exit code now even though it may not be marked valid
3392	* as it could help with debugging.
3393	*/
3394	exit_code = kvm_get_cached_sw_exit_code(control);
3395
3396	/* Only GHCB Usage code 0 is supported */
3397	if (svm->sev_es.ghcb->ghcb_usage) {
3398	reason = GHCB_ERR_INVALID_USAGE;
3399	goto vmgexit_err;
3400	}
3401
3402	reason = GHCB_ERR_MISSING_INPUT;
3403
3404	if (!kvm_ghcb_sw_exit_code_is_valid(svm) ||
3405	!kvm_ghcb_sw_exit_info_1_is_valid(svm) ||
3406	!kvm_ghcb_sw_exit_info_2_is_valid(svm))
3407	goto vmgexit_err;
3408
3409	switch (exit_code) {
3410	case SVM_EXIT_READ_DR7:
3411	break;
3412	case SVM_EXIT_WRITE_DR7:
3413	if (!kvm_ghcb_rax_is_valid(svm))
3414	goto vmgexit_err;
3415	break;
3416	case SVM_EXIT_RDTSC:
3417	break;
3418	case SVM_EXIT_RDPMC:
3419	if (!kvm_ghcb_rcx_is_valid(svm))
3420	goto vmgexit_err;
3421	break;
3422	case SVM_EXIT_CPUID:
3423	if (!kvm_ghcb_rax_is_valid(svm) ||
3424	!kvm_ghcb_rcx_is_valid(svm))
3425	goto vmgexit_err;
3426	if (vcpu->arch.regs[VCPU_REGS_RAX] == 0xd)
3427	if (!kvm_ghcb_xcr0_is_valid(svm))
3428	goto vmgexit_err;
3429	break;
3430	case SVM_EXIT_INVD:
3431	break;
3432	case SVM_EXIT_IOIO:
3433	if (control->exit_info_1 & SVM_IOIO_STR_MASK) {
3434	if (!kvm_ghcb_sw_scratch_is_valid(svm))
3435	goto vmgexit_err;
3436	} else {
3437	if (!(control->exit_info_1 & SVM_IOIO_TYPE_MASK))
3438	if (!kvm_ghcb_rax_is_valid(svm))
3439	goto vmgexit_err;
3440	}
3441	break;
3442	case SVM_EXIT_MSR:
3443	if (!kvm_ghcb_rcx_is_valid(svm))
3444	goto vmgexit_err;
3445	if (control->exit_info_1) {
3446	if (!kvm_ghcb_rax_is_valid(svm) ||
3447	!kvm_ghcb_rdx_is_valid(svm))
3448	goto vmgexit_err;
3449	}
3450	break;
3451	case SVM_EXIT_VMMCALL:
3452	if (!kvm_ghcb_rax_is_valid(svm) ||
3453	!kvm_ghcb_cpl_is_valid(svm))
3454	goto vmgexit_err;
3455	break;
3456	case SVM_EXIT_RDTSCP:
3457	break;
3458	case SVM_EXIT_WBINVD:
3459	break;
3460	case SVM_EXIT_MONITOR:
3461	if (!kvm_ghcb_rax_is_valid(svm) ||
3462	!kvm_ghcb_rcx_is_valid(svm) ||
3463	!kvm_ghcb_rdx_is_valid(svm))
3464	goto vmgexit_err;
3465	break;
3466	case SVM_EXIT_MWAIT:
3467	if (!kvm_ghcb_rax_is_valid(svm) ||
3468	!kvm_ghcb_rcx_is_valid(svm))
3469	goto vmgexit_err;
3470	break;
3471	case SVM_VMGEXIT_MMIO_READ:
3472	case SVM_VMGEXIT_MMIO_WRITE:
3473	if (!kvm_ghcb_sw_scratch_is_valid(svm))
3474	goto vmgexit_err;
3475	break;
3476	case SVM_VMGEXIT_AP_CREATION:
3477	if (!sev_snp_guest(kvm: vcpu->kvm))
3478	goto vmgexit_err;
3479	if (lower_32_bits(control->exit_info_1) != SVM_VMGEXIT_AP_DESTROY)
3480	if (!kvm_ghcb_rax_is_valid(svm))
3481	goto vmgexit_err;
3482	break;
3483	case SVM_VMGEXIT_NMI_COMPLETE:
3484	case SVM_VMGEXIT_AP_HLT_LOOP:
3485	case SVM_VMGEXIT_AP_JUMP_TABLE:
3486	case SVM_VMGEXIT_UNSUPPORTED_EVENT:
3487	case SVM_VMGEXIT_HV_FEATURES:
3488	case SVM_VMGEXIT_TERM_REQUEST:
3489	break;
3490	case SVM_VMGEXIT_PSC:
3491	if (!sev_snp_guest(kvm: vcpu->kvm) || !kvm_ghcb_sw_scratch_is_valid(svm))
3492	goto vmgexit_err;
3493	break;
3494	case SVM_VMGEXIT_GUEST_REQUEST:
3495	case SVM_VMGEXIT_EXT_GUEST_REQUEST:
3496	if (!sev_snp_guest(kvm: vcpu->kvm) ||
3497	!PAGE_ALIGNED(control->exit_info_1) ||
3498	!PAGE_ALIGNED(control->exit_info_2) ||
3499	control->exit_info_1 == control->exit_info_2)
3500	goto vmgexit_err;
3501	break;
3502	default:
3503	reason = GHCB_ERR_INVALID_EVENT;
3504	goto vmgexit_err;
3505	}
3506
3507	return 0;
3508
3509	vmgexit_err:
3510	if (reason == GHCB_ERR_INVALID_USAGE) {
3511	vcpu_unimpl(vcpu, "vmgexit: ghcb usage %#x is not valid\n",
3512	svm->sev_es.ghcb->ghcb_usage);
3513	} else if (reason == GHCB_ERR_INVALID_EVENT) {
3514	vcpu_unimpl(vcpu, "vmgexit: exit code %#llx is not valid\n",
3515	exit_code);
3516	} else {
3517	vcpu_unimpl(vcpu, "vmgexit: exit code %#llx input is not valid\n",
3518	exit_code);
3519	dump_ghcb(svm);
3520	}
3521
3522	svm_vmgexit_bad_input(svm, suberror: reason);
3523
3524	/* Resume the guest to "return" the error code. */
3525	return 1;
3526	}
3527
3528	void sev_es_unmap_ghcb(struct vcpu_svm *svm)
3529	{
3530	/* Clear any indication that the vCPU is in a type of AP Reset Hold */
3531	svm->sev_es.ap_reset_hold_type = AP_RESET_HOLD_NONE;
3532
3533	if (!svm->sev_es.ghcb)
3534	return;
3535
3536	if (svm->sev_es.ghcb_sa_free) {
3537	/*
3538	* The scratch area lives outside the GHCB, so there is a
3539	* buffer that, depending on the operation performed, may
3540	* need to be synced, then freed.
3541	*/
3542	if (svm->sev_es.ghcb_sa_sync) {
3543	kvm_write_guest(kvm: svm->vcpu.kvm,
3544	gpa: svm->sev_es.sw_scratch,
3545	data: svm->sev_es.ghcb_sa,
3546	len: svm->sev_es.ghcb_sa_len);
3547	svm->sev_es.ghcb_sa_sync = false;
3548	}
3549
3550	kvfree(addr: svm->sev_es.ghcb_sa);
3551	svm->sev_es.ghcb_sa = NULL;
3552	svm->sev_es.ghcb_sa_free = false;
3553	}
3554
3555	trace_kvm_vmgexit_exit(svm->vcpu.vcpu_id, svm->sev_es.ghcb);
3556
3557	sev_es_sync_to_ghcb(svm);
3558
3559	kvm_vcpu_unmap(vcpu: &svm->vcpu, map: &svm->sev_es.ghcb_map);
3560	svm->sev_es.ghcb = NULL;
3561	}
3562
3563	int pre_sev_run(struct vcpu_svm *svm, int cpu)
3564	{
3565	struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, cpu);
3566	struct kvm *kvm = svm->vcpu.kvm;
3567	unsigned int asid = sev_get_asid(kvm);
3568
3569	/*
3570	* Reject KVM_RUN if userspace attempts to run the vCPU with an invalid
3571	* VMSA, e.g. if userspace forces the vCPU to be RUNNABLE after an SNP
3572	* AP Destroy event.
3573	*/
3574	if (sev_es_guest(kvm) && !VALID_PAGE(svm->vmcb->control.vmsa_pa))
3575	return -EINVAL;
3576
3577	/*
3578	* To optimize cache flushes when memory is reclaimed from an SEV VM,
3579	* track physical CPUs that enter the guest for SEV VMs and thus can
3580	* have encrypted, dirty data in the cache, and flush caches only for
3581	* CPUs that have entered the guest.
3582	*/
3583	if (!cpumask_test_cpu(cpu, cpumask: to_kvm_sev_info(kvm)->have_run_cpus))
3584	cpumask_set_cpu(cpu, dstp: to_kvm_sev_info(kvm)->have_run_cpus);
3585
3586	/* Assign the asid allocated with this SEV guest */
3587	svm->asid = asid;
3588
3589	/*
3590	* Flush guest TLB:
3591	*
3592	* 1) when different VMCB for the same ASID is to be run on the same host CPU.
3593	* 2) or this VMCB was executed on different host CPU in previous VMRUNs.
3594	*/
3595	if (sd->sev_vmcbs[asid] == svm->vmcb &&
3596	svm->vcpu.arch.last_vmentry_cpu == cpu)
3597	return 0;
3598
3599	sd->sev_vmcbs[asid] = svm->vmcb;
3600	svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
3601	vmcb_mark_dirty(vmcb: svm->vmcb, bit: VMCB_ASID);
3602	return 0;
3603	}
3604
3605	#define GHCB_SCRATCH_AREA_LIMIT (16ULL * PAGE_SIZE)
3606	static int setup_vmgexit_scratch(struct vcpu_svm *svm, bool sync, u64 len)
3607	{
3608	struct vmcb_control_area *control = &svm->vmcb->control;
3609	u64 ghcb_scratch_beg, ghcb_scratch_end;
3610	u64 scratch_gpa_beg, scratch_gpa_end;
3611	void *scratch_va;
3612
3613	scratch_gpa_beg = svm->sev_es.sw_scratch;
3614	if (!scratch_gpa_beg) {
3615	pr_err("vmgexit: scratch gpa not provided\n");
3616	goto e_scratch;
3617	}
3618
3619	scratch_gpa_end = scratch_gpa_beg + len;
3620	if (scratch_gpa_end < scratch_gpa_beg) {
3621	pr_err("vmgexit: scratch length (%#llx) not valid for scratch address (%#llx)\n",
3622	len, scratch_gpa_beg);
3623	goto e_scratch;
3624	}
3625
3626	if ((scratch_gpa_beg & PAGE_MASK) == control->ghcb_gpa) {
3627	/* Scratch area begins within GHCB */
3628	ghcb_scratch_beg = control->ghcb_gpa +
3629	offsetof(struct ghcb, shared_buffer);
3630	ghcb_scratch_end = control->ghcb_gpa +
3631	offsetof(struct ghcb, reserved_0xff0);
3632
3633	/*
3634	* If the scratch area begins within the GHCB, it must be
3635	* completely contained in the GHCB shared buffer area.
3636	*/
3637	if (scratch_gpa_beg < ghcb_scratch_beg ||
3638	scratch_gpa_end > ghcb_scratch_end) {
3639	pr_err("vmgexit: scratch area is outside of GHCB shared buffer area (%#llx - %#llx)\n",
3640	scratch_gpa_beg, scratch_gpa_end);
3641	goto e_scratch;
3642	}
3643
3644	scratch_va = (void *)svm->sev_es.ghcb;
3645	scratch_va += (scratch_gpa_beg - control->ghcb_gpa);
3646	} else {
3647	/*
3648	* The guest memory must be read into a kernel buffer, so
3649	* limit the size
3650	*/
3651	if (len > GHCB_SCRATCH_AREA_LIMIT) {
3652	pr_err("vmgexit: scratch area exceeds KVM limits (%#llx requested, %#llx limit)\n",
3653	len, GHCB_SCRATCH_AREA_LIMIT);
3654	goto e_scratch;
3655	}
3656	scratch_va = kvzalloc(len, GFP_KERNEL_ACCOUNT);
3657	if (!scratch_va)
3658	return -ENOMEM;
3659
3660	if (kvm_read_guest(kvm: svm->vcpu.kvm, gpa: scratch_gpa_beg, data: scratch_va, len)) {
3661	/* Unable to copy scratch area from guest */
3662	pr_err("vmgexit: kvm_read_guest for scratch area failed\n");
3663
3664	kvfree(addr: scratch_va);
3665	return -EFAULT;
3666	}
3667
3668	/*
3669	* The scratch area is outside the GHCB. The operation will
3670	* dictate whether the buffer needs to be synced before running
3671	* the vCPU next time (i.e. a read was requested so the data
3672	* must be written back to the guest memory).
3673	*/
3674	svm->sev_es.ghcb_sa_sync = sync;
3675	svm->sev_es.ghcb_sa_free = true;
3676	}
3677
3678	svm->sev_es.ghcb_sa = scratch_va;
3679	svm->sev_es.ghcb_sa_len = len;
3680
3681	return 0;
3682
3683	e_scratch:
3684	svm_vmgexit_bad_input(svm, GHCB_ERR_INVALID_SCRATCH_AREA);
3685
3686	return 1;
3687	}
3688
3689	static void set_ghcb_msr_bits(struct vcpu_svm *svm, u64 value, u64 mask,
3690	unsigned int pos)
3691	{
3692	svm->vmcb->control.ghcb_gpa &= ~(mask << pos);
3693	svm->vmcb->control.ghcb_gpa |= (value & mask) << pos;
3694	}
3695
3696	static u64 get_ghcb_msr_bits(struct vcpu_svm *svm, u64 mask, unsigned int pos)
3697	{
3698	return (svm->vmcb->control.ghcb_gpa >> pos) & mask;
3699	}
3700
3701	static void set_ghcb_msr(struct vcpu_svm *svm, u64 value)
3702	{
3703	svm->vmcb->control.ghcb_gpa = value;
3704	}
3705
3706	static int snp_rmptable_psmash(kvm_pfn_t pfn)
3707	{
3708	int ret;
3709
3710	pfn = pfn & ~(KVM_PAGES_PER_HPAGE(PG_LEVEL_2M) - 1);
3711
3712	/*
3713	* PSMASH_FAIL_INUSE indicates another processor is modifying the
3714	* entry, so retry until that's no longer the case.
3715	*/
3716	do {
3717	ret = psmash(pfn);
3718	} while (ret == PSMASH_FAIL_INUSE);
3719
3720	return ret;
3721	}
3722
3723	static int snp_complete_psc_msr(struct kvm_vcpu *vcpu)
3724	{
3725	struct vcpu_svm *svm = to_svm(vcpu);
3726
3727	if (vcpu->run->hypercall.ret)
3728	set_ghcb_msr(svm, GHCB_MSR_PSC_RESP_ERROR);
3729	else
3730	set_ghcb_msr(svm, GHCB_MSR_PSC_RESP);
3731
3732	return 1; /* resume guest */
3733	}
3734
3735	static int snp_begin_psc_msr(struct vcpu_svm *svm, u64 ghcb_msr)
3736	{
3737	u64 gpa = gfn_to_gpa(GHCB_MSR_PSC_REQ_TO_GFN(ghcb_msr));
3738	u8 op = GHCB_MSR_PSC_REQ_TO_OP(ghcb_msr);
3739	struct kvm_vcpu *vcpu = &svm->vcpu;
3740
3741	if (op != SNP_PAGE_STATE_PRIVATE && op != SNP_PAGE_STATE_SHARED) {
3742	set_ghcb_msr(svm, GHCB_MSR_PSC_RESP_ERROR);
3743	return 1; /* resume guest */
3744	}
3745
3746	if (!user_exit_on_hypercall(vcpu->kvm, KVM_HC_MAP_GPA_RANGE)) {
3747	set_ghcb_msr(svm, GHCB_MSR_PSC_RESP_ERROR);
3748	return 1; /* resume guest */
3749	}
3750
3751	vcpu->run->exit_reason = KVM_EXIT_HYPERCALL;
3752	vcpu->run->hypercall.nr = KVM_HC_MAP_GPA_RANGE;
3753	/*
3754	* In principle this should have been -KVM_ENOSYS, but userspace (QEMU <=9.2)
3755	* assumed that vcpu->run->hypercall.ret is never changed by KVM and thus that
3756	* it was always zero on KVM_EXIT_HYPERCALL. Since KVM is now overwriting
3757	* vcpu->run->hypercall.ret, ensuring that it is zero to not break QEMU.
3758	*/
3759	vcpu->run->hypercall.ret = 0;
3760	vcpu->run->hypercall.args[0] = gpa;
3761	vcpu->run->hypercall.args[1] = 1;
3762	vcpu->run->hypercall.args[2] = (op == SNP_PAGE_STATE_PRIVATE)
3763	? KVM_MAP_GPA_RANGE_ENCRYPTED
3764	: KVM_MAP_GPA_RANGE_DECRYPTED;
3765	vcpu->run->hypercall.args[2] |= KVM_MAP_GPA_RANGE_PAGE_SZ_4K;
3766
3767	vcpu->arch.complete_userspace_io = snp_complete_psc_msr;
3768
3769	return 0; /* forward request to userspace */
3770	}
3771
3772	struct psc_buffer {
3773	struct psc_hdr hdr;
3774	struct psc_entry entries[];
3775	} __packed;
3776
3777	static int snp_begin_psc(struct vcpu_svm *svm, struct psc_buffer *psc);
3778
3779	static void snp_complete_psc(struct vcpu_svm *svm, u64 psc_ret)
3780	{
3781	svm->sev_es.psc_inflight = 0;
3782	svm->sev_es.psc_idx = 0;
3783	svm->sev_es.psc_2m = false;
3784
3785	/*
3786	* PSC requests always get a "no action" response in SW_EXITINFO1, with
3787	* a PSC-specific return code in SW_EXITINFO2 that provides the "real"
3788	* return code. E.g. if the PSC request was interrupted, the need to
3789	* retry is communicated via SW_EXITINFO2, not SW_EXITINFO1.
3790	*/
3791	svm_vmgexit_no_action(svm, data: psc_ret);
3792	}
3793
3794	static void __snp_complete_one_psc(struct vcpu_svm *svm)
3795	{
3796	struct psc_buffer *psc = svm->sev_es.ghcb_sa;
3797	struct psc_entry *entries = psc->entries;
3798	struct psc_hdr *hdr = &psc->hdr;
3799	__u16 idx;
3800
3801	/*
3802	* Everything in-flight has been processed successfully. Update the
3803	* corresponding entries in the guest's PSC buffer and zero out the
3804	* count of in-flight PSC entries.
3805	*/
3806	for (idx = svm->sev_es.psc_idx; svm->sev_es.psc_inflight;
3807	svm->sev_es.psc_inflight--, idx++) {
3808	struct psc_entry *entry = &entries[idx];
3809
3810	entry->cur_page = entry->pagesize ? 512 : 1;
3811	}
3812
3813	hdr->cur_entry = idx;
3814	}
3815
3816	static int snp_complete_one_psc(struct kvm_vcpu *vcpu)
3817	{
3818	struct vcpu_svm *svm = to_svm(vcpu);
3819	struct psc_buffer *psc = svm->sev_es.ghcb_sa;
3820
3821	if (vcpu->run->hypercall.ret) {
3822	snp_complete_psc(svm, VMGEXIT_PSC_ERROR_GENERIC);
3823	return 1; /* resume guest */
3824	}
3825
3826	__snp_complete_one_psc(svm);
3827
3828	/* Handle the next range (if any). */
3829	return snp_begin_psc(svm, psc);
3830	}
3831
3832	static int snp_begin_psc(struct vcpu_svm *svm, struct psc_buffer *psc)
3833	{
3834	struct psc_entry *entries = psc->entries;
3835	struct kvm_vcpu *vcpu = &svm->vcpu;
3836	struct psc_hdr *hdr = &psc->hdr;
3837	struct psc_entry entry_start;
3838	u16 idx, idx_start, idx_end;
3839	int npages;
3840	bool huge;
3841	u64 gfn;
3842
3843	if (!user_exit_on_hypercall(vcpu->kvm, KVM_HC_MAP_GPA_RANGE)) {
3844	snp_complete_psc(svm, VMGEXIT_PSC_ERROR_GENERIC);
3845	return 1;
3846	}
3847
3848	next_range:
3849	/* There should be no other PSCs in-flight at this point. */
3850	if (WARN_ON_ONCE(svm->sev_es.psc_inflight)) {
3851	snp_complete_psc(svm, VMGEXIT_PSC_ERROR_GENERIC);
3852	return 1;
3853	}
3854
3855	/*
3856	* The PSC descriptor buffer can be modified by a misbehaved guest after
3857	* validation, so take care to only use validated copies of values used
3858	* for things like array indexing.
3859	*/
3860	idx_start = hdr->cur_entry;
3861	idx_end = hdr->end_entry;
3862
3863	if (idx_end >= VMGEXIT_PSC_MAX_COUNT) {
3864	snp_complete_psc(svm, VMGEXIT_PSC_ERROR_INVALID_HDR);
3865	return 1;
3866	}
3867
3868	/* Find the start of the next range which needs processing. */
3869	for (idx = idx_start; idx <= idx_end; idx++, hdr->cur_entry++) {
3870	entry_start = entries[idx];
3871
3872	gfn = entry_start.gfn;
3873	huge = entry_start.pagesize;
3874	npages = huge ? 512 : 1;
3875
3876	if (entry_start.cur_page > npages || !IS_ALIGNED(gfn, npages)) {
3877	snp_complete_psc(svm, VMGEXIT_PSC_ERROR_INVALID_ENTRY);
3878	return 1;
3879	}
3880
3881	if (entry_start.cur_page) {
3882	/*
3883	* If this is a partially-completed 2M range, force 4K handling
3884	* for the remaining pages since they're effectively split at
3885	* this point. Subsequent code should ensure this doesn't get
3886	* combined with adjacent PSC entries where 2M handling is still
3887	* possible.
3888	*/
3889	npages -= entry_start.cur_page;
3890	gfn += entry_start.cur_page;
3891	huge = false;
3892	}
3893
3894	if (npages)
3895	break;
3896	}
3897
3898	if (idx > idx_end) {
3899	/* Nothing more to process. */
3900	snp_complete_psc(svm, psc_ret: 0);
3901	return 1;
3902	}
3903
3904	svm->sev_es.psc_2m = huge;
3905	svm->sev_es.psc_idx = idx;
3906	svm->sev_es.psc_inflight = 1;
3907
3908	/*
3909	* Find all subsequent PSC entries that contain adjacent GPA
3910	* ranges/operations and can be combined into a single
3911	* KVM_HC_MAP_GPA_RANGE exit.
3912	*/
3913	while (++idx <= idx_end) {
3914	struct psc_entry entry = entries[idx];
3915
3916	if (entry.operation != entry_start.operation ||
3917	entry.gfn != entry_start.gfn + npages ||
3918	entry.cur_page || !!entry.pagesize != huge)
3919	break;
3920
3921	svm->sev_es.psc_inflight++;
3922	npages += huge ? 512 : 1;
3923	}
3924
3925	switch (entry_start.operation) {
3926	case VMGEXIT_PSC_OP_PRIVATE:
3927	case VMGEXIT_PSC_OP_SHARED:
3928	vcpu->run->exit_reason = KVM_EXIT_HYPERCALL;
3929	vcpu->run->hypercall.nr = KVM_HC_MAP_GPA_RANGE;
3930	/*
3931	* In principle this should have been -KVM_ENOSYS, but userspace (QEMU <=9.2)
3932	* assumed that vcpu->run->hypercall.ret is never changed by KVM and thus that
3933	* it was always zero on KVM_EXIT_HYPERCALL. Since KVM is now overwriting
3934	* vcpu->run->hypercall.ret, ensuring that it is zero to not break QEMU.
3935	*/
3936	vcpu->run->hypercall.ret = 0;
3937	vcpu->run->hypercall.args[0] = gfn_to_gpa(gfn);
3938	vcpu->run->hypercall.args[1] = npages;
3939	vcpu->run->hypercall.args[2] = entry_start.operation == VMGEXIT_PSC_OP_PRIVATE
3940	? KVM_MAP_GPA_RANGE_ENCRYPTED
3941	: KVM_MAP_GPA_RANGE_DECRYPTED;
3942	vcpu->run->hypercall.args[2] |= entry_start.pagesize
3943	? KVM_MAP_GPA_RANGE_PAGE_SZ_2M
3944	: KVM_MAP_GPA_RANGE_PAGE_SZ_4K;
3945	vcpu->arch.complete_userspace_io = snp_complete_one_psc;
3946	return 0; /* forward request to userspace */
3947	default:
3948	/*
3949	* Only shared/private PSC operations are currently supported, so if the
3950	* entire range consists of unsupported operations (e.g. SMASH/UNSMASH),
3951	* then consider the entire range completed and avoid exiting to
3952	* userspace. In theory snp_complete_psc() can always be called directly
3953	* at this point to complete the current range and start the next one,
3954	* but that could lead to unexpected levels of recursion.
3955	*/
3956	__snp_complete_one_psc(svm);
3957	goto next_range;
3958	}
3959
3960	BUG();
3961	}
3962
3963	/*
3964	* Invoked as part of svm_vcpu_reset() processing of an init event.
3965	*/
3966	static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu)
3967	{
3968	struct vcpu_svm *svm = to_svm(vcpu);
3969	struct kvm_memory_slot *slot;
3970	struct page *page;
3971	kvm_pfn_t pfn;
3972	gfn_t gfn;
3973
3974	guard(mutex)(T: &svm->sev_es.snp_vmsa_mutex);
3975
3976	if (!svm->sev_es.snp_ap_waiting_for_reset)
3977	return;
3978
3979	svm->sev_es.snp_ap_waiting_for_reset = false;
3980
3981	/* Mark the vCPU as offline and not runnable */
3982	vcpu->arch.pv.pv_unhalted = false;
3983	kvm_set_mp_state(vcpu, KVM_MP_STATE_HALTED);
3984
3985	/* Clear use of the VMSA */
3986	svm->vmcb->control.vmsa_pa = INVALID_PAGE;
3987
3988	/*
3989	* When replacing the VMSA during SEV-SNP AP creation,
3990	* mark the VMCB dirty so that full state is always reloaded.
3991	*/
3992	vmcb_mark_all_dirty(vmcb: svm->vmcb);
3993
3994	if (!VALID_PAGE(svm->sev_es.snp_vmsa_gpa))
3995	return;
3996
3997	gfn = gpa_to_gfn(gpa: svm->sev_es.snp_vmsa_gpa);
3998	svm->sev_es.snp_vmsa_gpa = INVALID_PAGE;
3999
4000	slot = gfn_to_memslot(kvm: vcpu->kvm, gfn);
4001	if (!slot)
4002	return;
4003
4004	/*
4005	* The new VMSA will be private memory guest memory, so retrieve the
4006	* PFN from the gmem backend.
4007	*/
4008	if (kvm_gmem_get_pfn(kvm: vcpu->kvm, slot, gfn, pfn: &pfn, page: &page, NULL))
4009	return;
4010
4011	/*
4012	* From this point forward, the VMSA will always be a guest-mapped page
4013	* rather than the initial one allocated by KVM in svm->sev_es.vmsa. In
4014	* theory, svm->sev_es.vmsa could be free'd and cleaned up here, but
4015	* that involves cleanups like flushing caches, which would ideally be
4016	* handled during teardown rather than guest boot. Deferring that also
4017	* allows the existing logic for SEV-ES VMSAs to be re-used with
4018	* minimal SNP-specific changes.
4019	*/
4020	svm->sev_es.snp_has_guest_vmsa = true;
4021
4022	/* Use the new VMSA */
4023	svm->vmcb->control.vmsa_pa = pfn_to_hpa(pfn);
4024
4025	/* Mark the vCPU as runnable */
4026	kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE);
4027
4028	/*
4029	* gmem pages aren't currently migratable, but if this ever changes
4030	* then care should be taken to ensure svm->sev_es.vmsa is pinned
4031	* through some other means.
4032	*/
4033	kvm_release_page_clean(page);
4034	}
4035
4036	static int sev_snp_ap_creation(struct vcpu_svm *svm)
4037	{
4038	struct kvm_sev_info *sev = to_kvm_sev_info(kvm: svm->vcpu.kvm);
4039	struct kvm_vcpu *vcpu = &svm->vcpu;
4040	struct kvm_vcpu *target_vcpu;
4041	struct vcpu_svm *target_svm;
4042	unsigned int request;
4043	unsigned int apic_id;
4044
4045	request = lower_32_bits(svm->vmcb->control.exit_info_1);
4046	apic_id = upper_32_bits(svm->vmcb->control.exit_info_1);
4047
4048	/* Validate the APIC ID */
4049	target_vcpu = kvm_get_vcpu_by_id(kvm: vcpu->kvm, id: apic_id);
4050	if (!target_vcpu) {
4051	vcpu_unimpl(vcpu, "vmgexit: invalid AP APIC ID [%#x] from guest\n",
4052	apic_id);
4053	return -EINVAL;
4054	}
4055
4056	target_svm = to_svm(vcpu: target_vcpu);
4057
4058	guard(mutex)(T: &target_svm->sev_es.snp_vmsa_mutex);
4059
4060	switch (request) {
4061	case SVM_VMGEXIT_AP_CREATE_ON_INIT:
4062	case SVM_VMGEXIT_AP_CREATE:
4063	if (vcpu->arch.regs[VCPU_REGS_RAX] != sev->vmsa_features) {
4064	vcpu_unimpl(vcpu, "vmgexit: mismatched AP sev_features [%#lx] != [%#llx] from guest\n",
4065	vcpu->arch.regs[VCPU_REGS_RAX], sev->vmsa_features);
4066	return -EINVAL;
4067	}
4068
4069	if (!page_address_valid(vcpu, svm->vmcb->control.exit_info_2)) {
4070	vcpu_unimpl(vcpu, "vmgexit: invalid AP VMSA address [%#llx] from guest\n",
4071	svm->vmcb->control.exit_info_2);
4072	return -EINVAL;
4073	}
4074
4075	/*
4076	* Malicious guest can RMPADJUST a large page into VMSA which
4077	* will hit the SNP erratum where the CPU will incorrectly signal
4078	* an RMP violation #PF if a hugepage collides with the RMP entry
4079	* of VMSA page, reject the AP CREATE request if VMSA address from
4080	* guest is 2M aligned.
4081	*/
4082	if (IS_ALIGNED(svm->vmcb->control.exit_info_2, PMD_SIZE)) {
4083	vcpu_unimpl(vcpu,
4084	"vmgexit: AP VMSA address [%llx] from guest is unsafe as it is 2M aligned\n",
4085	svm->vmcb->control.exit_info_2);
4086	return -EINVAL;
4087	}
4088
4089	target_svm->sev_es.snp_vmsa_gpa = svm->vmcb->control.exit_info_2;
4090	break;
4091	case SVM_VMGEXIT_AP_DESTROY:
4092	target_svm->sev_es.snp_vmsa_gpa = INVALID_PAGE;
4093	break;
4094	default:
4095	vcpu_unimpl(vcpu, "vmgexit: invalid AP creation request [%#x] from guest\n",
4096	request);
4097	return -EINVAL;
4098	}
4099
4100	target_svm->sev_es.snp_ap_waiting_for_reset = true;
4101
4102	/*
4103	* Unless Creation is deferred until INIT, signal the vCPU to update
4104	* its state.
4105	*/
4106	if (request != SVM_VMGEXIT_AP_CREATE_ON_INIT)
4107	kvm_make_request_and_kick(KVM_REQ_UPDATE_PROTECTED_GUEST_STATE, vcpu: target_vcpu);
4108
4109	return 0;
4110	}
4111
4112	static int snp_handle_guest_req(struct vcpu_svm *svm, gpa_t req_gpa, gpa_t resp_gpa)
4113	{
4114	struct sev_data_snp_guest_request data = {0};
4115	struct kvm *kvm = svm->vcpu.kvm;
4116	struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
4117	sev_ret_code fw_err = 0;
4118	int ret;
4119
4120	if (!sev_snp_guest(kvm))
4121	return -EINVAL;
4122
4123	mutex_lock(&sev->guest_req_mutex);
4124
4125	if (kvm_read_guest(kvm, gpa: req_gpa, data: sev->guest_req_buf, PAGE_SIZE)) {
4126	ret = -EIO;
4127	goto out_unlock;
4128	}
4129
4130	data.gctx_paddr = __psp_pa(sev->snp_context);
4131	data.req_paddr = __psp_pa(sev->guest_req_buf);
4132	data.res_paddr = __psp_pa(sev->guest_resp_buf);
4133
4134	/*
4135	* Firmware failures are propagated on to guest, but any other failure
4136	* condition along the way should be reported to userspace. E.g. if
4137	* the PSP is dead and commands are timing out.
4138	*/
4139	ret = sev_issue_cmd(kvm, id: SEV_CMD_SNP_GUEST_REQUEST, data: &data, error: &fw_err);
4140	if (ret && !fw_err)
4141	goto out_unlock;
4142
4143	if (kvm_write_guest(kvm, gpa: resp_gpa, data: sev->guest_resp_buf, PAGE_SIZE)) {
4144	ret = -EIO;
4145	goto out_unlock;
4146	}
4147
4148	/* No action is requested *from KVM* if there was a firmware error. */
4149	svm_vmgexit_no_action(svm, SNP_GUEST_ERR(0, fw_err));
4150
4151	ret = 1; /* resume guest */
4152
4153	out_unlock:
4154	mutex_unlock(lock: &sev->guest_req_mutex);
4155	return ret;
4156	}
4157
4158	static int snp_handle_ext_guest_req(struct vcpu_svm *svm, gpa_t req_gpa, gpa_t resp_gpa)
4159	{
4160	struct kvm *kvm = svm->vcpu.kvm;
4161	u8 msg_type;
4162
4163	if (!sev_snp_guest(kvm))
4164	return -EINVAL;
4165
4166	if (kvm_read_guest(kvm, gpa: req_gpa + offsetof(struct snp_guest_msg_hdr, msg_type),
4167	data: &msg_type, len: 1))
4168	return -EIO;
4169
4170	/*
4171	* As per GHCB spec, requests of type MSG_REPORT_REQ also allow for
4172	* additional certificate data to be provided alongside the attestation
4173	* report via the guest-provided data pages indicated by RAX/RBX. The
4174	* certificate data is optional and requires additional KVM enablement
4175	* to provide an interface for userspace to provide it, but KVM still
4176	* needs to be able to handle extended guest requests either way. So
4177	* provide a stub implementation that will always return an empty
4178	* certificate table in the guest-provided data pages.
4179	*/
4180	if (msg_type == SNP_MSG_REPORT_REQ) {
4181	struct kvm_vcpu *vcpu = &svm->vcpu;
4182	u64 data_npages;
4183	gpa_t data_gpa;
4184
4185	if (!kvm_ghcb_rax_is_valid(svm) || !kvm_ghcb_rbx_is_valid(svm))
4186	goto request_invalid;
4187
4188	data_gpa = vcpu->arch.regs[VCPU_REGS_RAX];
4189	data_npages = vcpu->arch.regs[VCPU_REGS_RBX];
4190
4191	if (!PAGE_ALIGNED(data_gpa))
4192	goto request_invalid;
4193
4194	/*
4195	* As per GHCB spec (see "SNP Extended Guest Request"), the
4196	* certificate table is terminated by 24-bytes of zeroes.
4197	*/
4198	if (data_npages && kvm_clear_guest(kvm, gpa: data_gpa, len: 24))
4199	return -EIO;
4200	}
4201
4202	return snp_handle_guest_req(svm, req_gpa, resp_gpa);
4203
4204	request_invalid:
4205	svm_vmgexit_bad_input(svm, GHCB_ERR_INVALID_INPUT);
4206	return 1; /* resume guest */
4207	}
4208
4209	static int sev_handle_vmgexit_msr_protocol(struct vcpu_svm *svm)
4210	{
4211	struct vmcb_control_area *control = &svm->vmcb->control;
4212	struct kvm_vcpu *vcpu = &svm->vcpu;
4213	struct kvm_sev_info *sev = to_kvm_sev_info(kvm: vcpu->kvm);
4214	u64 ghcb_info;
4215	int ret = 1;
4216
4217	ghcb_info = control->ghcb_gpa & GHCB_MSR_INFO_MASK;
4218
4219	trace_kvm_vmgexit_msr_protocol_enter(svm->vcpu.vcpu_id,
4220	control->ghcb_gpa);
4221
4222	switch (ghcb_info) {
4223	case GHCB_MSR_SEV_INFO_REQ:
4224	set_ghcb_msr(svm, GHCB_MSR_SEV_INFO((__u64)sev->ghcb_version,
4225	GHCB_VERSION_MIN,
4226	sev_enc_bit));
4227	break;
4228	case GHCB_MSR_CPUID_REQ: {
4229	u64 cpuid_fn, cpuid_reg, cpuid_value;
4230
4231	cpuid_fn = get_ghcb_msr_bits(svm,
4232	GHCB_MSR_CPUID_FUNC_MASK,
4233	GHCB_MSR_CPUID_FUNC_POS);
4234
4235	/* Initialize the registers needed by the CPUID intercept */
4236	vcpu->arch.regs[VCPU_REGS_RAX] = cpuid_fn;
4237	vcpu->arch.regs[VCPU_REGS_RCX] = 0;
4238
4239	ret = svm_invoke_exit_handler(vcpu, SVM_EXIT_CPUID);
4240	if (!ret) {
4241	/* Error, keep GHCB MSR value as-is */
4242	break;
4243	}
4244
4245	cpuid_reg = get_ghcb_msr_bits(svm,
4246	GHCB_MSR_CPUID_REG_MASK,
4247	GHCB_MSR_CPUID_REG_POS);
4248	if (cpuid_reg == 0)
4249	cpuid_value = vcpu->arch.regs[VCPU_REGS_RAX];
4250	else if (cpuid_reg == 1)
4251	cpuid_value = vcpu->arch.regs[VCPU_REGS_RBX];
4252	else if (cpuid_reg == 2)
4253	cpuid_value = vcpu->arch.regs[VCPU_REGS_RCX];
4254	else
4255	cpuid_value = vcpu->arch.regs[VCPU_REGS_RDX];
4256
4257	set_ghcb_msr_bits(svm, value: cpuid_value,
4258	GHCB_MSR_CPUID_VALUE_MASK,
4259	GHCB_MSR_CPUID_VALUE_POS);
4260
4261	set_ghcb_msr_bits(svm, GHCB_MSR_CPUID_RESP,
4262	GHCB_MSR_INFO_MASK,
4263	GHCB_MSR_INFO_POS);
4264	break;
4265	}
4266	case GHCB_MSR_AP_RESET_HOLD_REQ:
4267	svm->sev_es.ap_reset_hold_type = AP_RESET_HOLD_MSR_PROTO;
4268	ret = kvm_emulate_ap_reset_hold(vcpu: &svm->vcpu);
4269
4270	/*
4271	* Preset the result to a non-SIPI return and then only set
4272	* the result to non-zero when delivering a SIPI.
4273	*/
4274	set_ghcb_msr_bits(svm, value: 0,
4275	GHCB_MSR_AP_RESET_HOLD_RESULT_MASK,
4276	GHCB_MSR_AP_RESET_HOLD_RESULT_POS);
4277
4278	set_ghcb_msr_bits(svm, GHCB_MSR_AP_RESET_HOLD_RESP,
4279	GHCB_MSR_INFO_MASK,
4280	GHCB_MSR_INFO_POS);
4281	break;
4282	case GHCB_MSR_HV_FT_REQ:
4283	set_ghcb_msr_bits(svm, GHCB_HV_FT_SUPPORTED,
4284	GHCB_MSR_HV_FT_MASK, GHCB_MSR_HV_FT_POS);
4285	set_ghcb_msr_bits(svm, GHCB_MSR_HV_FT_RESP,
4286	GHCB_MSR_INFO_MASK, GHCB_MSR_INFO_POS);
4287	break;
4288	case GHCB_MSR_PREF_GPA_REQ:
4289	if (!sev_snp_guest(kvm: vcpu->kvm))
4290	goto out_terminate;
4291
4292	set_ghcb_msr_bits(svm, GHCB_MSR_PREF_GPA_NONE, GHCB_MSR_GPA_VALUE_MASK,
4293	GHCB_MSR_GPA_VALUE_POS);
4294	set_ghcb_msr_bits(svm, GHCB_MSR_PREF_GPA_RESP, GHCB_MSR_INFO_MASK,
4295	GHCB_MSR_INFO_POS);
4296	break;
4297	case GHCB_MSR_REG_GPA_REQ: {
4298	u64 gfn;
4299
4300	if (!sev_snp_guest(kvm: vcpu->kvm))
4301	goto out_terminate;
4302
4303	gfn = get_ghcb_msr_bits(svm, GHCB_MSR_GPA_VALUE_MASK,
4304	GHCB_MSR_GPA_VALUE_POS);
4305
4306	svm->sev_es.ghcb_registered_gpa = gfn_to_gpa(gfn);
4307
4308	set_ghcb_msr_bits(svm, value: gfn, GHCB_MSR_GPA_VALUE_MASK,
4309	GHCB_MSR_GPA_VALUE_POS);
4310	set_ghcb_msr_bits(svm, GHCB_MSR_REG_GPA_RESP, GHCB_MSR_INFO_MASK,
4311	GHCB_MSR_INFO_POS);
4312	break;
4313	}
4314	case GHCB_MSR_PSC_REQ:
4315	if (!sev_snp_guest(kvm: vcpu->kvm))
4316	goto out_terminate;
4317
4318	ret = snp_begin_psc_msr(svm, ghcb_msr: control->ghcb_gpa);
4319	break;
4320	case GHCB_MSR_TERM_REQ: {
4321	u64 reason_set, reason_code;
4322
4323	reason_set = get_ghcb_msr_bits(svm,
4324	GHCB_MSR_TERM_REASON_SET_MASK,
4325	GHCB_MSR_TERM_REASON_SET_POS);
4326	reason_code = get_ghcb_msr_bits(svm,
4327	GHCB_MSR_TERM_REASON_MASK,
4328	GHCB_MSR_TERM_REASON_POS);
4329	pr_info("SEV-ES guest requested termination: %#llx:%#llx\n",
4330	reason_set, reason_code);
4331
4332	goto out_terminate;
4333	}
4334	default:
4335	/* Error, keep GHCB MSR value as-is */
4336	break;
4337	}
4338
4339	trace_kvm_vmgexit_msr_protocol_exit(svm->vcpu.vcpu_id,
4340	control->ghcb_gpa, ret);
4341
4342	return ret;
4343
4344	out_terminate:
4345	vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
4346	vcpu->run->system_event.type = KVM_SYSTEM_EVENT_SEV_TERM;
4347	vcpu->run->system_event.ndata = 1;
4348	vcpu->run->system_event.data[0] = control->ghcb_gpa;
4349
4350	return 0;
4351	}
4352
4353	int sev_handle_vmgexit(struct kvm_vcpu *vcpu)
4354	{
4355	struct vcpu_svm *svm = to_svm(vcpu);
4356	struct vmcb_control_area *control = &svm->vmcb->control;
4357	u64 ghcb_gpa, exit_code;
4358	int ret;
4359
4360	/* Validate the GHCB */
4361	ghcb_gpa = control->ghcb_gpa;
4362	if (ghcb_gpa & GHCB_MSR_INFO_MASK)
4363	return sev_handle_vmgexit_msr_protocol(svm);
4364
4365	if (!ghcb_gpa) {
4366	vcpu_unimpl(vcpu, "vmgexit: GHCB gpa is not set\n");
4367
4368	/* Without a GHCB, just return right back to the guest */
4369	return 1;
4370	}
4371
4372	if (kvm_vcpu_map(vcpu, gpa: ghcb_gpa >> PAGE_SHIFT, map: &svm->sev_es.ghcb_map)) {
4373	/* Unable to map GHCB from guest */
4374	vcpu_unimpl(vcpu, "vmgexit: error mapping GHCB [%#llx] from guest\n",
4375	ghcb_gpa);
4376
4377	/* Without a GHCB, just return right back to the guest */
4378	return 1;
4379	}
4380
4381	svm->sev_es.ghcb = svm->sev_es.ghcb_map.hva;
4382
4383	trace_kvm_vmgexit_enter(vcpu->vcpu_id, svm->sev_es.ghcb);
4384
4385	sev_es_sync_from_ghcb(svm);
4386
4387	/* SEV-SNP guest requires that the GHCB GPA must be registered */
4388	if (sev_snp_guest(kvm: svm->vcpu.kvm) && !ghcb_gpa_is_registered(svm, val: ghcb_gpa)) {
4389	vcpu_unimpl(&svm->vcpu, "vmgexit: GHCB GPA [%#llx] is not registered.\n", ghcb_gpa);
4390	return -EINVAL;
4391	}
4392
4393	ret = sev_es_validate_vmgexit(svm);
4394	if (ret)
4395	return ret;
4396
4397	svm_vmgexit_success(svm, data: 0);
4398
4399	exit_code = kvm_get_cached_sw_exit_code(control);
4400	switch (exit_code) {
4401	case SVM_VMGEXIT_MMIO_READ:
4402	ret = setup_vmgexit_scratch(svm, sync: true, len: control->exit_info_2);
4403	if (ret)
4404	break;
4405
4406	ret = kvm_sev_es_mmio_read(vcpu,
4407	control->exit_info_1,
4408	control->exit_info_2,
4409	svm->sev_es.ghcb_sa);
4410	break;
4411	case SVM_VMGEXIT_MMIO_WRITE:
4412	ret = setup_vmgexit_scratch(svm, sync: false, len: control->exit_info_2);
4413	if (ret)
4414	break;
4415
4416	ret = kvm_sev_es_mmio_write(vcpu,
4417	control->exit_info_1,
4418	control->exit_info_2,
4419	svm->sev_es.ghcb_sa);
4420	break;
4421	case SVM_VMGEXIT_NMI_COMPLETE:
4422	++vcpu->stat.nmi_window_exits;
4423	svm->nmi_masked = false;
4424	kvm_make_request(KVM_REQ_EVENT, vcpu);
4425	ret = 1;
4426	break;
4427	case SVM_VMGEXIT_AP_HLT_LOOP:
4428	svm->sev_es.ap_reset_hold_type = AP_RESET_HOLD_NAE_EVENT;
4429	ret = kvm_emulate_ap_reset_hold(vcpu);
4430	break;
4431	case SVM_VMGEXIT_AP_JUMP_TABLE: {
4432	struct kvm_sev_info *sev = to_kvm_sev_info(kvm: vcpu->kvm);
4433
4434	switch (control->exit_info_1) {
4435	case 0:
4436	/* Set AP jump table address */
4437	sev->ap_jump_table = control->exit_info_2;
4438	break;
4439	case 1:
4440	/* Get AP jump table address */
4441	svm_vmgexit_success(svm, data: sev->ap_jump_table);
4442	break;
4443	default:
4444	pr_err("svm: vmgexit: unsupported AP jump table request - exit_info_1=%#llx\n",
4445	control->exit_info_1);
4446	svm_vmgexit_bad_input(svm, GHCB_ERR_INVALID_INPUT);
4447	}
4448
4449	ret = 1;
4450	break;
4451	}
4452	case SVM_VMGEXIT_HV_FEATURES:
4453	svm_vmgexit_success(svm, GHCB_HV_FT_SUPPORTED);
4454	ret = 1;
4455	break;
4456	case SVM_VMGEXIT_TERM_REQUEST:
4457	pr_info("SEV-ES guest requested termination: reason %#llx info %#llx\n",
4458	control->exit_info_1, control->exit_info_2);
4459	vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
4460	vcpu->run->system_event.type = KVM_SYSTEM_EVENT_SEV_TERM;
4461	vcpu->run->system_event.ndata = 1;
4462	vcpu->run->system_event.data[0] = control->ghcb_gpa;
4463	break;
4464	case SVM_VMGEXIT_PSC:
4465	ret = setup_vmgexit_scratch(svm, sync: true, len: control->exit_info_2);
4466	if (ret)
4467	break;
4468
4469	ret = snp_begin_psc(svm, psc: svm->sev_es.ghcb_sa);
4470	break;
4471	case SVM_VMGEXIT_AP_CREATION:
4472	ret = sev_snp_ap_creation(svm);
4473	if (ret) {
4474	svm_vmgexit_bad_input(svm, GHCB_ERR_INVALID_INPUT);
4475	}
4476
4477	ret = 1;
4478	break;
4479	case SVM_VMGEXIT_GUEST_REQUEST:
4480	ret = snp_handle_guest_req(svm, req_gpa: control->exit_info_1, resp_gpa: control->exit_info_2);
4481	break;
4482	case SVM_VMGEXIT_EXT_GUEST_REQUEST:
4483	ret = snp_handle_ext_guest_req(svm, req_gpa: control->exit_info_1, resp_gpa: control->exit_info_2);
4484	break;
4485	case SVM_VMGEXIT_UNSUPPORTED_EVENT:
4486	vcpu_unimpl(vcpu,
4487	"vmgexit: unsupported event - exit_info_1=%#llx, exit_info_2=%#llx\n",
4488	control->exit_info_1, control->exit_info_2);
4489	ret = -EINVAL;
4490	break;
4491	default:
4492	ret = svm_invoke_exit_handler(vcpu, exit_code);
4493	}
4494
4495	return ret;
4496	}
4497
4498	int sev_es_string_io(struct vcpu_svm *svm, int size, unsigned int port, int in)
4499	{
4500	int count;
4501	int bytes;
4502	int r;
4503
4504	if (svm->vmcb->control.exit_info_2 > INT_MAX)
4505	return -EINVAL;
4506
4507	count = svm->vmcb->control.exit_info_2;
4508	if (unlikely(check_mul_overflow(count, size, &bytes)))
4509	return -EINVAL;
4510
4511	r = setup_vmgexit_scratch(svm, sync: in, len: bytes);
4512	if (r)
4513	return r;
4514
4515	return kvm_sev_es_string_io(&svm->vcpu, size, port, svm->sev_es.ghcb_sa,
4516	count, in);
4517	}
4518
4519	void sev_es_recalc_msr_intercepts(struct kvm_vcpu *vcpu)
4520	{
4521	/* Clear intercepts on MSRs that are context switched by hardware. */
4522	svm_disable_intercept_for_msr(vcpu, MSR_AMD64_SEV_ES_GHCB, type: MSR_TYPE_RW);
4523	svm_disable_intercept_for_msr(vcpu, MSR_EFER, type: MSR_TYPE_RW);
4524	svm_disable_intercept_for_msr(vcpu, MSR_IA32_CR_PAT, type: MSR_TYPE_RW);
4525
4526	if (boot_cpu_has(X86_FEATURE_V_TSC_AUX))
4527	svm_set_intercept_for_msr(vcpu, MSR_TSC_AUX, type: MSR_TYPE_RW,
4528	set: !guest_cpu_cap_has(vcpu, X86_FEATURE_RDTSCP) &&
4529	!guest_cpu_cap_has(vcpu, X86_FEATURE_RDPID));
4530
4531	svm_set_intercept_for_msr(vcpu, MSR_AMD64_GUEST_TSC_FREQ, type: MSR_TYPE_R,
4532	set: !snp_is_secure_tsc_enabled(kvm: vcpu->kvm));
4533
4534	/*
4535	* For SEV-ES, accesses to MSR_IA32_XSS should not be intercepted if
4536	* the host/guest supports its use.
4537	*
4538	* KVM treats the guest as being capable of using XSAVES even if XSAVES
4539	* isn't enabled in guest CPUID as there is no intercept for XSAVES,
4540	* i.e. the guest can use XSAVES/XRSTOR to read/write XSS if XSAVE is
4541	* exposed to the guest and XSAVES is supported in hardware. Condition
4542	* full XSS passthrough on the guest being able to use XSAVES *and*
4543	* XSAVES being exposed to the guest so that KVM can at least honor
4544	* guest CPUID for RDMSR and WRMSR.
4545	*/
4546	svm_set_intercept_for_msr(vcpu, MSR_IA32_XSS, type: MSR_TYPE_RW,
4547	set: !guest_cpu_cap_has(vcpu, X86_FEATURE_XSAVES) ||
4548	!guest_cpuid_has(vcpu, X86_FEATURE_XSAVES));
4549	}
4550
4551	void sev_vcpu_after_set_cpuid(struct vcpu_svm *svm)
4552	{
4553	struct kvm_vcpu *vcpu = &svm->vcpu;
4554	struct kvm_cpuid_entry2 *best;
4555
4556	/* For sev guests, the memory encryption bit is not reserved in CR3. */
4557	best = kvm_find_cpuid_entry(vcpu, 0x8000001F);
4558	if (best)
4559	vcpu->arch.reserved_gpa_bits &= ~(1UL << (best->ebx & 0x3f));
4560	}
4561
4562	static void sev_es_init_vmcb(struct vcpu_svm *svm, bool init_event)
4563	{
4564	struct kvm_sev_info *sev = to_kvm_sev_info(kvm: svm->vcpu.kvm);
4565	struct vmcb *vmcb = svm->vmcb01.ptr;
4566
4567	svm->vmcb->control.nested_ctl |= SVM_NESTED_CTL_SEV_ES_ENABLE;
4568
4569	/*
4570	* An SEV-ES guest requires a VMSA area that is a separate from the
4571	* VMCB page. Do not include the encryption mask on the VMSA physical
4572	* address since hardware will access it using the guest key. Note,
4573	* the VMSA will be NULL if this vCPU is the destination for intrahost
4574	* migration, and will be copied later.
4575	*/
4576	if (!svm->sev_es.snp_has_guest_vmsa) {
4577	if (svm->sev_es.vmsa)
4578	svm->vmcb->control.vmsa_pa = __pa(svm->sev_es.vmsa);
4579	else
4580	svm->vmcb->control.vmsa_pa = INVALID_PAGE;
4581	}
4582
4583	if (cpu_feature_enabled(X86_FEATURE_ALLOWED_SEV_FEATURES))
4584	svm->vmcb->control.allowed_sev_features = sev->vmsa_features |
4585	VMCB_ALLOWED_SEV_FEATURES_VALID;
4586
4587	/* Can't intercept CR register access, HV can't modify CR registers */
4588	svm_clr_intercept(svm, bit: INTERCEPT_CR0_READ);
4589	svm_clr_intercept(svm, bit: INTERCEPT_CR4_READ);
4590	svm_clr_intercept(svm, bit: INTERCEPT_CR8_READ);
4591	svm_clr_intercept(svm, bit: INTERCEPT_CR0_WRITE);
4592	svm_clr_intercept(svm, bit: INTERCEPT_CR4_WRITE);
4593	svm_clr_intercept(svm, bit: INTERCEPT_CR8_WRITE);
4594
4595	svm_clr_intercept(svm, bit: INTERCEPT_SELECTIVE_CR0);
4596
4597	/* Track EFER/CR register changes */
4598	svm_set_intercept(svm, bit: TRAP_EFER_WRITE);
4599	svm_set_intercept(svm, bit: TRAP_CR0_WRITE);
4600	svm_set_intercept(svm, bit: TRAP_CR4_WRITE);
4601	svm_set_intercept(svm, bit: TRAP_CR8_WRITE);
4602
4603	vmcb->control.intercepts[INTERCEPT_DR] = 0;
4604	if (!sev_vcpu_has_debug_swap(svm)) {
4605	vmcb_set_intercept(control: &vmcb->control, bit: INTERCEPT_DR7_READ);
4606	vmcb_set_intercept(control: &vmcb->control, bit: INTERCEPT_DR7_WRITE);
4607	recalc_intercepts(svm);
4608	} else {
4609	/*
4610	* Disable #DB intercept iff DebugSwap is enabled. KVM doesn't
4611	* allow debugging SEV-ES guests, and enables DebugSwap iff
4612	* NO_NESTED_DATA_BP is supported, so there's no reason to
4613	* intercept #DB when DebugSwap is enabled. For simplicity
4614	* with respect to guest debug, intercept #DB for other VMs
4615	* even if NO_NESTED_DATA_BP is supported, i.e. even if the
4616	* guest can't DoS the CPU with infinite #DB vectoring.
4617	*/
4618	clr_exception_intercept(svm, DB_VECTOR);
4619	}
4620
4621	/* Can't intercept XSETBV, HV can't modify XCR0 directly */
4622	svm_clr_intercept(svm, bit: INTERCEPT_XSETBV);
4623
4624	/*
4625	* Set the GHCB MSR value as per the GHCB specification when emulating
4626	* vCPU RESET for an SEV-ES guest.
4627	*/
4628	if (!init_event)
4629	set_ghcb_msr(svm, GHCB_MSR_SEV_INFO((__u64)sev->ghcb_version,
4630	GHCB_VERSION_MIN,
4631	sev_enc_bit));
4632	}
4633
4634	void sev_init_vmcb(struct vcpu_svm *svm, bool init_event)
4635	{
4636	struct kvm_vcpu *vcpu = &svm->vcpu;
4637
4638	svm->vmcb->control.nested_ctl |= SVM_NESTED_CTL_SEV_ENABLE;
4639	clr_exception_intercept(svm, UD_VECTOR);
4640
4641	/*
4642	* Don't intercept #GP for SEV guests, e.g. for the VMware backdoor, as
4643	* KVM can't decrypt guest memory to decode the faulting instruction.
4644	*/
4645	clr_exception_intercept(svm, GP_VECTOR);
4646
4647	if (init_event && sev_snp_guest(kvm: vcpu->kvm))
4648	sev_snp_init_protected_guest_state(vcpu);
4649
4650	if (sev_es_guest(kvm: vcpu->kvm))
4651	sev_es_init_vmcb(svm, init_event);
4652	}
4653
4654	int sev_vcpu_create(struct kvm_vcpu *vcpu)
4655	{
4656	struct vcpu_svm *svm = to_svm(vcpu);
4657	struct page *vmsa_page;
4658
4659	mutex_init(&svm->sev_es.snp_vmsa_mutex);
4660
4661	if (!sev_es_guest(kvm: vcpu->kvm))
4662	return 0;
4663
4664	/*
4665	* SEV-ES guests require a separate (from the VMCB) VMSA page used to
4666	* contain the encrypted register state of the guest.
4667	*/
4668	vmsa_page = snp_safe_alloc_page();
4669	if (!vmsa_page)
4670	return -ENOMEM;
4671
4672	svm->sev_es.vmsa = page_address(vmsa_page);
4673
4674	vcpu->arch.guest_tsc_protected = snp_is_secure_tsc_enabled(kvm: vcpu->kvm);
4675
4676	return 0;
4677	}
4678
4679	void sev_es_prepare_switch_to_guest(struct vcpu_svm *svm, struct sev_es_save_area *hostsa)
4680	{
4681	struct kvm *kvm = svm->vcpu.kvm;
4682
4683	/*
4684	* All host state for SEV-ES guests is categorized into three swap types
4685	* based on how it is handled by hardware during a world switch:
4686	*
4687	* A: VMRUN: Host state saved in host save area
4688	* VMEXIT: Host state loaded from host save area
4689	*
4690	* B: VMRUN: Host state _NOT_ saved in host save area
4691	* VMEXIT: Host state loaded from host save area
4692	*
4693	* C: VMRUN: Host state _NOT_ saved in host save area
4694	* VMEXIT: Host state initialized to default(reset) values
4695	*
4696	* Manually save type-B state, i.e. state that is loaded by VMEXIT but
4697	* isn't saved by VMRUN, that isn't already saved by VMSAVE (performed
4698	* by common SVM code).
4699	*/
4700	hostsa->xcr0 = kvm_host.xcr0;
4701	hostsa->pkru = read_pkru();
4702	hostsa->xss = kvm_host.xss;
4703
4704	/*
4705	* If DebugSwap is enabled, debug registers are loaded but NOT saved by
4706	* the CPU (Type-B). If DebugSwap is disabled/unsupported, the CPU does
4707	* not save or load debug registers. Sadly, KVM can't prevent SNP
4708	* guests from lying about DebugSwap on secondary vCPUs, i.e. the
4709	* SEV_FEATURES provided at "AP Create" isn't guaranteed to match what
4710	* the guest has actually enabled (or not!) in the VMSA.
4711	*
4712	* If DebugSwap is *possible*, save the masks so that they're restored
4713	* if the guest enables DebugSwap. But for the DRs themselves, do NOT
4714	* rely on the CPU to restore the host values; KVM will restore them as
4715	* needed in common code, via hw_breakpoint_restore(). Note, KVM does
4716	* NOT support virtualizing Breakpoint Extensions, i.e. the mask MSRs
4717	* don't need to be restored per se, KVM just needs to ensure they are
4718	* loaded with the correct values *if* the CPU writes the MSRs.
4719	*/
4720	if (sev_vcpu_has_debug_swap(svm) ||
4721	(sev_snp_guest(kvm) && cpu_feature_enabled(X86_FEATURE_DEBUG_SWAP))) {
4722	hostsa->dr0_addr_mask = amd_get_dr_addr_mask(dr: 0);
4723	hostsa->dr1_addr_mask = amd_get_dr_addr_mask(dr: 1);
4724	hostsa->dr2_addr_mask = amd_get_dr_addr_mask(dr: 2);
4725	hostsa->dr3_addr_mask = amd_get_dr_addr_mask(dr: 3);
4726	}
4727
4728	/*
4729	* TSC_AUX is always virtualized for SEV-ES guests when the feature is
4730	* available, i.e. TSC_AUX is loaded on #VMEXIT from the host save area.
4731	* Set the save area to the current hardware value, i.e. the current
4732	* user return value, so that the correct value is restored on #VMEXIT.
4733	*/
4734	if (cpu_feature_enabled(X86_FEATURE_V_TSC_AUX) &&
4735	!WARN_ON_ONCE(tsc_aux_uret_slot < 0))
4736	hostsa->tsc_aux = kvm_get_user_return_msr(slot: tsc_aux_uret_slot);
4737	}
4738
4739	void sev_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector)
4740	{
4741	struct vcpu_svm *svm = to_svm(vcpu);
4742
4743	/* First SIPI: Use the values as initially set by the VMM */
4744	if (!svm->sev_es.received_first_sipi) {
4745	svm->sev_es.received_first_sipi = true;
4746	return;
4747	}
4748
4749	/* Subsequent SIPI */
4750	switch (svm->sev_es.ap_reset_hold_type) {
4751	case AP_RESET_HOLD_NAE_EVENT:
4752	/*
4753	* Return from an AP Reset Hold VMGEXIT, where the guest will
4754	* set the CS and RIP. Set SW_EXIT_INFO_2 to a non-zero value.
4755	*/
4756	svm_vmgexit_success(svm, data: 1);
4757	break;
4758	case AP_RESET_HOLD_MSR_PROTO:
4759	/*
4760	* Return from an AP Reset Hold VMGEXIT, where the guest will
4761	* set the CS and RIP. Set GHCB data field to a non-zero value.
4762	*/
4763	set_ghcb_msr_bits(svm, value: 1,
4764	GHCB_MSR_AP_RESET_HOLD_RESULT_MASK,
4765	GHCB_MSR_AP_RESET_HOLD_RESULT_POS);
4766
4767	set_ghcb_msr_bits(svm, GHCB_MSR_AP_RESET_HOLD_RESP,
4768	GHCB_MSR_INFO_MASK,
4769	GHCB_MSR_INFO_POS);
4770	break;
4771	default:
4772	break;
4773	}
4774	}
4775
4776	struct page *snp_safe_alloc_page_node(int node, gfp_t gfp)
4777	{
4778	unsigned long pfn;
4779	struct page *p;
4780
4781	if (!cc_platform_has(attr: CC_ATTR_HOST_SEV_SNP))
4782	return alloc_pages_node(node, gfp | __GFP_ZERO, 0);
4783
4784	/*
4785	* Allocate an SNP-safe page to workaround the SNP erratum where
4786	* the CPU will incorrectly signal an RMP violation #PF if a
4787	* hugepage (2MB or 1GB) collides with the RMP entry of a
4788	* 2MB-aligned VMCB, VMSA, or AVIC backing page.
4789	*
4790	* Allocate one extra page, choose a page which is not
4791	* 2MB-aligned, and free the other.
4792	*/
4793	p = alloc_pages_node(node, gfp | __GFP_ZERO, 1);
4794	if (!p)
4795	return NULL;
4796
4797	split_page(page: p, order: 1);
4798
4799	pfn = page_to_pfn(p);
4800	if (IS_ALIGNED(pfn, PTRS_PER_PMD))
4801	__free_page(p++);
4802	else
4803	__free_page(p + 1);
4804
4805	return p;
4806	}
4807
4808	void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code)
4809	{
4810	struct kvm_memory_slot *slot;
4811	struct kvm *kvm = vcpu->kvm;
4812	int order, rmp_level, ret;
4813	struct page *page;
4814	bool assigned;
4815	kvm_pfn_t pfn;
4816	gfn_t gfn;
4817
4818	gfn = gpa >> PAGE_SHIFT;
4819
4820	/*
4821	* The only time RMP faults occur for shared pages is when the guest is
4822	* triggering an RMP fault for an implicit page-state change from
4823	* shared->private. Implicit page-state changes are forwarded to
4824	* userspace via KVM_EXIT_MEMORY_FAULT events, however, so RMP faults
4825	* for shared pages should not end up here.
4826	*/
4827	if (!kvm_mem_is_private(kvm, gfn)) {
4828	pr_warn_ratelimited("SEV: Unexpected RMP fault for non-private GPA 0x%llx\n",
4829	gpa);
4830	return;
4831	}
4832
4833	slot = gfn_to_memslot(kvm, gfn);
4834	if (!kvm_slot_has_gmem(slot)) {
4835	pr_warn_ratelimited("SEV: Unexpected RMP fault, non-private slot for GPA 0x%llx\n",
4836	gpa);
4837	return;
4838	}
4839
4840	ret = kvm_gmem_get_pfn(kvm, slot, gfn, pfn: &pfn, page: &page, max_order: &order);
4841	if (ret) {
4842	pr_warn_ratelimited("SEV: Unexpected RMP fault, no backing page for private GPA 0x%llx\n",
4843	gpa);
4844	return;
4845	}
4846
4847	ret = snp_lookup_rmpentry(pfn, assigned: &assigned, level: &rmp_level);
4848	if (ret || !assigned) {
4849	pr_warn_ratelimited("SEV: Unexpected RMP fault, no assigned RMP entry found for GPA 0x%llx PFN 0x%llx error %d\n",
4850	gpa, pfn, ret);
4851	goto out_no_trace;
4852	}
4853
4854	/*
4855	* There are 2 cases where a PSMASH may be needed to resolve an #NPF
4856	* with PFERR_GUEST_RMP_BIT set:
4857	*
4858	* 1) RMPADJUST/PVALIDATE can trigger an #NPF with PFERR_GUEST_SIZEM
4859	* bit set if the guest issues them with a smaller granularity than
4860	* what is indicated by the page-size bit in the 2MB RMP entry for
4861	* the PFN that backs the GPA.
4862	*
4863	* 2) Guest access via NPT can trigger an #NPF if the NPT mapping is
4864	* smaller than what is indicated by the 2MB RMP entry for the PFN
4865	* that backs the GPA.
4866	*
4867	* In both these cases, the corresponding 2M RMP entry needs to
4868	* be PSMASH'd to 512 4K RMP entries. If the RMP entry is already
4869	* split into 4K RMP entries, then this is likely a spurious case which
4870	* can occur when there are concurrent accesses by the guest to a 2MB
4871	* GPA range that is backed by a 2MB-aligned PFN who's RMP entry is in
4872	* the process of being PMASH'd into 4K entries. These cases should
4873	* resolve automatically on subsequent accesses, so just ignore them
4874	* here.
4875	*/
4876	if (rmp_level == PG_LEVEL_4K)
4877	goto out;
4878
4879	ret = snp_rmptable_psmash(pfn);
4880	if (ret) {
4881	/*
4882	* Look it up again. If it's 4K now then the PSMASH may have
4883	* raced with another process and the issue has already resolved
4884	* itself.
4885	*/
4886	if (!snp_lookup_rmpentry(pfn, assigned: &assigned, level: &rmp_level) &&
4887	assigned && rmp_level == PG_LEVEL_4K)
4888	goto out;
4889
4890	pr_warn_ratelimited("SEV: Unable to split RMP entry for GPA 0x%llx PFN 0x%llx ret %d\n",
4891	gpa, pfn, ret);
4892	}
4893
4894	kvm_zap_gfn_range(kvm, gfn_start: gfn, gfn_end: gfn + PTRS_PER_PMD);
4895	out:
4896	trace_kvm_rmp_fault(vcpu, gpa, pfn, error_code, rmp_level, ret);
4897	out_no_trace:
4898	kvm_release_page_unused(page);
4899	}
4900
4901	static bool is_pfn_range_shared(kvm_pfn_t start, kvm_pfn_t end)
4902	{
4903	kvm_pfn_t pfn = start;
4904
4905	while (pfn < end) {
4906	int ret, rmp_level;
4907	bool assigned;
4908
4909	ret = snp_lookup_rmpentry(pfn, assigned: &assigned, level: &rmp_level);
4910	if (ret) {
4911	pr_warn_ratelimited("SEV: Failed to retrieve RMP entry: PFN 0x%llx GFN start 0x%llx GFN end 0x%llx RMP level %d error %d\n",
4912	pfn, start, end, rmp_level, ret);
4913	return false;
4914	}
4915
4916	if (assigned) {
4917	pr_debug("%s: overlap detected, PFN 0x%llx start 0x%llx end 0x%llx RMP level %d\n",
4918	__func__, pfn, start, end, rmp_level);
4919	return false;
4920	}
4921
4922	pfn++;
4923	}
4924
4925	return true;
4926	}
4927
4928	static u8 max_level_for_order(int order)
4929	{
4930	if (order >= KVM_HPAGE_GFN_SHIFT(PG_LEVEL_2M))
4931	return PG_LEVEL_2M;
4932
4933	return PG_LEVEL_4K;
4934	}
4935
4936	static bool is_large_rmp_possible(struct kvm *kvm, kvm_pfn_t pfn, int order)
4937	{
4938	kvm_pfn_t pfn_aligned = ALIGN_DOWN(pfn, PTRS_PER_PMD);
4939
4940	/*
4941	* If this is a large folio, and the entire 2M range containing the
4942	* PFN is currently shared, then the entire 2M-aligned range can be
4943	* set to private via a single 2M RMP entry.
4944	*/
4945	if (max_level_for_order(order) > PG_LEVEL_4K &&
4946	is_pfn_range_shared(start: pfn_aligned, end: pfn_aligned + PTRS_PER_PMD))
4947	return true;
4948
4949	return false;
4950	}
4951
4952	int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order)
4953	{
4954	struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
4955	kvm_pfn_t pfn_aligned;
4956	gfn_t gfn_aligned;
4957	int level, rc;
4958	bool assigned;
4959
4960	if (!sev_snp_guest(kvm))
4961	return 0;
4962
4963	rc = snp_lookup_rmpentry(pfn, assigned: &assigned, level: &level);
4964	if (rc) {
4965	pr_err_ratelimited("SEV: Failed to look up RMP entry: GFN %llx PFN %llx error %d\n",
4966	gfn, pfn, rc);
4967	return -ENOENT;
4968	}
4969
4970	if (assigned) {
4971	pr_debug("%s: already assigned: gfn %llx pfn %llx max_order %d level %d\n",
4972	__func__, gfn, pfn, max_order, level);
4973	return 0;
4974	}
4975
4976	if (is_large_rmp_possible(kvm, pfn, order: max_order)) {
4977	level = PG_LEVEL_2M;
4978	pfn_aligned = ALIGN_DOWN(pfn, PTRS_PER_PMD);
4979	gfn_aligned = ALIGN_DOWN(gfn, PTRS_PER_PMD);
4980	} else {
4981	level = PG_LEVEL_4K;
4982	pfn_aligned = pfn;
4983	gfn_aligned = gfn;
4984	}
4985
4986	rc = rmp_make_private(pfn: pfn_aligned, gpa: gfn_to_gpa(gfn: gfn_aligned), level, asid: sev->asid, immutable: false);
4987	if (rc) {
4988	pr_err_ratelimited("SEV: Failed to update RMP entry: GFN %llx PFN %llx level %d error %d\n",
4989	gfn, pfn, level, rc);
4990	return -EINVAL;
4991	}
4992
4993	pr_debug("%s: updated: gfn %llx pfn %llx pfn_aligned %llx max_order %d level %d\n",
4994	__func__, gfn, pfn, pfn_aligned, max_order, level);
4995
4996	return 0;
4997	}
4998
4999	void sev_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end)
5000	{
5001	kvm_pfn_t pfn;
5002
5003	if (!cc_platform_has(attr: CC_ATTR_HOST_SEV_SNP))
5004	return;
5005
5006	pr_debug("%s: PFN start 0x%llx PFN end 0x%llx\n", __func__, start, end);
5007
5008	for (pfn = start; pfn < end;) {
5009	bool use_2m_update = false;
5010	int rc, rmp_level;
5011	bool assigned;
5012
5013	rc = snp_lookup_rmpentry(pfn, assigned: &assigned, level: &rmp_level);
5014	if (rc || !assigned)
5015	goto next_pfn;
5016
5017	use_2m_update = IS_ALIGNED(pfn, PTRS_PER_PMD) &&
5018	end >= (pfn + PTRS_PER_PMD) &&
5019	rmp_level > PG_LEVEL_4K;
5020
5021	/*
5022	* If an unaligned PFN corresponds to a 2M region assigned as a
5023	* large page in the RMP table, PSMASH the region into individual
5024	* 4K RMP entries before attempting to convert a 4K sub-page.
5025	*/
5026	if (!use_2m_update && rmp_level > PG_LEVEL_4K) {
5027	/*
5028	* This shouldn't fail, but if it does, report it, but
5029	* still try to update RMP entry to shared and pray this
5030	* was a spurious error that can be addressed later.
5031	*/
5032	rc = snp_rmptable_psmash(pfn);
5033	WARN_ONCE(rc, "SEV: Failed to PSMASH RMP entry for PFN 0x%llx error %d\n",
5034	pfn, rc);
5035	}
5036
5037	rc = rmp_make_shared(pfn, level: use_2m_update ? PG_LEVEL_2M : PG_LEVEL_4K);
5038	if (WARN_ONCE(rc, "SEV: Failed to update RMP entry for PFN 0x%llx error %d\n",
5039	pfn, rc))
5040	goto next_pfn;
5041
5042	/*
5043	* SEV-ES avoids host/guest cache coherency issues through
5044	* WBNOINVD hooks issued via MMU notifiers during run-time, and
5045	* KVM's VM destroy path at shutdown. Those MMU notifier events
5046	* don't cover gmem since there is no requirement to map pages
5047	* to a HVA in order to use them for a running guest. While the
5048	* shutdown path would still likely cover things for SNP guests,
5049	* userspace may also free gmem pages during run-time via
5050	* hole-punching operations on the guest_memfd, so flush the
5051	* cache entries for these pages before free'ing them back to
5052	* the host.
5053	*/
5054	clflush_cache_range(__va(pfn_to_hpa(pfn)),
5055	size: use_2m_update ? PMD_SIZE : PAGE_SIZE);
5056	next_pfn:
5057	pfn += use_2m_update ? PTRS_PER_PMD : 1;
5058	cond_resched();
5059	}
5060	}
5061
5062	int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private)
5063	{
5064	int level, rc;
5065	bool assigned;
5066
5067	if (!sev_snp_guest(kvm))
5068	return 0;
5069
5070	rc = snp_lookup_rmpentry(pfn, assigned: &assigned, level: &level);
5071	if (rc || !assigned)
5072	return PG_LEVEL_4K;
5073
5074	return level;
5075	}
5076
5077	struct vmcb_save_area *sev_decrypt_vmsa(struct kvm_vcpu *vcpu)
5078	{
5079	struct vcpu_svm *svm = to_svm(vcpu);
5080	struct vmcb_save_area *vmsa;
5081	struct kvm_sev_info *sev;
5082	int error = 0;
5083	int ret;
5084
5085	if (!sev_es_guest(kvm: vcpu->kvm))
5086	return NULL;
5087
5088	/*
5089	* If the VMSA has not yet been encrypted, return a pointer to the
5090	* current un-encrypted VMSA.
5091	*/
5092	if (!vcpu->arch.guest_state_protected)
5093	return (struct vmcb_save_area *)svm->sev_es.vmsa;
5094
5095	sev = to_kvm_sev_info(kvm: vcpu->kvm);
5096
5097	/* Check if the SEV policy allows debugging */
5098	if (sev_snp_guest(kvm: vcpu->kvm)) {
5099	if (!(sev->policy & SNP_POLICY_MASK_DEBUG))
5100	return NULL;
5101	} else {
5102	if (sev->policy & SEV_POLICY_MASK_NODBG)
5103	return NULL;
5104	}
5105
5106	if (sev_snp_guest(kvm: vcpu->kvm)) {
5107	struct sev_data_snp_dbg dbg = {0};
5108
5109	vmsa = snp_alloc_firmware_page(__GFP_ZERO);
5110	if (!vmsa)
5111	return NULL;
5112
5113	dbg.gctx_paddr = __psp_pa(sev->snp_context);
5114	dbg.src_addr = svm->vmcb->control.vmsa_pa;
5115	dbg.dst_addr = __psp_pa(vmsa);
5116
5117	ret = sev_do_cmd(cmd: SEV_CMD_SNP_DBG_DECRYPT, data: &dbg, psp_ret: &error);
5118
5119	/*
5120	* Return the target page to a hypervisor page no matter what.
5121	* If this fails, the page can't be used, so leak it and don't
5122	* try to use it.
5123	*/
5124	if (snp_page_reclaim(kvm: vcpu->kvm, PHYS_PFN(__pa(vmsa))))
5125	return NULL;
5126
5127	if (ret) {
5128	pr_err("SEV: SNP_DBG_DECRYPT failed ret=%d, fw_error=%d (%#x)\n",
5129	ret, error, error);
5130	free_page((unsigned long)vmsa);
5131
5132	return NULL;
5133	}
5134	} else {
5135	struct sev_data_dbg dbg = {0};
5136	struct page *vmsa_page;
5137
5138	vmsa_page = alloc_page(GFP_KERNEL);
5139	if (!vmsa_page)
5140	return NULL;
5141
5142	vmsa = page_address(vmsa_page);
5143
5144	dbg.handle = sev->handle;
5145	dbg.src_addr = svm->vmcb->control.vmsa_pa;
5146	dbg.dst_addr = __psp_pa(vmsa);
5147	dbg.len = PAGE_SIZE;
5148
5149	ret = sev_do_cmd(cmd: SEV_CMD_DBG_DECRYPT, data: &dbg, psp_ret: &error);
5150	if (ret) {
5151	pr_err("SEV: SEV_CMD_DBG_DECRYPT failed ret=%d, fw_error=%d (0x%x)\n",
5152	ret, error, error);
5153	__free_page(vmsa_page);
5154
5155	return NULL;
5156	}
5157	}
5158
5159	return vmsa;
5160	}
5161
5162	void sev_free_decrypted_vmsa(struct kvm_vcpu *vcpu, struct vmcb_save_area *vmsa)
5163	{
5164	/* If the VMSA has not yet been encrypted, nothing was allocated */
5165	if (!vcpu->arch.guest_state_protected || !vmsa)
5166	return;
5167
5168	free_page((unsigned long)vmsa);
5169	}
5170