1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* hosting IBM Z kernel virtual machines (s390x)
4	*
5	* Copyright IBM Corp. 2008, 2020
6	*
7	* Author(s): Carsten Otte <cotte@de.ibm.com>
8	* Christian Borntraeger <borntraeger@de.ibm.com>
9	* Christian Ehrhardt <ehrhardt@de.ibm.com>
10	* Jason J. Herne <jjherne@us.ibm.com>
11	*/
12
13	#define pr_fmt(fmt) "kvm-s390: " fmt
14
15	#include <linux/compiler.h>
16	#include <linux/entry-virt.h>
17	#include <linux/export.h>
18	#include <linux/err.h>
19	#include <linux/fs.h>
20	#include <linux/hrtimer.h>
21	#include <linux/init.h>
22	#include <linux/kvm.h>
23	#include <linux/kvm_host.h>
24	#include <linux/mman.h>
25	#include <linux/module.h>
26	#include <linux/moduleparam.h>
27	#include <linux/cpufeature.h>
28	#include <linux/random.h>
29	#include <linux/slab.h>
30	#include <linux/timer.h>
31	#include <linux/vmalloc.h>
32	#include <linux/bitmap.h>
33	#include <linux/sched/signal.h>
34	#include <linux/string.h>
35	#include <linux/pgtable.h>
36	#include <linux/mmu_notifier.h>
37
38	#include <asm/access-regs.h>
39	#include <asm/asm-offsets.h>
40	#include <asm/lowcore.h>
41	#include <asm/machine.h>
42	#include <asm/stp.h>
43	#include <asm/gmap.h>
44	#include <asm/gmap_helpers.h>
45	#include <asm/nmi.h>
46	#include <asm/isc.h>
47	#include <asm/sclp.h>
48	#include <asm/cpacf.h>
49	#include <asm/timex.h>
50	#include <asm/asm.h>
51	#include <asm/fpu.h>
52	#include <asm/ap.h>
53	#include <asm/uv.h>
54	#include "kvm-s390.h"
55	#include "gaccess.h"
56	#include "pci.h"
57
58	#define CREATE_TRACE_POINTS
59	#include "trace.h"
60	#include "trace-s390.h"
61
62	#define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
63	#define LOCAL_IRQS 32
64	#define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
65	(KVM_MAX_VCPUS + LOCAL_IRQS))
66
67	const struct _kvm_stats_desc kvm_vm_stats_desc[] = {
68	KVM_GENERIC_VM_STATS(),
69	STATS_DESC_COUNTER(VM, inject_io),
70	STATS_DESC_COUNTER(VM, inject_float_mchk),
71	STATS_DESC_COUNTER(VM, inject_pfault_done),
72	STATS_DESC_COUNTER(VM, inject_service_signal),
73	STATS_DESC_COUNTER(VM, inject_virtio),
74	STATS_DESC_COUNTER(VM, aen_forward),
75	STATS_DESC_COUNTER(VM, gmap_shadow_reuse),
76	STATS_DESC_COUNTER(VM, gmap_shadow_create),
77	STATS_DESC_COUNTER(VM, gmap_shadow_r1_entry),
78	STATS_DESC_COUNTER(VM, gmap_shadow_r2_entry),
79	STATS_DESC_COUNTER(VM, gmap_shadow_r3_entry),
80	STATS_DESC_COUNTER(VM, gmap_shadow_sg_entry),
81	STATS_DESC_COUNTER(VM, gmap_shadow_pg_entry),
82	};
83
84	const struct kvm_stats_header kvm_vm_stats_header = {
85	.name_size = KVM_STATS_NAME_SIZE,
86	.num_desc = ARRAY_SIZE(kvm_vm_stats_desc),
87	.id_offset = sizeof(struct kvm_stats_header),
88	.desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
89	.data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
90	sizeof(kvm_vm_stats_desc),
91	};
92
93	const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = {
94	KVM_GENERIC_VCPU_STATS(),
95	STATS_DESC_COUNTER(VCPU, exit_userspace),
96	STATS_DESC_COUNTER(VCPU, exit_null),
97	STATS_DESC_COUNTER(VCPU, exit_external_request),
98	STATS_DESC_COUNTER(VCPU, exit_io_request),
99	STATS_DESC_COUNTER(VCPU, exit_external_interrupt),
100	STATS_DESC_COUNTER(VCPU, exit_stop_request),
101	STATS_DESC_COUNTER(VCPU, exit_validity),
102	STATS_DESC_COUNTER(VCPU, exit_instruction),
103	STATS_DESC_COUNTER(VCPU, exit_pei),
104	STATS_DESC_COUNTER(VCPU, halt_no_poll_steal),
105	STATS_DESC_COUNTER(VCPU, instruction_lctl),
106	STATS_DESC_COUNTER(VCPU, instruction_lctlg),
107	STATS_DESC_COUNTER(VCPU, instruction_stctl),
108	STATS_DESC_COUNTER(VCPU, instruction_stctg),
109	STATS_DESC_COUNTER(VCPU, exit_program_interruption),
110	STATS_DESC_COUNTER(VCPU, exit_instr_and_program),
111	STATS_DESC_COUNTER(VCPU, exit_operation_exception),
112	STATS_DESC_COUNTER(VCPU, deliver_ckc),
113	STATS_DESC_COUNTER(VCPU, deliver_cputm),
114	STATS_DESC_COUNTER(VCPU, deliver_external_call),
115	STATS_DESC_COUNTER(VCPU, deliver_emergency_signal),
116	STATS_DESC_COUNTER(VCPU, deliver_service_signal),
117	STATS_DESC_COUNTER(VCPU, deliver_virtio),
118	STATS_DESC_COUNTER(VCPU, deliver_stop_signal),
119	STATS_DESC_COUNTER(VCPU, deliver_prefix_signal),
120	STATS_DESC_COUNTER(VCPU, deliver_restart_signal),
121	STATS_DESC_COUNTER(VCPU, deliver_program),
122	STATS_DESC_COUNTER(VCPU, deliver_io),
123	STATS_DESC_COUNTER(VCPU, deliver_machine_check),
124	STATS_DESC_COUNTER(VCPU, exit_wait_state),
125	STATS_DESC_COUNTER(VCPU, inject_ckc),
126	STATS_DESC_COUNTER(VCPU, inject_cputm),
127	STATS_DESC_COUNTER(VCPU, inject_external_call),
128	STATS_DESC_COUNTER(VCPU, inject_emergency_signal),
129	STATS_DESC_COUNTER(VCPU, inject_mchk),
130	STATS_DESC_COUNTER(VCPU, inject_pfault_init),
131	STATS_DESC_COUNTER(VCPU, inject_program),
132	STATS_DESC_COUNTER(VCPU, inject_restart),
133	STATS_DESC_COUNTER(VCPU, inject_set_prefix),
134	STATS_DESC_COUNTER(VCPU, inject_stop_signal),
135	STATS_DESC_COUNTER(VCPU, instruction_epsw),
136	STATS_DESC_COUNTER(VCPU, instruction_gs),
137	STATS_DESC_COUNTER(VCPU, instruction_io_other),
138	STATS_DESC_COUNTER(VCPU, instruction_lpsw),
139	STATS_DESC_COUNTER(VCPU, instruction_lpswe),
140	STATS_DESC_COUNTER(VCPU, instruction_lpswey),
141	STATS_DESC_COUNTER(VCPU, instruction_pfmf),
142	STATS_DESC_COUNTER(VCPU, instruction_ptff),
143	STATS_DESC_COUNTER(VCPU, instruction_sck),
144	STATS_DESC_COUNTER(VCPU, instruction_sckpf),
145	STATS_DESC_COUNTER(VCPU, instruction_stidp),
146	STATS_DESC_COUNTER(VCPU, instruction_spx),
147	STATS_DESC_COUNTER(VCPU, instruction_stpx),
148	STATS_DESC_COUNTER(VCPU, instruction_stap),
149	STATS_DESC_COUNTER(VCPU, instruction_iske),
150	STATS_DESC_COUNTER(VCPU, instruction_ri),
151	STATS_DESC_COUNTER(VCPU, instruction_rrbe),
152	STATS_DESC_COUNTER(VCPU, instruction_sske),
153	STATS_DESC_COUNTER(VCPU, instruction_ipte_interlock),
154	STATS_DESC_COUNTER(VCPU, instruction_stsi),
155	STATS_DESC_COUNTER(VCPU, instruction_stfl),
156	STATS_DESC_COUNTER(VCPU, instruction_tb),
157	STATS_DESC_COUNTER(VCPU, instruction_tpi),
158	STATS_DESC_COUNTER(VCPU, instruction_tprot),
159	STATS_DESC_COUNTER(VCPU, instruction_tsch),
160	STATS_DESC_COUNTER(VCPU, instruction_sie),
161	STATS_DESC_COUNTER(VCPU, instruction_essa),
162	STATS_DESC_COUNTER(VCPU, instruction_sthyi),
163	STATS_DESC_COUNTER(VCPU, instruction_sigp_sense),
164	STATS_DESC_COUNTER(VCPU, instruction_sigp_sense_running),
165	STATS_DESC_COUNTER(VCPU, instruction_sigp_external_call),
166	STATS_DESC_COUNTER(VCPU, instruction_sigp_emergency),
167	STATS_DESC_COUNTER(VCPU, instruction_sigp_cond_emergency),
168	STATS_DESC_COUNTER(VCPU, instruction_sigp_start),
169	STATS_DESC_COUNTER(VCPU, instruction_sigp_stop),
170	STATS_DESC_COUNTER(VCPU, instruction_sigp_stop_store_status),
171	STATS_DESC_COUNTER(VCPU, instruction_sigp_store_status),
172	STATS_DESC_COUNTER(VCPU, instruction_sigp_store_adtl_status),
173	STATS_DESC_COUNTER(VCPU, instruction_sigp_arch),
174	STATS_DESC_COUNTER(VCPU, instruction_sigp_prefix),
175	STATS_DESC_COUNTER(VCPU, instruction_sigp_restart),
176	STATS_DESC_COUNTER(VCPU, instruction_sigp_init_cpu_reset),
177	STATS_DESC_COUNTER(VCPU, instruction_sigp_cpu_reset),
178	STATS_DESC_COUNTER(VCPU, instruction_sigp_unknown),
179	STATS_DESC_COUNTER(VCPU, instruction_diagnose_10),
180	STATS_DESC_COUNTER(VCPU, instruction_diagnose_44),
181	STATS_DESC_COUNTER(VCPU, instruction_diagnose_9c),
182	STATS_DESC_COUNTER(VCPU, diag_9c_ignored),
183	STATS_DESC_COUNTER(VCPU, diag_9c_forward),
184	STATS_DESC_COUNTER(VCPU, instruction_diagnose_258),
185	STATS_DESC_COUNTER(VCPU, instruction_diagnose_308),
186	STATS_DESC_COUNTER(VCPU, instruction_diagnose_500),
187	STATS_DESC_COUNTER(VCPU, instruction_diagnose_other),
188	STATS_DESC_COUNTER(VCPU, pfault_sync),
189	STATS_DESC_COUNTER(VCPU, signal_exits)
190	};
191
192	const struct kvm_stats_header kvm_vcpu_stats_header = {
193	.name_size = KVM_STATS_NAME_SIZE,
194	.num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc),
195	.id_offset = sizeof(struct kvm_stats_header),
196	.desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
197	.data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
198	sizeof(kvm_vcpu_stats_desc),
199	};
200
201	/* allow nested virtualization in KVM (if enabled by user space) */
202	static int nested;
203	module_param(nested, int, S_IRUGO);
204	MODULE_PARM_DESC(nested, "Nested virtualization support");
205
206	/* allow 1m huge page guest backing, if !nested */
207	static int hpage;
208	module_param(hpage, int, 0444);
209	MODULE_PARM_DESC(hpage, "1m huge page backing support");
210
211	/* maximum percentage of steal time for polling. >100 is treated like 100 */
212	static u8 halt_poll_max_steal = 10;
213	module_param(halt_poll_max_steal, byte, 0644);
214	MODULE_PARM_DESC(halt_poll_max_steal, "Maximum percentage of steal time to allow polling");
215
216	/* if set to true, the GISA will be initialized and used if available */
217	static bool use_gisa = true;
218	module_param(use_gisa, bool, 0644);
219	MODULE_PARM_DESC(use_gisa, "Use the GISA if the host supports it.");
220
221	/* maximum diag9c forwarding per second */
222	unsigned int diag9c_forwarding_hz;
223	module_param(diag9c_forwarding_hz, uint, 0644);
224	MODULE_PARM_DESC(diag9c_forwarding_hz, "Maximum diag9c forwarding per second, 0 to turn off");
225
226	/*
227	* allow asynchronous deinit for protected guests; enable by default since
228	* the feature is opt-in anyway
229	*/
230	static int async_destroy = 1;
231	module_param(async_destroy, int, 0444);
232	MODULE_PARM_DESC(async_destroy, "Asynchronous destroy for protected guests");
233
234	/*
235	* For now we handle at most 16 double words as this is what the s390 base
236	* kernel handles and stores in the prefix page. If we ever need to go beyond
237	* this, this requires changes to code, but the external uapi can stay.
238	*/
239	#define SIZE_INTERNAL 16
240
241	/*
242	* Base feature mask that defines default mask for facilities. Consists of the
243	* defines in FACILITIES_KVM and the non-hypervisor managed bits.
244	*/
245	static unsigned long kvm_s390_fac_base[SIZE_INTERNAL] = { FACILITIES_KVM };
246	/*
247	* Extended feature mask. Consists of the defines in FACILITIES_KVM_CPUMODEL
248	* and defines the facilities that can be enabled via a cpu model.
249	*/
250	static unsigned long kvm_s390_fac_ext[SIZE_INTERNAL] = { FACILITIES_KVM_CPUMODEL };
251
252	static unsigned long kvm_s390_fac_size(void)
253	{
254	BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_MASK_SIZE_U64);
255	BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_LIST_SIZE_U64);
256	BUILD_BUG_ON(SIZE_INTERNAL * sizeof(unsigned long) >
257	sizeof(stfle_fac_list));
258
259	return SIZE_INTERNAL;
260	}
261
262	/* available cpu features supported by kvm */
263	static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
264	/* available subfunctions indicated via query / "test bit" */
265	static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
266
267	static struct gmap_notifier gmap_notifier;
268	static struct gmap_notifier vsie_gmap_notifier;
269	debug_info_t *kvm_s390_dbf;
270	debug_info_t *kvm_s390_dbf_uv;
271
272	/* Section: not file related */
273	/* forward declarations */
274	static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
275	unsigned long end);
276
277	static void kvm_clock_sync_scb(struct kvm_s390_sie_block *scb, u64 delta)
278	{
279	u8 delta_idx = 0;
280
281	/*
282	* The TOD jumps by delta, we have to compensate this by adding
283	* -delta to the epoch.
284	*/
285	delta = -delta;
286
287	/* sign-extension - we're adding to signed values below */
288	if ((s64)delta < 0)
289	delta_idx = -1;
290
291	scb->epoch += delta;
292	if (scb->ecd & ECD_MEF) {
293	scb->epdx += delta_idx;
294	if (scb->epoch < delta)
295	scb->epdx += 1;
296	}
297	}
298
299	/*
300	* This callback is executed during stop_machine(). All CPUs are therefore
301	* temporarily stopped. In order not to change guest behavior, we have to
302	* disable preemption whenever we touch the epoch of kvm and the VCPUs,
303	* so a CPU won't be stopped while calculating with the epoch.
304	*/
305	static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
306	void *v)
307	{
308	struct kvm *kvm;
309	struct kvm_vcpu *vcpu;
310	unsigned long i;
311	unsigned long long *delta = v;
312
313	list_for_each_entry(kvm, &vm_list, vm_list) {
314	kvm_for_each_vcpu(i, vcpu, kvm) {
315	kvm_clock_sync_scb(scb: vcpu->arch.sie_block, delta: *delta);
316	if (i == 0) {
317	kvm->arch.epoch = vcpu->arch.sie_block->epoch;
318	kvm->arch.epdx = vcpu->arch.sie_block->epdx;
319	}
320	if (vcpu->arch.cputm_enabled)
321	vcpu->arch.cputm_start += *delta;
322	if (vcpu->arch.vsie_block)
323	kvm_clock_sync_scb(scb: vcpu->arch.vsie_block,
324	delta: *delta);
325	}
326	}
327	return NOTIFY_OK;
328	}
329
330	static struct notifier_block kvm_clock_notifier = {
331	.notifier_call = kvm_clock_sync,
332	};
333
334	static void allow_cpu_feat(unsigned long nr)
335	{
336	set_bit_inv(nr, kvm_s390_available_cpu_feat);
337	}
338
339	static inline int plo_test_bit(unsigned char nr)
340	{
341	unsigned long function = (unsigned long)nr | 0x100;
342	int cc;
343
344	asm volatile(
345	" lgr 0,%[function]\n"
346	/* Parameter registers are ignored for "test bit" */
347	" plo 0,0,0,0(0)\n"
348	CC_IPM(cc)
349	: CC_OUT(cc, cc)
350	: [function] "d" (function)
351	: CC_CLOBBER_LIST("0"));
352	return CC_TRANSFORM(cc) == 0;
353	}
354
355	static __always_inline void pfcr_query(u8 (*query)[16])
356	{
357	asm volatile(
358	" lghi 0,0\n"
359	" .insn rsy,0xeb0000000016,0,0,%[query]"
360	: [query] "=QS" (*query)
361	:
362	: "cc", "0");
363	}
364
365	static __always_inline void __sortl_query(u8 (*query)[32])
366	{
367	asm volatile(
368	" lghi 0,0\n"
369	" la 1,%[query]\n"
370	/* Parameter registers are ignored */
371	" .insn rre,0xb9380000,2,4"
372	: [query] "=R" (*query)
373	:
374	: "cc", "0", "1");
375	}
376
377	static __always_inline void __dfltcc_query(u8 (*query)[32])
378	{
379	asm volatile(
380	" lghi 0,0\n"
381	" la 1,%[query]\n"
382	/* Parameter registers are ignored */
383	" .insn rrf,0xb9390000,2,4,6,0"
384	: [query] "=R" (*query)
385	:
386	: "cc", "0", "1");
387	}
388
389	static void __init kvm_s390_cpu_feat_init(void)
390	{
391	int i;
392
393	for (i = 0; i < 256; ++i) {
394	if (plo_test_bit(nr: i))
395	kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
396	}
397
398	if (test_facility(28)) /* TOD-clock steering */
399	ptff(kvm_s390_available_subfunc.ptff,
400	sizeof(kvm_s390_available_subfunc.ptff),
401	PTFF_QAF);
402
403	if (test_facility(17)) { /* MSA */
404	__cpacf_query(CPACF_KMAC, (cpacf_mask_t *)
405	kvm_s390_available_subfunc.kmac);
406	__cpacf_query(CPACF_KMC, (cpacf_mask_t *)
407	kvm_s390_available_subfunc.kmc);
408	__cpacf_query(CPACF_KM, (cpacf_mask_t *)
409	kvm_s390_available_subfunc.km);
410	__cpacf_query(CPACF_KIMD, (cpacf_mask_t *)
411	kvm_s390_available_subfunc.kimd);
412	__cpacf_query(CPACF_KLMD, (cpacf_mask_t *)
413	kvm_s390_available_subfunc.klmd);
414	}
415	if (test_facility(76)) /* MSA3 */
416	__cpacf_query(CPACF_PCKMO, (cpacf_mask_t *)
417	kvm_s390_available_subfunc.pckmo);
418	if (test_facility(77)) { /* MSA4 */
419	__cpacf_query(CPACF_KMCTR, (cpacf_mask_t *)
420	kvm_s390_available_subfunc.kmctr);
421	__cpacf_query(CPACF_KMF, (cpacf_mask_t *)
422	kvm_s390_available_subfunc.kmf);
423	__cpacf_query(CPACF_KMO, (cpacf_mask_t *)
424	kvm_s390_available_subfunc.kmo);
425	__cpacf_query(CPACF_PCC, (cpacf_mask_t *)
426	kvm_s390_available_subfunc.pcc);
427	}
428	if (test_facility(57)) /* MSA5 */
429	__cpacf_query(CPACF_PRNO, (cpacf_mask_t *)
430	kvm_s390_available_subfunc.ppno);
431
432	if (test_facility(146)) /* MSA8 */
433	__cpacf_query(CPACF_KMA, (cpacf_mask_t *)
434	kvm_s390_available_subfunc.kma);
435
436	if (test_facility(155)) /* MSA9 */
437	__cpacf_query(CPACF_KDSA, (cpacf_mask_t *)
438	kvm_s390_available_subfunc.kdsa);
439
440	if (test_facility(150)) /* SORTL */
441	__sortl_query(query: &kvm_s390_available_subfunc.sortl);
442
443	if (test_facility(151)) /* DFLTCC */
444	__dfltcc_query(query: &kvm_s390_available_subfunc.dfltcc);
445
446	if (test_facility(201)) /* PFCR */
447	pfcr_query(query: &kvm_s390_available_subfunc.pfcr);
448
449	if (machine_has_esop())
450	allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
451	/*
452	* We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
453	* 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
454	*/
455	if (!sclp.has_sief2 || !machine_has_esop() || !sclp.has_64bscao ||
456	!test_facility(3) || !nested)
457	return;
458	allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2);
459	if (sclp.has_64bscao)
460	allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO);
461	if (sclp.has_siif)
462	allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF);
463	if (sclp.has_gpere)
464	allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE);
465	if (sclp.has_gsls)
466	allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS);
467	if (sclp.has_ib)
468	allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB);
469	if (sclp.has_cei)
470	allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI);
471	if (sclp.has_ibs)
472	allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS);
473	if (sclp.has_kss)
474	allow_cpu_feat(KVM_S390_VM_CPU_FEAT_KSS);
475	/*
476	* KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make
477	* all skey handling functions read/set the skey from the PGSTE
478	* instead of the real storage key.
479	*
480	* KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make
481	* pages being detected as preserved although they are resident.
482	*
483	* KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will
484	* have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY.
485	*
486	* For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and
487	* KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be
488	* correctly shadowed. We can do that for the PGSTE but not for PTE.I.
489	*
490	* KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We
491	* cannot easily shadow the SCA because of the ipte lock.
492	*/
493	}
494
495	static int __init __kvm_s390_init(void)
496	{
497	int rc = -ENOMEM;
498
499	kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
500	if (!kvm_s390_dbf)
501	return -ENOMEM;
502
503	kvm_s390_dbf_uv = debug_register("kvm-uv", 32, 1, 7 * sizeof(long));
504	if (!kvm_s390_dbf_uv)
505	goto err_kvm_uv;
506
507	if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view) ||
508	debug_register_view(kvm_s390_dbf_uv, &debug_sprintf_view))
509	goto err_debug_view;
510
511	kvm_s390_cpu_feat_init();
512
513	/* Register floating interrupt controller interface. */
514	rc = kvm_register_device_ops(ops: &kvm_flic_ops, KVM_DEV_TYPE_FLIC);
515	if (rc) {
516	pr_err("A FLIC registration call failed with rc=%d\n", rc);
517	goto err_flic;
518	}
519
520	if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM)) {
521	rc = kvm_s390_pci_init();
522	if (rc) {
523	pr_err("Unable to allocate AIFT for PCI\n");
524	goto err_pci;
525	}
526	}
527
528	rc = kvm_s390_gib_init(GAL_ISC);
529	if (rc)
530	goto err_gib;
531
532	gmap_notifier.notifier_call = kvm_gmap_notifier;
533	gmap_register_pte_notifier(&gmap_notifier);
534	vsie_gmap_notifier.notifier_call = kvm_s390_vsie_gmap_notifier;
535	gmap_register_pte_notifier(&vsie_gmap_notifier);
536	atomic_notifier_chain_register(&s390_epoch_delta_notifier,
537	&kvm_clock_notifier);
538
539	return 0;
540
541	err_gib:
542	if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM))
543	kvm_s390_pci_exit();
544	err_pci:
545	err_flic:
546	err_debug_view:
547	debug_unregister(kvm_s390_dbf_uv);
548	err_kvm_uv:
549	debug_unregister(kvm_s390_dbf);
550	return rc;
551	}
552
553	static void __kvm_s390_exit(void)
554	{
555	gmap_unregister_pte_notifier(&gmap_notifier);
556	gmap_unregister_pte_notifier(&vsie_gmap_notifier);
557	atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
558	&kvm_clock_notifier);
559
560	kvm_s390_gib_destroy();
561	if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM))
562	kvm_s390_pci_exit();
563	debug_unregister(kvm_s390_dbf);
564	debug_unregister(kvm_s390_dbf_uv);
565	}
566
567	/* Section: device related */
568	long kvm_arch_dev_ioctl(struct file *filp,
569	unsigned int ioctl, unsigned long arg)
570	{
571	if (ioctl == KVM_S390_ENABLE_SIE)
572	return s390_enable_sie();
573	return -EINVAL;
574	}
575
576	int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
577	{
578	int r;
579
580	switch (ext) {
581	case KVM_CAP_S390_PSW:
582	case KVM_CAP_S390_GMAP:
583	case KVM_CAP_SYNC_MMU:
584	#ifdef CONFIG_KVM_S390_UCONTROL
585	case KVM_CAP_S390_UCONTROL:
586	#endif
587	case KVM_CAP_ASYNC_PF:
588	case KVM_CAP_SYNC_REGS:
589	case KVM_CAP_ONE_REG:
590	case KVM_CAP_ENABLE_CAP:
591	case KVM_CAP_S390_CSS_SUPPORT:
592	case KVM_CAP_IOEVENTFD:
593	case KVM_CAP_S390_IRQCHIP:
594	case KVM_CAP_VM_ATTRIBUTES:
595	case KVM_CAP_MP_STATE:
596	case KVM_CAP_IMMEDIATE_EXIT:
597	case KVM_CAP_S390_INJECT_IRQ:
598	case KVM_CAP_S390_USER_SIGP:
599	case KVM_CAP_S390_USER_STSI:
600	case KVM_CAP_S390_SKEYS:
601	case KVM_CAP_S390_IRQ_STATE:
602	case KVM_CAP_S390_USER_INSTR0:
603	case KVM_CAP_S390_CMMA_MIGRATION:
604	case KVM_CAP_S390_AIS:
605	case KVM_CAP_S390_AIS_MIGRATION:
606	case KVM_CAP_S390_VCPU_RESETS:
607	case KVM_CAP_SET_GUEST_DEBUG:
608	case KVM_CAP_S390_DIAG318:
609	case KVM_CAP_IRQFD_RESAMPLE:
610	case KVM_CAP_S390_USER_OPEREXEC:
611	r = 1;
612	break;
613	case KVM_CAP_SET_GUEST_DEBUG2:
614	r = KVM_GUESTDBG_VALID_MASK;
615	break;
616	case KVM_CAP_S390_HPAGE_1M:
617	r = 0;
618	if (hpage && !(kvm && kvm_is_ucontrol(kvm)))
619	r = 1;
620	break;
621	case KVM_CAP_S390_MEM_OP:
622	r = MEM_OP_MAX_SIZE;
623	break;
624	case KVM_CAP_S390_MEM_OP_EXTENSION:
625	/*
626	* Flag bits indicating which extensions are supported.
627	* If r > 0, the base extension must also be supported/indicated,
628	* in order to maintain backwards compatibility.
629	*/
630	r = KVM_S390_MEMOP_EXTENSION_CAP_BASE |
631	KVM_S390_MEMOP_EXTENSION_CAP_CMPXCHG;
632	break;
633	case KVM_CAP_NR_VCPUS:
634	case KVM_CAP_MAX_VCPUS:
635	case KVM_CAP_MAX_VCPU_ID:
636	/*
637	* Return the same value for KVM_CAP_MAX_VCPUS and
638	* KVM_CAP_MAX_VCPU_ID to conform with the KVM API.
639	*/
640	r = KVM_S390_ESCA_CPU_SLOTS;
641	if (!kvm_s390_use_sca_entries())
642	r = KVM_MAX_VCPUS;
643	if (ext == KVM_CAP_NR_VCPUS)
644	r = min_t(unsigned int, num_online_cpus(), r);
645	break;
646	case KVM_CAP_S390_COW:
647	r = machine_has_esop();
648	break;
649	case KVM_CAP_S390_VECTOR_REGISTERS:
650	r = test_facility(129);
651	break;
652	case KVM_CAP_S390_RI:
653	r = test_facility(64);
654	break;
655	case KVM_CAP_S390_GS:
656	r = test_facility(133);
657	break;
658	case KVM_CAP_S390_BPB:
659	r = test_facility(82);
660	break;
661	case KVM_CAP_S390_PROTECTED_ASYNC_DISABLE:
662	r = async_destroy && is_prot_virt_host();
663	break;
664	case KVM_CAP_S390_PROTECTED:
665	r = is_prot_virt_host();
666	break;
667	case KVM_CAP_S390_PROTECTED_DUMP: {
668	u64 pv_cmds_dump[] = {
669	BIT_UVC_CMD_DUMP_INIT,
670	BIT_UVC_CMD_DUMP_CONFIG_STOR_STATE,
671	BIT_UVC_CMD_DUMP_CPU,
672	BIT_UVC_CMD_DUMP_COMPLETE,
673	};
674	int i;
675
676	r = is_prot_virt_host();
677
678	for (i = 0; i < ARRAY_SIZE(pv_cmds_dump); i++) {
679	if (!test_bit_inv(pv_cmds_dump[i],
680	(unsigned long *)&uv_info.inst_calls_list)) {
681	r = 0;
682	break;
683	}
684	}
685	break;
686	}
687	case KVM_CAP_S390_ZPCI_OP:
688	r = kvm_s390_pci_interp_allowed();
689	break;
690	case KVM_CAP_S390_CPU_TOPOLOGY:
691	r = test_facility(11);
692	break;
693	default:
694	r = 0;
695	}
696	return r;
697	}
698
699	void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
700	{
701	int i;
702	gfn_t cur_gfn, last_gfn;
703	unsigned long gaddr, vmaddr;
704	struct gmap *gmap = kvm->arch.gmap;
705	DECLARE_BITMAP(bitmap, _PAGE_ENTRIES);
706
707	/* Loop over all guest segments */
708	cur_gfn = memslot->base_gfn;
709	last_gfn = memslot->base_gfn + memslot->npages;
710	for (; cur_gfn <= last_gfn; cur_gfn += _PAGE_ENTRIES) {
711	gaddr = gfn_to_gpa(gfn: cur_gfn);
712	vmaddr = gfn_to_hva_memslot(slot: memslot, gfn: cur_gfn);
713	if (kvm_is_error_hva(addr: vmaddr))
714	continue;
715
716	bitmap_zero(bitmap, _PAGE_ENTRIES);
717	gmap_sync_dirty_log_pmd(gmap, bitmap, gaddr, vmaddr);
718	for (i = 0; i < _PAGE_ENTRIES; i++) {
719	if (test_bit(i, bitmap))
720	mark_page_dirty(kvm, cur_gfn + i);
721	}
722
723	if (fatal_signal_pending(current))
724	return;
725	cond_resched();
726	}
727	}
728
729	/* Section: vm related */
730	static void sca_del_vcpu(struct kvm_vcpu *vcpu);
731
732	/*
733	* Get (and clear) the dirty memory log for a memory slot.
734	*/
735	int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
736	struct kvm_dirty_log *log)
737	{
738	int r;
739	unsigned long n;
740	struct kvm_memory_slot *memslot;
741	int is_dirty;
742
743	if (kvm_is_ucontrol(kvm))
744	return -EINVAL;
745
746	mutex_lock(&kvm->slots_lock);
747
748	r = -EINVAL;
749	if (log->slot >= KVM_USER_MEM_SLOTS)
750	goto out;
751
752	r = kvm_get_dirty_log(kvm, log, &is_dirty, &memslot);
753	if (r)
754	goto out;
755
756	/* Clear the dirty log */
757	if (is_dirty) {
758	n = kvm_dirty_bitmap_bytes(memslot);
759	memset(memslot->dirty_bitmap, 0, n);
760	}
761	r = 0;
762	out:
763	mutex_unlock(lock: &kvm->slots_lock);
764	return r;
765	}
766
767	static void icpt_operexc_on_all_vcpus(struct kvm *kvm)
768	{
769	unsigned long i;
770	struct kvm_vcpu *vcpu;
771
772	kvm_for_each_vcpu(i, vcpu, kvm) {
773	kvm_s390_sync_request(KVM_REQ_ICPT_OPEREXC, vcpu);
774	}
775	}
776
777	int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
778	{
779	int r;
780
781	if (cap->flags)
782	return -EINVAL;
783
784	switch (cap->cap) {
785	case KVM_CAP_S390_IRQCHIP:
786	VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
787	kvm->arch.use_irqchip = 1;
788	r = 0;
789	break;
790	case KVM_CAP_S390_USER_SIGP:
791	VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
792	kvm->arch.user_sigp = 1;
793	r = 0;
794	break;
795	case KVM_CAP_S390_VECTOR_REGISTERS:
796	mutex_lock(&kvm->lock);
797	if (kvm->created_vcpus) {
798	r = -EBUSY;
799	} else if (cpu_has_vx()) {
800	set_kvm_facility(fac_list: kvm->arch.model.fac_mask, nr: 129);
801	set_kvm_facility(fac_list: kvm->arch.model.fac_list, nr: 129);
802	if (test_facility(134)) {
803	set_kvm_facility(fac_list: kvm->arch.model.fac_mask, nr: 134);
804	set_kvm_facility(fac_list: kvm->arch.model.fac_list, nr: 134);
805	}
806	if (test_facility(135)) {
807	set_kvm_facility(fac_list: kvm->arch.model.fac_mask, nr: 135);
808	set_kvm_facility(fac_list: kvm->arch.model.fac_list, nr: 135);
809	}
810	if (test_facility(148)) {
811	set_kvm_facility(fac_list: kvm->arch.model.fac_mask, nr: 148);
812	set_kvm_facility(fac_list: kvm->arch.model.fac_list, nr: 148);
813	}
814	if (test_facility(152)) {
815	set_kvm_facility(fac_list: kvm->arch.model.fac_mask, nr: 152);
816	set_kvm_facility(fac_list: kvm->arch.model.fac_list, nr: 152);
817	}
818	if (test_facility(192)) {
819	set_kvm_facility(fac_list: kvm->arch.model.fac_mask, nr: 192);
820	set_kvm_facility(fac_list: kvm->arch.model.fac_list, nr: 192);
821	}
822	if (test_facility(198)) {
823	set_kvm_facility(fac_list: kvm->arch.model.fac_mask, nr: 198);
824	set_kvm_facility(fac_list: kvm->arch.model.fac_list, nr: 198);
825	}
826	if (test_facility(199)) {
827	set_kvm_facility(fac_list: kvm->arch.model.fac_mask, nr: 199);
828	set_kvm_facility(fac_list: kvm->arch.model.fac_list, nr: 199);
829	}
830	r = 0;
831	} else
832	r = -EINVAL;
833	mutex_unlock(lock: &kvm->lock);
834	VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
835	r ? "(not available)" : "(success)");
836	break;
837	case KVM_CAP_S390_RI:
838	r = -EINVAL;
839	mutex_lock(&kvm->lock);
840	if (kvm->created_vcpus) {
841	r = -EBUSY;
842	} else if (test_facility(64)) {
843	set_kvm_facility(fac_list: kvm->arch.model.fac_mask, nr: 64);
844	set_kvm_facility(fac_list: kvm->arch.model.fac_list, nr: 64);
845	r = 0;
846	}
847	mutex_unlock(lock: &kvm->lock);
848	VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
849	r ? "(not available)" : "(success)");
850	break;
851	case KVM_CAP_S390_AIS:
852	mutex_lock(&kvm->lock);
853	if (kvm->created_vcpus) {
854	r = -EBUSY;
855	} else {
856	set_kvm_facility(fac_list: kvm->arch.model.fac_mask, nr: 72);
857	set_kvm_facility(fac_list: kvm->arch.model.fac_list, nr: 72);
858	r = 0;
859	}
860	mutex_unlock(lock: &kvm->lock);
861	VM_EVENT(kvm, 3, "ENABLE: AIS %s",
862	r ? "(not available)" : "(success)");
863	break;
864	case KVM_CAP_S390_GS:
865	r = -EINVAL;
866	mutex_lock(&kvm->lock);
867	if (kvm->created_vcpus) {
868	r = -EBUSY;
869	} else if (test_facility(133)) {
870	set_kvm_facility(fac_list: kvm->arch.model.fac_mask, nr: 133);
871	set_kvm_facility(fac_list: kvm->arch.model.fac_list, nr: 133);
872	r = 0;
873	}
874	mutex_unlock(lock: &kvm->lock);
875	VM_EVENT(kvm, 3, "ENABLE: CAP_S390_GS %s",
876	r ? "(not available)" : "(success)");
877	break;
878	case KVM_CAP_S390_HPAGE_1M:
879	mutex_lock(&kvm->lock);
880	if (kvm->created_vcpus)
881	r = -EBUSY;
882	else if (!hpage || kvm->arch.use_cmma || kvm_is_ucontrol(kvm))
883	r = -EINVAL;
884	else {
885	r = 0;
886	mmap_write_lock(mm: kvm->mm);
887	kvm->mm->context.allow_gmap_hpage_1m = 1;
888	mmap_write_unlock(mm: kvm->mm);
889	/*
890	* We might have to create fake 4k page
891	* tables. To avoid that the hardware works on
892	* stale PGSTEs, we emulate these instructions.
893	*/
894	kvm->arch.use_skf = 0;
895	kvm->arch.use_pfmfi = 0;
896	}
897	mutex_unlock(lock: &kvm->lock);
898	VM_EVENT(kvm, 3, "ENABLE: CAP_S390_HPAGE %s",
899	r ? "(not available)" : "(success)");
900	break;
901	case KVM_CAP_S390_USER_STSI:
902	VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
903	kvm->arch.user_stsi = 1;
904	r = 0;
905	break;
906	case KVM_CAP_S390_USER_INSTR0:
907	VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0");
908	kvm->arch.user_instr0 = 1;
909	icpt_operexc_on_all_vcpus(kvm);
910	r = 0;
911	break;
912	case KVM_CAP_S390_CPU_TOPOLOGY:
913	r = -EINVAL;
914	mutex_lock(&kvm->lock);
915	if (kvm->created_vcpus) {
916	r = -EBUSY;
917	} else if (test_facility(11)) {
918	set_kvm_facility(fac_list: kvm->arch.model.fac_mask, nr: 11);
919	set_kvm_facility(fac_list: kvm->arch.model.fac_list, nr: 11);
920	r = 0;
921	}
922	mutex_unlock(lock: &kvm->lock);
923	VM_EVENT(kvm, 3, "ENABLE: CAP_S390_CPU_TOPOLOGY %s",
924	r ? "(not available)" : "(success)");
925	break;
926	case KVM_CAP_S390_USER_OPEREXEC:
927	VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_OPEREXEC");
928	kvm->arch.user_operexec = 1;
929	icpt_operexc_on_all_vcpus(kvm);
930	r = 0;
931	break;
932	default:
933	r = -EINVAL;
934	break;
935	}
936	return r;
937	}
938
939	static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
940	{
941	int ret;
942
943	switch (attr->attr) {
944	case KVM_S390_VM_MEM_LIMIT_SIZE:
945	ret = 0;
946	VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
947	kvm->arch.mem_limit);
948	if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
949	ret = -EFAULT;
950	break;
951	default:
952	ret = -ENXIO;
953	break;
954	}
955	return ret;
956	}
957
958	static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
959	{
960	int ret;
961	unsigned int idx;
962	switch (attr->attr) {
963	case KVM_S390_VM_MEM_ENABLE_CMMA:
964	ret = -ENXIO;
965	if (!sclp.has_cmma)
966	break;
967
968	VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
969	mutex_lock(&kvm->lock);
970	if (kvm->created_vcpus)
971	ret = -EBUSY;
972	else if (kvm->mm->context.allow_gmap_hpage_1m)
973	ret = -EINVAL;
974	else {
975	kvm->arch.use_cmma = 1;
976	/* Not compatible with cmma. */
977	kvm->arch.use_pfmfi = 0;
978	ret = 0;
979	}
980	mutex_unlock(lock: &kvm->lock);
981	break;
982	case KVM_S390_VM_MEM_CLR_CMMA:
983	ret = -ENXIO;
984	if (!sclp.has_cmma)
985	break;
986	ret = -EINVAL;
987	if (!kvm->arch.use_cmma)
988	break;
989
990	VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
991	mutex_lock(&kvm->lock);
992	idx = srcu_read_lock(ssp: &kvm->srcu);
993	s390_reset_cmma(kvm->arch.gmap->mm);
994	srcu_read_unlock(ssp: &kvm->srcu, idx);
995	mutex_unlock(lock: &kvm->lock);
996	ret = 0;
997	break;
998	case KVM_S390_VM_MEM_LIMIT_SIZE: {
999	unsigned long new_limit;
1000
1001	if (kvm_is_ucontrol(kvm))
1002	return -EINVAL;
1003
1004	if (get_user(new_limit, (u64 __user *)attr->addr))
1005	return -EFAULT;
1006
1007	if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
1008	new_limit > kvm->arch.mem_limit)
1009	return -E2BIG;
1010
1011	if (!new_limit)
1012	return -EINVAL;
1013
1014	/* gmap_create takes last usable address */
1015	if (new_limit != KVM_S390_NO_MEM_LIMIT)
1016	new_limit -= 1;
1017
1018	ret = -EBUSY;
1019	mutex_lock(&kvm->lock);
1020	if (!kvm->created_vcpus) {
1021	/* gmap_create will round the limit up */
1022	struct gmap *new = gmap_create(current->mm, new_limit);
1023
1024	if (!new) {
1025	ret = -ENOMEM;
1026	} else {
1027	gmap_remove(kvm->arch.gmap);
1028	new->private = kvm;
1029	kvm->arch.gmap = new;
1030	ret = 0;
1031	}
1032	}
1033	mutex_unlock(lock: &kvm->lock);
1034	VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
1035	VM_EVENT(kvm, 3, "New guest asce: 0x%p",
1036	(void *) kvm->arch.gmap->asce);
1037	break;
1038	}
1039	default:
1040	ret = -ENXIO;
1041	break;
1042	}
1043	return ret;
1044	}
1045
1046	static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
1047
1048	void kvm_s390_vcpu_crypto_reset_all(struct kvm *kvm)
1049	{
1050	struct kvm_vcpu *vcpu;
1051	unsigned long i;
1052
1053	kvm_s390_vcpu_block_all(kvm);
1054
1055	kvm_for_each_vcpu(i, vcpu, kvm) {
1056	kvm_s390_vcpu_crypto_setup(vcpu);
1057	/* recreate the shadow crycb by leaving the VSIE handler */
1058	kvm_s390_sync_request(KVM_REQ_VSIE_RESTART, vcpu);
1059	}
1060
1061	kvm_s390_vcpu_unblock_all(kvm);
1062	}
1063
1064	static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
1065	{
1066	mutex_lock(&kvm->lock);
1067	switch (attr->attr) {
1068	case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1069	if (!test_kvm_facility(kvm, nr: 76)) {
1070	mutex_unlock(lock: &kvm->lock);
1071	return -EINVAL;
1072	}
1073	get_random_bytes(
1074	buf: kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1075	len: sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1076	kvm->arch.crypto.aes_kw = 1;
1077	VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
1078	break;
1079	case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1080	if (!test_kvm_facility(kvm, nr: 76)) {
1081	mutex_unlock(lock: &kvm->lock);
1082	return -EINVAL;
1083	}
1084	get_random_bytes(
1085	buf: kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1086	len: sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
1087	kvm->arch.crypto.dea_kw = 1;
1088	VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
1089	break;
1090	case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1091	if (!test_kvm_facility(kvm, nr: 76)) {
1092	mutex_unlock(lock: &kvm->lock);
1093	return -EINVAL;
1094	}
1095	kvm->arch.crypto.aes_kw = 0;
1096	memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
1097	sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1098	VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
1099	break;
1100	case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1101	if (!test_kvm_facility(kvm, nr: 76)) {
1102	mutex_unlock(lock: &kvm->lock);
1103	return -EINVAL;
1104	}
1105	kvm->arch.crypto.dea_kw = 0;
1106	memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
1107	sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
1108	VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
1109	break;
1110	case KVM_S390_VM_CRYPTO_ENABLE_APIE:
1111	if (!ap_instructions_available()) {
1112	mutex_unlock(lock: &kvm->lock);
1113	return -EOPNOTSUPP;
1114	}
1115	kvm->arch.crypto.apie = 1;
1116	break;
1117	case KVM_S390_VM_CRYPTO_DISABLE_APIE:
1118	if (!ap_instructions_available()) {
1119	mutex_unlock(lock: &kvm->lock);
1120	return -EOPNOTSUPP;
1121	}
1122	kvm->arch.crypto.apie = 0;
1123	break;
1124	default:
1125	mutex_unlock(lock: &kvm->lock);
1126	return -ENXIO;
1127	}
1128
1129	kvm_s390_vcpu_crypto_reset_all(kvm);
1130	mutex_unlock(lock: &kvm->lock);
1131	return 0;
1132	}
1133
1134	static void kvm_s390_vcpu_pci_setup(struct kvm_vcpu *vcpu)
1135	{
1136	/* Only set the ECB bits after guest requests zPCI interpretation */
1137	if (!vcpu->kvm->arch.use_zpci_interp)
1138	return;
1139
1140	vcpu->arch.sie_block->ecb2 |= ECB2_ZPCI_LSI;
1141	vcpu->arch.sie_block->ecb3 |= ECB3_AISII + ECB3_AISI;
1142	}
1143
1144	void kvm_s390_vcpu_pci_enable_interp(struct kvm *kvm)
1145	{
1146	struct kvm_vcpu *vcpu;
1147	unsigned long i;
1148
1149	lockdep_assert_held(&kvm->lock);
1150
1151	if (!kvm_s390_pci_interp_allowed())
1152	return;
1153
1154	/*
1155	* If host is configured for PCI and the necessary facilities are
1156	* available, turn on interpretation for the life of this guest
1157	*/
1158	kvm->arch.use_zpci_interp = 1;
1159
1160	kvm_s390_vcpu_block_all(kvm);
1161
1162	kvm_for_each_vcpu(i, vcpu, kvm) {
1163	kvm_s390_vcpu_pci_setup(vcpu);
1164	kvm_s390_sync_request(KVM_REQ_VSIE_RESTART, vcpu);
1165	}
1166
1167	kvm_s390_vcpu_unblock_all(kvm);
1168	}
1169
1170	static void kvm_s390_sync_request_broadcast(struct kvm *kvm, int req)
1171	{
1172	unsigned long cx;
1173	struct kvm_vcpu *vcpu;
1174
1175	kvm_for_each_vcpu(cx, vcpu, kvm)
1176	kvm_s390_sync_request(req, vcpu);
1177	}
1178
1179	/*
1180	* Must be called with kvm->srcu held to avoid races on memslots, and with
1181	* kvm->slots_lock to avoid races with ourselves and kvm_s390_vm_stop_migration.
1182	*/
1183	static int kvm_s390_vm_start_migration(struct kvm *kvm)
1184	{
1185	struct kvm_memory_slot *ms;
1186	struct kvm_memslots *slots;
1187	unsigned long ram_pages = 0;
1188	int bkt;
1189
1190	/* migration mode already enabled */
1191	if (kvm->arch.migration_mode)
1192	return 0;
1193	slots = kvm_memslots(kvm);
1194	if (!slots || kvm_memslots_empty(slots))
1195	return -EINVAL;
1196
1197	if (!kvm->arch.use_cmma) {
1198	kvm->arch.migration_mode = 1;
1199	return 0;
1200	}
1201	/* mark all the pages in active slots as dirty */
1202	kvm_for_each_memslot(ms, bkt, slots) {
1203	if (!ms->dirty_bitmap)
1204	return -EINVAL;
1205	/*
1206	* The second half of the bitmap is only used on x86,
1207	* and would be wasted otherwise, so we put it to good
1208	* use here to keep track of the state of the storage
1209	* attributes.
1210	*/
1211	memset(kvm_second_dirty_bitmap(ms), 0xff, kvm_dirty_bitmap_bytes(ms));
1212	ram_pages += ms->npages;
1213	}
1214	atomic64_set(v: &kvm->arch.cmma_dirty_pages, i: ram_pages);
1215	kvm->arch.migration_mode = 1;
1216	kvm_s390_sync_request_broadcast(kvm, KVM_REQ_START_MIGRATION);
1217	return 0;
1218	}
1219
1220	/*
1221	* Must be called with kvm->slots_lock to avoid races with ourselves and
1222	* kvm_s390_vm_start_migration.
1223	*/
1224	static int kvm_s390_vm_stop_migration(struct kvm *kvm)
1225	{
1226	/* migration mode already disabled */
1227	if (!kvm->arch.migration_mode)
1228	return 0;
1229	kvm->arch.migration_mode = 0;
1230	if (kvm->arch.use_cmma)
1231	kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION);
1232	return 0;
1233	}
1234
1235	static int kvm_s390_vm_set_migration(struct kvm *kvm,
1236	struct kvm_device_attr *attr)
1237	{
1238	int res = -ENXIO;
1239
1240	mutex_lock(&kvm->slots_lock);
1241	switch (attr->attr) {
1242	case KVM_S390_VM_MIGRATION_START:
1243	res = kvm_s390_vm_start_migration(kvm);
1244	break;
1245	case KVM_S390_VM_MIGRATION_STOP:
1246	res = kvm_s390_vm_stop_migration(kvm);
1247	break;
1248	default:
1249	break;
1250	}
1251	mutex_unlock(lock: &kvm->slots_lock);
1252
1253	return res;
1254	}
1255
1256	static int kvm_s390_vm_get_migration(struct kvm *kvm,
1257	struct kvm_device_attr *attr)
1258	{
1259	u64 mig = kvm->arch.migration_mode;
1260
1261	if (attr->attr != KVM_S390_VM_MIGRATION_STATUS)
1262	return -ENXIO;
1263
1264	if (copy_to_user(to: (void __user *)attr->addr, from: &mig, n: sizeof(mig)))
1265	return -EFAULT;
1266	return 0;
1267	}
1268
1269	static void __kvm_s390_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clock *gtod);
1270
1271	static int kvm_s390_set_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
1272	{
1273	struct kvm_s390_vm_tod_clock gtod;
1274
1275	if (copy_from_user(to: &gtod, from: (void __user *)attr->addr, n: sizeof(gtod)))
1276	return -EFAULT;
1277
1278	if (!test_kvm_facility(kvm, nr: 139) && gtod.epoch_idx)
1279	return -EINVAL;
1280	__kvm_s390_set_tod_clock(kvm, gtod: &gtod);
1281
1282	VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x, TOD base: 0x%llx",
1283	gtod.epoch_idx, gtod.tod);
1284
1285	return 0;
1286	}
1287
1288	static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1289	{
1290	u8 gtod_high;
1291
1292	if (copy_from_user(to: &gtod_high, from: (void __user *)attr->addr,
1293	n: sizeof(gtod_high)))
1294	return -EFAULT;
1295
1296	if (gtod_high != 0)
1297	return -EINVAL;
1298	VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
1299
1300	return 0;
1301	}
1302
1303	static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1304	{
1305	struct kvm_s390_vm_tod_clock gtod = { 0 };
1306
1307	if (copy_from_user(to: &gtod.tod, from: (void __user *)attr->addr,
1308	n: sizeof(gtod.tod)))
1309	return -EFAULT;
1310
1311	__kvm_s390_set_tod_clock(kvm, gtod: &gtod);
1312	VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod.tod);
1313	return 0;
1314	}
1315
1316	static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1317	{
1318	int ret;
1319
1320	if (attr->flags)
1321	return -EINVAL;
1322
1323	mutex_lock(&kvm->lock);
1324	/*
1325	* For protected guests, the TOD is managed by the ultravisor, so trying
1326	* to change it will never bring the expected results.
1327	*/
1328	if (kvm_s390_pv_is_protected(kvm)) {
1329	ret = -EOPNOTSUPP;
1330	goto out_unlock;
1331	}
1332
1333	switch (attr->attr) {
1334	case KVM_S390_VM_TOD_EXT:
1335	ret = kvm_s390_set_tod_ext(kvm, attr);
1336	break;
1337	case KVM_S390_VM_TOD_HIGH:
1338	ret = kvm_s390_set_tod_high(kvm, attr);
1339	break;
1340	case KVM_S390_VM_TOD_LOW:
1341	ret = kvm_s390_set_tod_low(kvm, attr);
1342	break;
1343	default:
1344	ret = -ENXIO;
1345	break;
1346	}
1347
1348	out_unlock:
1349	mutex_unlock(lock: &kvm->lock);
1350	return ret;
1351	}
1352
1353	static void kvm_s390_get_tod_clock(struct kvm *kvm,
1354	struct kvm_s390_vm_tod_clock *gtod)
1355	{
1356	union tod_clock clk;
1357
1358	preempt_disable();
1359
1360	store_tod_clock_ext(&clk);
1361
1362	gtod->tod = clk.tod + kvm->arch.epoch;
1363	gtod->epoch_idx = 0;
1364	if (test_kvm_facility(kvm, nr: 139)) {
1365	gtod->epoch_idx = clk.ei + kvm->arch.epdx;
1366	if (gtod->tod < clk.tod)
1367	gtod->epoch_idx += 1;
1368	}
1369
1370	preempt_enable();
1371	}
1372
1373	static int kvm_s390_get_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
1374	{
1375	struct kvm_s390_vm_tod_clock gtod;
1376
1377	memset(&gtod, 0, sizeof(gtod));
1378	kvm_s390_get_tod_clock(kvm, gtod: &gtod);
1379	if (copy_to_user(to: (void __user *)attr->addr, from: &gtod, n: sizeof(gtod)))
1380	return -EFAULT;
1381
1382	VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x, TOD base: 0x%llx",
1383	gtod.epoch_idx, gtod.tod);
1384	return 0;
1385	}
1386
1387	static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1388	{
1389	u8 gtod_high = 0;
1390
1391	if (copy_to_user(to: (void __user *)attr->addr, from: &gtod_high,
1392	n: sizeof(gtod_high)))
1393	return -EFAULT;
1394	VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
1395
1396	return 0;
1397	}
1398
1399	static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1400	{
1401	u64 gtod;
1402
1403	gtod = kvm_s390_get_tod_clock_fast(kvm);
1404	if (copy_to_user(to: (void __user *)attr->addr, from: &gtod, n: sizeof(gtod)))
1405	return -EFAULT;
1406	VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
1407
1408	return 0;
1409	}
1410
1411	static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1412	{
1413	int ret;
1414
1415	if (attr->flags)
1416	return -EINVAL;
1417
1418	switch (attr->attr) {
1419	case KVM_S390_VM_TOD_EXT:
1420	ret = kvm_s390_get_tod_ext(kvm, attr);
1421	break;
1422	case KVM_S390_VM_TOD_HIGH:
1423	ret = kvm_s390_get_tod_high(kvm, attr);
1424	break;
1425	case KVM_S390_VM_TOD_LOW:
1426	ret = kvm_s390_get_tod_low(kvm, attr);
1427	break;
1428	default:
1429	ret = -ENXIO;
1430	break;
1431	}
1432	return ret;
1433	}
1434
1435	static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1436	{
1437	struct kvm_s390_vm_cpu_processor *proc;
1438	u16 lowest_ibc, unblocked_ibc;
1439	int ret = 0;
1440
1441	mutex_lock(&kvm->lock);
1442	if (kvm->created_vcpus) {
1443	ret = -EBUSY;
1444	goto out;
1445	}
1446	proc = kzalloc(sizeof(*proc), GFP_KERNEL_ACCOUNT);
1447	if (!proc) {
1448	ret = -ENOMEM;
1449	goto out;
1450	}
1451	if (!copy_from_user(proc, (void __user *)attr->addr,
1452	sizeof(*proc))) {
1453	kvm->arch.model.cpuid = proc->cpuid;
1454	lowest_ibc = sclp.ibc >> 16 & 0xfff;
1455	unblocked_ibc = sclp.ibc & 0xfff;
1456	if (lowest_ibc && proc->ibc) {
1457	if (proc->ibc > unblocked_ibc)
1458	kvm->arch.model.ibc = unblocked_ibc;
1459	else if (proc->ibc < lowest_ibc)
1460	kvm->arch.model.ibc = lowest_ibc;
1461	else
1462	kvm->arch.model.ibc = proc->ibc;
1463	}
1464	memcpy(kvm->arch.model.fac_list, proc->fac_list,
1465	S390_ARCH_FAC_LIST_SIZE_BYTE);
1466	VM_EVENT(kvm, 3, "SET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1467	kvm->arch.model.ibc,
1468	kvm->arch.model.cpuid);
1469	VM_EVENT(kvm, 3, "SET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1470	kvm->arch.model.fac_list[0],
1471	kvm->arch.model.fac_list[1],
1472	kvm->arch.model.fac_list[2]);
1473	} else
1474	ret = -EFAULT;
1475	kfree(objp: proc);
1476	out:
1477	mutex_unlock(lock: &kvm->lock);
1478	return ret;
1479	}
1480
1481	static int kvm_s390_set_processor_feat(struct kvm *kvm,
1482	struct kvm_device_attr *attr)
1483	{
1484	struct kvm_s390_vm_cpu_feat data;
1485
1486	if (copy_from_user(to: &data, from: (void __user *)attr->addr, n: sizeof(data)))
1487	return -EFAULT;
1488	if (!bitmap_subset((unsigned long *) data.feat,
1489	kvm_s390_available_cpu_feat,
1490	KVM_S390_VM_CPU_FEAT_NR_BITS))
1491	return -EINVAL;
1492
1493	mutex_lock(&kvm->lock);
1494	if (kvm->created_vcpus) {
1495	mutex_unlock(lock: &kvm->lock);
1496	return -EBUSY;
1497	}
1498	bitmap_from_arr64(kvm->arch.cpu_feat, data.feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
1499	mutex_unlock(lock: &kvm->lock);
1500	VM_EVENT(kvm, 3, "SET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1501	data.feat[0],
1502	data.feat[1],
1503	data.feat[2]);
1504	return 0;
1505	}
1506
1507	static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
1508	struct kvm_device_attr *attr)
1509	{
1510	mutex_lock(&kvm->lock);
1511	if (kvm->created_vcpus) {
1512	mutex_unlock(lock: &kvm->lock);
1513	return -EBUSY;
1514	}
1515
1516	if (copy_from_user(&kvm->arch.model.subfuncs, (void __user *)attr->addr,
1517	sizeof(struct kvm_s390_vm_cpu_subfunc))) {
1518	mutex_unlock(lock: &kvm->lock);
1519	return -EFAULT;
1520	}
1521	mutex_unlock(lock: &kvm->lock);
1522
1523	VM_EVENT(kvm, 3, "SET: guest PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1524	((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
1525	((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
1526	((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
1527	((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
1528	VM_EVENT(kvm, 3, "SET: guest PTFF subfunc 0x%16.16lx.%16.16lx",
1529	((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
1530	((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
1531	VM_EVENT(kvm, 3, "SET: guest KMAC subfunc 0x%16.16lx.%16.16lx",
1532	((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
1533	((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
1534	VM_EVENT(kvm, 3, "SET: guest KMC subfunc 0x%16.16lx.%16.16lx",
1535	((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
1536	((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
1537	VM_EVENT(kvm, 3, "SET: guest KM subfunc 0x%16.16lx.%16.16lx",
1538	((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
1539	((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
1540	VM_EVENT(kvm, 3, "SET: guest KIMD subfunc 0x%16.16lx.%16.16lx",
1541	((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
1542	((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
1543	VM_EVENT(kvm, 3, "SET: guest KLMD subfunc 0x%16.16lx.%16.16lx",
1544	((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
1545	((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
1546	VM_EVENT(kvm, 3, "SET: guest PCKMO subfunc 0x%16.16lx.%16.16lx",
1547	((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
1548	((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
1549	VM_EVENT(kvm, 3, "SET: guest KMCTR subfunc 0x%16.16lx.%16.16lx",
1550	((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
1551	((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
1552	VM_EVENT(kvm, 3, "SET: guest KMF subfunc 0x%16.16lx.%16.16lx",
1553	((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
1554	((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
1555	VM_EVENT(kvm, 3, "SET: guest KMO subfunc 0x%16.16lx.%16.16lx",
1556	((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
1557	((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
1558	VM_EVENT(kvm, 3, "SET: guest PCC subfunc 0x%16.16lx.%16.16lx",
1559	((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
1560	((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
1561	VM_EVENT(kvm, 3, "SET: guest PPNO subfunc 0x%16.16lx.%16.16lx",
1562	((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
1563	((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
1564	VM_EVENT(kvm, 3, "SET: guest KMA subfunc 0x%16.16lx.%16.16lx",
1565	((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
1566	((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
1567	VM_EVENT(kvm, 3, "SET: guest KDSA subfunc 0x%16.16lx.%16.16lx",
1568	((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
1569	((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
1570	VM_EVENT(kvm, 3, "SET: guest SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1571	((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
1572	((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
1573	((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
1574	((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
1575	VM_EVENT(kvm, 3, "SET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1576	((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
1577	((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
1578	((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
1579	((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
1580	VM_EVENT(kvm, 3, "GET: guest PFCR subfunc 0x%16.16lx.%16.16lx",
1581	((unsigned long *) &kvm_s390_available_subfunc.pfcr)[0],
1582	((unsigned long *) &kvm_s390_available_subfunc.pfcr)[1]);
1583
1584	return 0;
1585	}
1586
1587	#define KVM_S390_VM_CPU_UV_FEAT_GUEST_MASK \
1588	( \
1589	((struct kvm_s390_vm_cpu_uv_feat){ \
1590	.ap = 1, \
1591	.ap_intr = 1, \
1592	}) \
1593	.feat \
1594	)
1595
1596	static int kvm_s390_set_uv_feat(struct kvm *kvm, struct kvm_device_attr *attr)
1597	{
1598	struct kvm_s390_vm_cpu_uv_feat __user *ptr = (void __user *)attr->addr;
1599	unsigned long data, filter;
1600
1601	filter = uv_info.uv_feature_indications & KVM_S390_VM_CPU_UV_FEAT_GUEST_MASK;
1602	if (get_user(data, &ptr->feat))
1603	return -EFAULT;
1604	if (!bitmap_subset(&data, &filter, KVM_S390_VM_CPU_UV_FEAT_NR_BITS))
1605	return -EINVAL;
1606
1607	mutex_lock(&kvm->lock);
1608	if (kvm->created_vcpus) {
1609	mutex_unlock(lock: &kvm->lock);
1610	return -EBUSY;
1611	}
1612	kvm->arch.model.uv_feat_guest.feat = data;
1613	mutex_unlock(lock: &kvm->lock);
1614
1615	VM_EVENT(kvm, 3, "SET: guest UV-feat: 0x%16.16lx", data);
1616
1617	return 0;
1618	}
1619
1620	static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1621	{
1622	int ret = -ENXIO;
1623
1624	switch (attr->attr) {
1625	case KVM_S390_VM_CPU_PROCESSOR:
1626	ret = kvm_s390_set_processor(kvm, attr);
1627	break;
1628	case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1629	ret = kvm_s390_set_processor_feat(kvm, attr);
1630	break;
1631	case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1632	ret = kvm_s390_set_processor_subfunc(kvm, attr);
1633	break;
1634	case KVM_S390_VM_CPU_PROCESSOR_UV_FEAT_GUEST:
1635	ret = kvm_s390_set_uv_feat(kvm, attr);
1636	break;
1637	}
1638	return ret;
1639	}
1640
1641	static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1642	{
1643	struct kvm_s390_vm_cpu_processor *proc;
1644	int ret = 0;
1645
1646	proc = kzalloc(sizeof(*proc), GFP_KERNEL_ACCOUNT);
1647	if (!proc) {
1648	ret = -ENOMEM;
1649	goto out;
1650	}
1651	proc->cpuid = kvm->arch.model.cpuid;
1652	proc->ibc = kvm->arch.model.ibc;
1653	memcpy(&proc->fac_list, kvm->arch.model.fac_list,
1654	S390_ARCH_FAC_LIST_SIZE_BYTE);
1655	VM_EVENT(kvm, 3, "GET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1656	kvm->arch.model.ibc,
1657	kvm->arch.model.cpuid);
1658	VM_EVENT(kvm, 3, "GET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1659	kvm->arch.model.fac_list[0],
1660	kvm->arch.model.fac_list[1],
1661	kvm->arch.model.fac_list[2]);
1662	if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
1663	ret = -EFAULT;
1664	kfree(objp: proc);
1665	out:
1666	return ret;
1667	}
1668
1669	static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
1670	{
1671	struct kvm_s390_vm_cpu_machine *mach;
1672	int ret = 0;
1673
1674	mach = kzalloc(sizeof(*mach), GFP_KERNEL_ACCOUNT);
1675	if (!mach) {
1676	ret = -ENOMEM;
1677	goto out;
1678	}
1679	get_cpu_id((struct cpuid *) &mach->cpuid);
1680	mach->ibc = sclp.ibc;
1681	memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
1682	S390_ARCH_FAC_LIST_SIZE_BYTE);
1683	memcpy((unsigned long *)&mach->fac_list, stfle_fac_list,
1684	sizeof(stfle_fac_list));
1685	VM_EVENT(kvm, 3, "GET: host ibc: 0x%4.4x, host cpuid: 0x%16.16llx",
1686	kvm->arch.model.ibc,
1687	kvm->arch.model.cpuid);
1688	VM_EVENT(kvm, 3, "GET: host facmask: 0x%16.16llx.%16.16llx.%16.16llx",
1689	mach->fac_mask[0],
1690	mach->fac_mask[1],
1691	mach->fac_mask[2]);
1692	VM_EVENT(kvm, 3, "GET: host faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1693	mach->fac_list[0],
1694	mach->fac_list[1],
1695	mach->fac_list[2]);
1696	if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
1697	ret = -EFAULT;
1698	kfree(objp: mach);
1699	out:
1700	return ret;
1701	}
1702
1703	static int kvm_s390_get_processor_feat(struct kvm *kvm,
1704	struct kvm_device_attr *attr)
1705	{
1706	struct kvm_s390_vm_cpu_feat data;
1707
1708	bitmap_to_arr64(data.feat, kvm->arch.cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
1709	if (copy_to_user(to: (void __user *)attr->addr, from: &data, n: sizeof(data)))
1710	return -EFAULT;
1711	VM_EVENT(kvm, 3, "GET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1712	data.feat[0],
1713	data.feat[1],
1714	data.feat[2]);
1715	return 0;
1716	}
1717
1718	static int kvm_s390_get_machine_feat(struct kvm *kvm,
1719	struct kvm_device_attr *attr)
1720	{
1721	struct kvm_s390_vm_cpu_feat data;
1722
1723	bitmap_to_arr64(data.feat, kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
1724	if (copy_to_user(to: (void __user *)attr->addr, from: &data, n: sizeof(data)))
1725	return -EFAULT;
1726	VM_EVENT(kvm, 3, "GET: host feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1727	data.feat[0],
1728	data.feat[1],
1729	data.feat[2]);
1730	return 0;
1731	}
1732
1733	static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
1734	struct kvm_device_attr *attr)
1735	{
1736	if (copy_to_user((void __user *)attr->addr, &kvm->arch.model.subfuncs,
1737	sizeof(struct kvm_s390_vm_cpu_subfunc)))
1738	return -EFAULT;
1739
1740	VM_EVENT(kvm, 3, "GET: guest PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1741	((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
1742	((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
1743	((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
1744	((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
1745	VM_EVENT(kvm, 3, "GET: guest PTFF subfunc 0x%16.16lx.%16.16lx",
1746	((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
1747	((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
1748	VM_EVENT(kvm, 3, "GET: guest KMAC subfunc 0x%16.16lx.%16.16lx",
1749	((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
1750	((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
1751	VM_EVENT(kvm, 3, "GET: guest KMC subfunc 0x%16.16lx.%16.16lx",
1752	((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
1753	((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
1754	VM_EVENT(kvm, 3, "GET: guest KM subfunc 0x%16.16lx.%16.16lx",
1755	((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
1756	((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
1757	VM_EVENT(kvm, 3, "GET: guest KIMD subfunc 0x%16.16lx.%16.16lx",
1758	((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
1759	((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
1760	VM_EVENT(kvm, 3, "GET: guest KLMD subfunc 0x%16.16lx.%16.16lx",
1761	((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
1762	((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
1763	VM_EVENT(kvm, 3, "GET: guest PCKMO subfunc 0x%16.16lx.%16.16lx",
1764	((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
1765	((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
1766	VM_EVENT(kvm, 3, "GET: guest KMCTR subfunc 0x%16.16lx.%16.16lx",
1767	((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
1768	((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
1769	VM_EVENT(kvm, 3, "GET: guest KMF subfunc 0x%16.16lx.%16.16lx",
1770	((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
1771	((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
1772	VM_EVENT(kvm, 3, "GET: guest KMO subfunc 0x%16.16lx.%16.16lx",
1773	((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
1774	((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
1775	VM_EVENT(kvm, 3, "GET: guest PCC subfunc 0x%16.16lx.%16.16lx",
1776	((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
1777	((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
1778	VM_EVENT(kvm, 3, "GET: guest PPNO subfunc 0x%16.16lx.%16.16lx",
1779	((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
1780	((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
1781	VM_EVENT(kvm, 3, "GET: guest KMA subfunc 0x%16.16lx.%16.16lx",
1782	((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
1783	((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
1784	VM_EVENT(kvm, 3, "GET: guest KDSA subfunc 0x%16.16lx.%16.16lx",
1785	((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
1786	((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
1787	VM_EVENT(kvm, 3, "GET: guest SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1788	((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
1789	((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
1790	((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
1791	((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
1792	VM_EVENT(kvm, 3, "GET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1793	((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
1794	((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
1795	((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
1796	((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
1797	VM_EVENT(kvm, 3, "GET: guest PFCR subfunc 0x%16.16lx.%16.16lx",
1798	((unsigned long *) &kvm_s390_available_subfunc.pfcr)[0],
1799	((unsigned long *) &kvm_s390_available_subfunc.pfcr)[1]);
1800
1801	return 0;
1802	}
1803
1804	static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
1805	struct kvm_device_attr *attr)
1806	{
1807	if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
1808	sizeof(struct kvm_s390_vm_cpu_subfunc)))
1809	return -EFAULT;
1810
1811	VM_EVENT(kvm, 3, "GET: host PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1812	((unsigned long *) &kvm_s390_available_subfunc.plo)[0],
1813	((unsigned long *) &kvm_s390_available_subfunc.plo)[1],
1814	((unsigned long *) &kvm_s390_available_subfunc.plo)[2],
1815	((unsigned long *) &kvm_s390_available_subfunc.plo)[3]);
1816	VM_EVENT(kvm, 3, "GET: host PTFF subfunc 0x%16.16lx.%16.16lx",
1817	((unsigned long *) &kvm_s390_available_subfunc.ptff)[0],
1818	((unsigned long *) &kvm_s390_available_subfunc.ptff)[1]);
1819	VM_EVENT(kvm, 3, "GET: host KMAC subfunc 0x%16.16lx.%16.16lx",
1820	((unsigned long *) &kvm_s390_available_subfunc.kmac)[0],
1821	((unsigned long *) &kvm_s390_available_subfunc.kmac)[1]);
1822	VM_EVENT(kvm, 3, "GET: host KMC subfunc 0x%16.16lx.%16.16lx",
1823	((unsigned long *) &kvm_s390_available_subfunc.kmc)[0],
1824	((unsigned long *) &kvm_s390_available_subfunc.kmc)[1]);
1825	VM_EVENT(kvm, 3, "GET: host KM subfunc 0x%16.16lx.%16.16lx",
1826	((unsigned long *) &kvm_s390_available_subfunc.km)[0],
1827	((unsigned long *) &kvm_s390_available_subfunc.km)[1]);
1828	VM_EVENT(kvm, 3, "GET: host KIMD subfunc 0x%16.16lx.%16.16lx",
1829	((unsigned long *) &kvm_s390_available_subfunc.kimd)[0],
1830	((unsigned long *) &kvm_s390_available_subfunc.kimd)[1]);
1831	VM_EVENT(kvm, 3, "GET: host KLMD subfunc 0x%16.16lx.%16.16lx",
1832	((unsigned long *) &kvm_s390_available_subfunc.klmd)[0],
1833	((unsigned long *) &kvm_s390_available_subfunc.klmd)[1]);
1834	VM_EVENT(kvm, 3, "GET: host PCKMO subfunc 0x%16.16lx.%16.16lx",
1835	((unsigned long *) &kvm_s390_available_subfunc.pckmo)[0],
1836	((unsigned long *) &kvm_s390_available_subfunc.pckmo)[1]);
1837	VM_EVENT(kvm, 3, "GET: host KMCTR subfunc 0x%16.16lx.%16.16lx",
1838	((unsigned long *) &kvm_s390_available_subfunc.kmctr)[0],
1839	((unsigned long *) &kvm_s390_available_subfunc.kmctr)[1]);
1840	VM_EVENT(kvm, 3, "GET: host KMF subfunc 0x%16.16lx.%16.16lx",
1841	((unsigned long *) &kvm_s390_available_subfunc.kmf)[0],
1842	((unsigned long *) &kvm_s390_available_subfunc.kmf)[1]);
1843	VM_EVENT(kvm, 3, "GET: host KMO subfunc 0x%16.16lx.%16.16lx",
1844	((unsigned long *) &kvm_s390_available_subfunc.kmo)[0],
1845	((unsigned long *) &kvm_s390_available_subfunc.kmo)[1]);
1846	VM_EVENT(kvm, 3, "GET: host PCC subfunc 0x%16.16lx.%16.16lx",
1847	((unsigned long *) &kvm_s390_available_subfunc.pcc)[0],
1848	((unsigned long *) &kvm_s390_available_subfunc.pcc)[1]);
1849	VM_EVENT(kvm, 3, "GET: host PPNO subfunc 0x%16.16lx.%16.16lx",
1850	((unsigned long *) &kvm_s390_available_subfunc.ppno)[0],
1851	((unsigned long *) &kvm_s390_available_subfunc.ppno)[1]);
1852	VM_EVENT(kvm, 3, "GET: host KMA subfunc 0x%16.16lx.%16.16lx",
1853	((unsigned long *) &kvm_s390_available_subfunc.kma)[0],
1854	((unsigned long *) &kvm_s390_available_subfunc.kma)[1]);
1855	VM_EVENT(kvm, 3, "GET: host KDSA subfunc 0x%16.16lx.%16.16lx",
1856	((unsigned long *) &kvm_s390_available_subfunc.kdsa)[0],
1857	((unsigned long *) &kvm_s390_available_subfunc.kdsa)[1]);
1858	VM_EVENT(kvm, 3, "GET: host SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1859	((unsigned long *) &kvm_s390_available_subfunc.sortl)[0],
1860	((unsigned long *) &kvm_s390_available_subfunc.sortl)[1],
1861	((unsigned long *) &kvm_s390_available_subfunc.sortl)[2],
1862	((unsigned long *) &kvm_s390_available_subfunc.sortl)[3]);
1863	VM_EVENT(kvm, 3, "GET: host DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1864	((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[0],
1865	((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[1],
1866	((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[2],
1867	((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[3]);
1868	VM_EVENT(kvm, 3, "GET: host PFCR subfunc 0x%16.16lx.%16.16lx",
1869	((unsigned long *) &kvm_s390_available_subfunc.pfcr)[0],
1870	((unsigned long *) &kvm_s390_available_subfunc.pfcr)[1]);
1871
1872	return 0;
1873	}
1874
1875	static int kvm_s390_get_processor_uv_feat(struct kvm *kvm, struct kvm_device_attr *attr)
1876	{
1877	struct kvm_s390_vm_cpu_uv_feat __user *dst = (void __user *)attr->addr;
1878	unsigned long feat = kvm->arch.model.uv_feat_guest.feat;
1879
1880	if (put_user(feat, &dst->feat))
1881	return -EFAULT;
1882	VM_EVENT(kvm, 3, "GET: guest UV-feat: 0x%16.16lx", feat);
1883
1884	return 0;
1885	}
1886
1887	static int kvm_s390_get_machine_uv_feat(struct kvm *kvm, struct kvm_device_attr *attr)
1888	{
1889	struct kvm_s390_vm_cpu_uv_feat __user *dst = (void __user *)attr->addr;
1890	unsigned long feat;
1891
1892	BUILD_BUG_ON(sizeof(*dst) != sizeof(uv_info.uv_feature_indications));
1893
1894	feat = uv_info.uv_feature_indications & KVM_S390_VM_CPU_UV_FEAT_GUEST_MASK;
1895	if (put_user(feat, &dst->feat))
1896	return -EFAULT;
1897	VM_EVENT(kvm, 3, "GET: guest UV-feat: 0x%16.16lx", feat);
1898
1899	return 0;
1900	}
1901
1902	static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1903	{
1904	int ret = -ENXIO;
1905
1906	switch (attr->attr) {
1907	case KVM_S390_VM_CPU_PROCESSOR:
1908	ret = kvm_s390_get_processor(kvm, attr);
1909	break;
1910	case KVM_S390_VM_CPU_MACHINE:
1911	ret = kvm_s390_get_machine(kvm, attr);
1912	break;
1913	case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1914	ret = kvm_s390_get_processor_feat(kvm, attr);
1915	break;
1916	case KVM_S390_VM_CPU_MACHINE_FEAT:
1917	ret = kvm_s390_get_machine_feat(kvm, attr);
1918	break;
1919	case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1920	ret = kvm_s390_get_processor_subfunc(kvm, attr);
1921	break;
1922	case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1923	ret = kvm_s390_get_machine_subfunc(kvm, attr);
1924	break;
1925	case KVM_S390_VM_CPU_PROCESSOR_UV_FEAT_GUEST:
1926	ret = kvm_s390_get_processor_uv_feat(kvm, attr);
1927	break;
1928	case KVM_S390_VM_CPU_MACHINE_UV_FEAT_GUEST:
1929	ret = kvm_s390_get_machine_uv_feat(kvm, attr);
1930	break;
1931	}
1932	return ret;
1933	}
1934
1935	/**
1936	* kvm_s390_update_topology_change_report - update CPU topology change report
1937	* @kvm: guest KVM description
1938	* @val: set or clear the MTCR bit
1939	*
1940	* Updates the Multiprocessor Topology-Change-Report bit to signal
1941	* the guest with a topology change.
1942	* This is only relevant if the topology facility is present.
1943	*/
1944	static void kvm_s390_update_topology_change_report(struct kvm *kvm, bool val)
1945	{
1946	union sca_utility new, old;
1947	struct esca_block *sca;
1948
1949	sca = kvm->arch.sca;
1950	old = READ_ONCE(sca->utility);
1951	do {
1952	new = old;
1953	new.mtcr = val;
1954	} while (!try_cmpxchg(&sca->utility.val, &old.val, new.val));
1955	}
1956
1957	static int kvm_s390_set_topo_change_indication(struct kvm *kvm,
1958	struct kvm_device_attr *attr)
1959	{
1960	if (!test_kvm_facility(kvm, nr: 11))
1961	return -ENXIO;
1962
1963	kvm_s390_update_topology_change_report(kvm, val: !!attr->attr);
1964	return 0;
1965	}
1966
1967	static int kvm_s390_get_topo_change_indication(struct kvm *kvm,
1968	struct kvm_device_attr *attr)
1969	{
1970	u8 topo;
1971
1972	if (!test_kvm_facility(kvm, nr: 11))
1973	return -ENXIO;
1974
1975	topo = kvm->arch.sca->utility.mtcr;
1976
1977	return put_user(topo, (u8 __user *)attr->addr);
1978	}
1979
1980	static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1981	{
1982	int ret;
1983
1984	switch (attr->group) {
1985	case KVM_S390_VM_MEM_CTRL:
1986	ret = kvm_s390_set_mem_control(kvm, attr);
1987	break;
1988	case KVM_S390_VM_TOD:
1989	ret = kvm_s390_set_tod(kvm, attr);
1990	break;
1991	case KVM_S390_VM_CPU_MODEL:
1992	ret = kvm_s390_set_cpu_model(kvm, attr);
1993	break;
1994	case KVM_S390_VM_CRYPTO:
1995	ret = kvm_s390_vm_set_crypto(kvm, attr);
1996	break;
1997	case KVM_S390_VM_MIGRATION:
1998	ret = kvm_s390_vm_set_migration(kvm, attr);
1999	break;
2000	case KVM_S390_VM_CPU_TOPOLOGY:
2001	ret = kvm_s390_set_topo_change_indication(kvm, attr);
2002	break;
2003	default:
2004	ret = -ENXIO;
2005	break;
2006	}
2007
2008	return ret;
2009	}
2010
2011	static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
2012	{
2013	int ret;
2014
2015	switch (attr->group) {
2016	case KVM_S390_VM_MEM_CTRL:
2017	ret = kvm_s390_get_mem_control(kvm, attr);
2018	break;
2019	case KVM_S390_VM_TOD:
2020	ret = kvm_s390_get_tod(kvm, attr);
2021	break;
2022	case KVM_S390_VM_CPU_MODEL:
2023	ret = kvm_s390_get_cpu_model(kvm, attr);
2024	break;
2025	case KVM_S390_VM_MIGRATION:
2026	ret = kvm_s390_vm_get_migration(kvm, attr);
2027	break;
2028	case KVM_S390_VM_CPU_TOPOLOGY:
2029	ret = kvm_s390_get_topo_change_indication(kvm, attr);
2030	break;
2031	default:
2032	ret = -ENXIO;
2033	break;
2034	}
2035
2036	return ret;
2037	}
2038
2039	static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
2040	{
2041	int ret;
2042
2043	switch (attr->group) {
2044	case KVM_S390_VM_MEM_CTRL:
2045	switch (attr->attr) {
2046	case KVM_S390_VM_MEM_ENABLE_CMMA:
2047	case KVM_S390_VM_MEM_CLR_CMMA:
2048	ret = sclp.has_cmma ? 0 : -ENXIO;
2049	break;
2050	case KVM_S390_VM_MEM_LIMIT_SIZE:
2051	ret = 0;
2052	break;
2053	default:
2054	ret = -ENXIO;
2055	break;
2056	}
2057	break;
2058	case KVM_S390_VM_TOD:
2059	switch (attr->attr) {
2060	case KVM_S390_VM_TOD_LOW:
2061	case KVM_S390_VM_TOD_HIGH:
2062	ret = 0;
2063	break;
2064	default:
2065	ret = -ENXIO;
2066	break;
2067	}
2068	break;
2069	case KVM_S390_VM_CPU_MODEL:
2070	switch (attr->attr) {
2071	case KVM_S390_VM_CPU_PROCESSOR:
2072	case KVM_S390_VM_CPU_MACHINE:
2073	case KVM_S390_VM_CPU_PROCESSOR_FEAT:
2074	case KVM_S390_VM_CPU_MACHINE_FEAT:
2075	case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
2076	case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
2077	case KVM_S390_VM_CPU_MACHINE_UV_FEAT_GUEST:
2078	case KVM_S390_VM_CPU_PROCESSOR_UV_FEAT_GUEST:
2079	ret = 0;
2080	break;
2081	default:
2082	ret = -ENXIO;
2083	break;
2084	}
2085	break;
2086	case KVM_S390_VM_CRYPTO:
2087	switch (attr->attr) {
2088	case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
2089	case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
2090	case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
2091	case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
2092	ret = 0;
2093	break;
2094	case KVM_S390_VM_CRYPTO_ENABLE_APIE:
2095	case KVM_S390_VM_CRYPTO_DISABLE_APIE:
2096	ret = ap_instructions_available() ? 0 : -ENXIO;
2097	break;
2098	default:
2099	ret = -ENXIO;
2100	break;
2101	}
2102	break;
2103	case KVM_S390_VM_MIGRATION:
2104	ret = 0;
2105	break;
2106	case KVM_S390_VM_CPU_TOPOLOGY:
2107	ret = test_kvm_facility(kvm, nr: 11) ? 0 : -ENXIO;
2108	break;
2109	default:
2110	ret = -ENXIO;
2111	break;
2112	}
2113
2114	return ret;
2115	}
2116
2117	static int kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
2118	{
2119	uint8_t *keys;
2120	uint64_t hva;
2121	int srcu_idx, i, r = 0;
2122
2123	if (args->flags != 0)
2124	return -EINVAL;
2125
2126	/* Is this guest using storage keys? */
2127	if (!mm_uses_skeys(current->mm))
2128	return KVM_S390_GET_SKEYS_NONE;
2129
2130	/* Enforce sane limit on memory allocation */
2131	if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
2132	return -EINVAL;
2133
2134	keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL_ACCOUNT);
2135	if (!keys)
2136	return -ENOMEM;
2137
2138	mmap_read_lock(current->mm);
2139	srcu_idx = srcu_read_lock(ssp: &kvm->srcu);
2140	for (i = 0; i < args->count; i++) {
2141	hva = gfn_to_hva(kvm, gfn: args->start_gfn + i);
2142	if (kvm_is_error_hva(addr: hva)) {
2143	r = -EFAULT;
2144	break;
2145	}
2146
2147	r = get_guest_storage_key(current->mm, hva, &keys[i]);
2148	if (r)
2149	break;
2150	}
2151	srcu_read_unlock(ssp: &kvm->srcu, idx: srcu_idx);
2152	mmap_read_unlock(current->mm);
2153
2154	if (!r) {
2155	r = copy_to_user(to: (uint8_t __user *)args->skeydata_addr, from: keys,
2156	n: sizeof(uint8_t) * args->count);
2157	if (r)
2158	r = -EFAULT;
2159	}
2160
2161	kvfree(addr: keys);
2162	return r;
2163	}
2164
2165	static int kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
2166	{
2167	uint8_t *keys;
2168	uint64_t hva;
2169	int srcu_idx, i, r = 0;
2170	bool unlocked;
2171
2172	if (args->flags != 0)
2173	return -EINVAL;
2174
2175	/* Enforce sane limit on memory allocation */
2176	if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
2177	return -EINVAL;
2178
2179	keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL_ACCOUNT);
2180	if (!keys)
2181	return -ENOMEM;
2182
2183	r = copy_from_user(to: keys, from: (uint8_t __user *)args->skeydata_addr,
2184	n: sizeof(uint8_t) * args->count);
2185	if (r) {
2186	r = -EFAULT;
2187	goto out;
2188	}
2189
2190	/* Enable storage key handling for the guest */
2191	r = s390_enable_skey();
2192	if (r)
2193	goto out;
2194
2195	i = 0;
2196	mmap_read_lock(current->mm);
2197	srcu_idx = srcu_read_lock(ssp: &kvm->srcu);
2198	while (i < args->count) {
2199	unlocked = false;
2200	hva = gfn_to_hva(kvm, gfn: args->start_gfn + i);
2201	if (kvm_is_error_hva(addr: hva)) {
2202	r = -EFAULT;
2203	break;
2204	}
2205
2206	/* Lowest order bit is reserved */
2207	if (keys[i] & 0x01) {
2208	r = -EINVAL;
2209	break;
2210	}
2211
2212	r = set_guest_storage_key(current->mm, hva, keys[i], 0);
2213	if (r) {
2214	r = fixup_user_fault(current->mm, address: hva,
2215	fault_flags: FAULT_FLAG_WRITE, unlocked: &unlocked);
2216	if (r)
2217	break;
2218	}
2219	if (!r)
2220	i++;
2221	}
2222	srcu_read_unlock(ssp: &kvm->srcu, idx: srcu_idx);
2223	mmap_read_unlock(current->mm);
2224	out:
2225	kvfree(addr: keys);
2226	return r;
2227	}
2228
2229	/*
2230	* Base address and length must be sent at the start of each block, therefore
2231	* it's cheaper to send some clean data, as long as it's less than the size of
2232	* two longs.
2233	*/
2234	#define KVM_S390_MAX_BIT_DISTANCE (2 * sizeof(void *))
2235	/* for consistency */
2236	#define KVM_S390_CMMA_SIZE_MAX ((u32)KVM_S390_SKEYS_MAX)
2237
2238	static int kvm_s390_peek_cmma(struct kvm *kvm, struct kvm_s390_cmma_log *args,
2239	u8 *res, unsigned long bufsize)
2240	{
2241	unsigned long pgstev, hva, cur_gfn = args->start_gfn;
2242
2243	args->count = 0;
2244	while (args->count < bufsize) {
2245	hva = gfn_to_hva(kvm, gfn: cur_gfn);
2246	/*
2247	* We return an error if the first value was invalid, but we
2248	* return successfully if at least one value was copied.
2249	*/
2250	if (kvm_is_error_hva(addr: hva))
2251	return args->count ? 0 : -EFAULT;
2252	if (get_pgste(kvm->mm, hva, &pgstev) < 0)
2253	pgstev = 0;
2254	res[args->count++] = (pgstev >> 24) & 0x43;
2255	cur_gfn++;
2256	}
2257
2258	return 0;
2259	}
2260
2261	static struct kvm_memory_slot *gfn_to_memslot_approx(struct kvm_memslots *slots,
2262	gfn_t gfn)
2263	{
2264	return ____gfn_to_memslot(slots, gfn, approx: true);
2265	}
2266
2267	static unsigned long kvm_s390_next_dirty_cmma(struct kvm_memslots *slots,
2268	unsigned long cur_gfn)
2269	{
2270	struct kvm_memory_slot *ms = gfn_to_memslot_approx(slots, gfn: cur_gfn);
2271	unsigned long ofs = cur_gfn - ms->base_gfn;
2272	struct rb_node *mnode = &ms->gfn_node[slots->node_idx];
2273
2274	if (ms->base_gfn + ms->npages <= cur_gfn) {
2275	mnode = rb_next(mnode);
2276	/* If we are above the highest slot, wrap around */
2277	if (!mnode)
2278	mnode = rb_first(root: &slots->gfn_tree);
2279
2280	ms = container_of(mnode, struct kvm_memory_slot, gfn_node[slots->node_idx]);
2281	ofs = 0;
2282	}
2283
2284	if (cur_gfn < ms->base_gfn)
2285	ofs = 0;
2286
2287	ofs = find_next_bit(addr: kvm_second_dirty_bitmap(memslot: ms), size: ms->npages, offset: ofs);
2288	while (ofs >= ms->npages && (mnode = rb_next(mnode))) {
2289	ms = container_of(mnode, struct kvm_memory_slot, gfn_node[slots->node_idx]);
2290	ofs = find_first_bit(addr: kvm_second_dirty_bitmap(memslot: ms), size: ms->npages);
2291	}
2292	return ms->base_gfn + ofs;
2293	}
2294
2295	static int kvm_s390_get_cmma(struct kvm *kvm, struct kvm_s390_cmma_log *args,
2296	u8 *res, unsigned long bufsize)
2297	{
2298	unsigned long mem_end, cur_gfn, next_gfn, hva, pgstev;
2299	struct kvm_memslots *slots = kvm_memslots(kvm);
2300	struct kvm_memory_slot *ms;
2301
2302	if (unlikely(kvm_memslots_empty(slots)))
2303	return 0;
2304
2305	cur_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn: args->start_gfn);
2306	ms = gfn_to_memslot(kvm, gfn: cur_gfn);
2307	args->count = 0;
2308	args->start_gfn = cur_gfn;
2309	if (!ms)
2310	return 0;
2311	next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn: cur_gfn + 1);
2312	mem_end = kvm_s390_get_gfn_end(slots);
2313
2314	while (args->count < bufsize) {
2315	hva = gfn_to_hva(kvm, gfn: cur_gfn);
2316	if (kvm_is_error_hva(addr: hva))
2317	return 0;
2318	/* Decrement only if we actually flipped the bit to 0 */
2319	if (test_and_clear_bit(nr: cur_gfn - ms->base_gfn, addr: kvm_second_dirty_bitmap(memslot: ms)))
2320	atomic64_dec(v: &kvm->arch.cmma_dirty_pages);
2321	if (get_pgste(kvm->mm, hva, &pgstev) < 0)
2322	pgstev = 0;
2323	/* Save the value */
2324	res[args->count++] = (pgstev >> 24) & 0x43;
2325	/* If the next bit is too far away, stop. */
2326	if (next_gfn > cur_gfn + KVM_S390_MAX_BIT_DISTANCE)
2327	return 0;
2328	/* If we reached the previous "next", find the next one */
2329	if (cur_gfn == next_gfn)
2330	next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn: cur_gfn + 1);
2331	/* Reached the end of memory or of the buffer, stop */
2332	if ((next_gfn >= mem_end) ||
2333	(next_gfn - args->start_gfn >= bufsize))
2334	return 0;
2335	cur_gfn++;
2336	/* Reached the end of the current memslot, take the next one. */
2337	if (cur_gfn - ms->base_gfn >= ms->npages) {
2338	ms = gfn_to_memslot(kvm, gfn: cur_gfn);
2339	if (!ms)
2340	return 0;
2341	}
2342	}
2343	return 0;
2344	}
2345
2346	/*
2347	* This function searches for the next page with dirty CMMA attributes, and
2348	* saves the attributes in the buffer up to either the end of the buffer or
2349	* until a block of at least KVM_S390_MAX_BIT_DISTANCE clean bits is found;
2350	* no trailing clean bytes are saved.
2351	* In case no dirty bits were found, or if CMMA was not enabled or used, the
2352	* output buffer will indicate 0 as length.
2353	*/
2354	static int kvm_s390_get_cmma_bits(struct kvm *kvm,
2355	struct kvm_s390_cmma_log *args)
2356	{
2357	unsigned long bufsize;
2358	int srcu_idx, peek, ret;
2359	u8 *values;
2360
2361	if (!kvm->arch.use_cmma)
2362	return -ENXIO;
2363	/* Invalid/unsupported flags were specified */
2364	if (args->flags & ~KVM_S390_CMMA_PEEK)
2365	return -EINVAL;
2366	/* Migration mode query, and we are not doing a migration */
2367	peek = !!(args->flags & KVM_S390_CMMA_PEEK);
2368	if (!peek && !kvm->arch.migration_mode)
2369	return -EINVAL;
2370	/* CMMA is disabled or was not used, or the buffer has length zero */
2371	bufsize = min(args->count, KVM_S390_CMMA_SIZE_MAX);
2372	if (!bufsize || !kvm->mm->context.uses_cmm) {
2373	memset(args, 0, sizeof(*args));
2374	return 0;
2375	}
2376	/* We are not peeking, and there are no dirty pages */
2377	if (!peek && !atomic64_read(v: &kvm->arch.cmma_dirty_pages)) {
2378	memset(args, 0, sizeof(*args));
2379	return 0;
2380	}
2381
2382	values = vmalloc(bufsize);
2383	if (!values)
2384	return -ENOMEM;
2385
2386	mmap_read_lock(mm: kvm->mm);
2387	srcu_idx = srcu_read_lock(ssp: &kvm->srcu);
2388	if (peek)
2389	ret = kvm_s390_peek_cmma(kvm, args, res: values, bufsize);
2390	else
2391	ret = kvm_s390_get_cmma(kvm, args, res: values, bufsize);
2392	srcu_read_unlock(ssp: &kvm->srcu, idx: srcu_idx);
2393	mmap_read_unlock(mm: kvm->mm);
2394
2395	if (kvm->arch.migration_mode)
2396	args->remaining = atomic64_read(v: &kvm->arch.cmma_dirty_pages);
2397	else
2398	args->remaining = 0;
2399
2400	if (copy_to_user(to: (void __user *)args->values, from: values, n: args->count))
2401	ret = -EFAULT;
2402
2403	vfree(addr: values);
2404	return ret;
2405	}
2406
2407	/*
2408	* This function sets the CMMA attributes for the given pages. If the input
2409	* buffer has zero length, no action is taken, otherwise the attributes are
2410	* set and the mm->context.uses_cmm flag is set.
2411	*/
2412	static int kvm_s390_set_cmma_bits(struct kvm *kvm,
2413	const struct kvm_s390_cmma_log *args)
2414	{
2415	unsigned long hva, mask, pgstev, i;
2416	uint8_t *bits;
2417	int srcu_idx, r = 0;
2418
2419	mask = args->mask;
2420
2421	if (!kvm->arch.use_cmma)
2422	return -ENXIO;
2423	/* invalid/unsupported flags */
2424	if (args->flags != 0)
2425	return -EINVAL;
2426	/* Enforce sane limit on memory allocation */
2427	if (args->count > KVM_S390_CMMA_SIZE_MAX)
2428	return -EINVAL;
2429	/* Nothing to do */
2430	if (args->count == 0)
2431	return 0;
2432
2433	bits = vmalloc(array_size(sizeof(*bits), args->count));
2434	if (!bits)
2435	return -ENOMEM;
2436
2437	r = copy_from_user(to: bits, from: (void __user *)args->values, n: args->count);
2438	if (r) {
2439	r = -EFAULT;
2440	goto out;
2441	}
2442
2443	mmap_read_lock(mm: kvm->mm);
2444	srcu_idx = srcu_read_lock(ssp: &kvm->srcu);
2445	for (i = 0; i < args->count; i++) {
2446	hva = gfn_to_hva(kvm, gfn: args->start_gfn + i);
2447	if (kvm_is_error_hva(addr: hva)) {
2448	r = -EFAULT;
2449	break;
2450	}
2451
2452	pgstev = bits[i];
2453	pgstev = pgstev << 24;
2454	mask &= _PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT;
2455	set_pgste_bits(kvm->mm, hva, mask, pgstev);
2456	}
2457	srcu_read_unlock(ssp: &kvm->srcu, idx: srcu_idx);
2458	mmap_read_unlock(mm: kvm->mm);
2459
2460	if (!kvm->mm->context.uses_cmm) {
2461	mmap_write_lock(mm: kvm->mm);
2462	kvm->mm->context.uses_cmm = 1;
2463	mmap_write_unlock(mm: kvm->mm);
2464	}
2465	out:
2466	vfree(addr: bits);
2467	return r;
2468	}
2469
2470	/**
2471	* kvm_s390_cpus_from_pv - Convert all protected vCPUs in a protected VM to
2472	* non protected.
2473	* @kvm: the VM whose protected vCPUs are to be converted
2474	* @rc: return value for the RC field of the UVC (in case of error)
2475	* @rrc: return value for the RRC field of the UVC (in case of error)
2476	*
2477	* Does not stop in case of error, tries to convert as many
2478	* CPUs as possible. In case of error, the RC and RRC of the last error are
2479	* returned.
2480	*
2481	* Return: 0 in case of success, otherwise -EIO
2482	*/
2483	int kvm_s390_cpus_from_pv(struct kvm *kvm, u16 *rc, u16 *rrc)
2484	{
2485	struct kvm_vcpu *vcpu;
2486	unsigned long i;
2487	u16 _rc, _rrc;
2488	int ret = 0;
2489
2490	/*
2491	* We ignore failures and try to destroy as many CPUs as possible.
2492	* At the same time we must not free the assigned resources when
2493	* this fails, as the ultravisor has still access to that memory.
2494	* So kvm_s390_pv_destroy_cpu can leave a "wanted" memory leak
2495	* behind.
2496	* We want to return the first failure rc and rrc, though.
2497	*/
2498	kvm_for_each_vcpu(i, vcpu, kvm) {
2499	mutex_lock(&vcpu->mutex);
2500	if (kvm_s390_pv_destroy_cpu(vcpu, rc: &_rc, rrc: &_rrc) && !ret) {
2501	*rc = _rc;
2502	*rrc = _rrc;
2503	ret = -EIO;
2504	}
2505	mutex_unlock(lock: &vcpu->mutex);
2506	}
2507	/* Ensure that we re-enable gisa if the non-PV guest used it but the PV guest did not. */
2508	if (use_gisa)
2509	kvm_s390_gisa_enable(kvm);
2510	return ret;
2511	}
2512
2513	/**
2514	* kvm_s390_cpus_to_pv - Convert all non-protected vCPUs in a protected VM
2515	* to protected.
2516	* @kvm: the VM whose protected vCPUs are to be converted
2517	* @rc: return value for the RC field of the UVC (in case of error)
2518	* @rrc: return value for the RRC field of the UVC (in case of error)
2519	*
2520	* Tries to undo the conversion in case of error.
2521	*
2522	* Return: 0 in case of success, otherwise -EIO
2523	*/
2524	static int kvm_s390_cpus_to_pv(struct kvm *kvm, u16 *rc, u16 *rrc)
2525	{
2526	unsigned long i;
2527	int r = 0;
2528	u16 dummy;
2529
2530	struct kvm_vcpu *vcpu;
2531
2532	/* Disable the GISA if the ultravisor does not support AIV. */
2533	if (!uv_has_feature(BIT_UV_FEAT_AIV))
2534	kvm_s390_gisa_disable(kvm);
2535
2536	kvm_for_each_vcpu(i, vcpu, kvm) {
2537	mutex_lock(&vcpu->mutex);
2538	r = kvm_s390_pv_create_cpu(vcpu, rc, rrc);
2539	mutex_unlock(lock: &vcpu->mutex);
2540	if (r)
2541	break;
2542	}
2543	if (r)
2544	kvm_s390_cpus_from_pv(kvm, rc: &dummy, rrc: &dummy);
2545	return r;
2546	}
2547
2548	/*
2549	* Here we provide user space with a direct interface to query UV
2550	* related data like UV maxima and available features as well as
2551	* feature specific data.
2552	*
2553	* To facilitate future extension of the data structures we'll try to
2554	* write data up to the maximum requested length.
2555	*/
2556	static ssize_t kvm_s390_handle_pv_info(struct kvm_s390_pv_info *info)
2557	{
2558	ssize_t len_min;
2559
2560	switch (info->header.id) {
2561	case KVM_PV_INFO_VM: {
2562	len_min = sizeof(info->header) + sizeof(info->vm);
2563
2564	if (info->header.len_max < len_min)
2565	return -EINVAL;
2566
2567	memcpy(info->vm.inst_calls_list,
2568	uv_info.inst_calls_list,
2569	sizeof(uv_info.inst_calls_list));
2570
2571	/* It's max cpuid not max cpus, so it's off by one */
2572	info->vm.max_cpus = uv_info.max_guest_cpu_id + 1;
2573	info->vm.max_guests = uv_info.max_num_sec_conf;
2574	info->vm.max_guest_addr = uv_info.max_sec_stor_addr;
2575	info->vm.feature_indication = uv_info.uv_feature_indications;
2576
2577	return len_min;
2578	}
2579	case KVM_PV_INFO_DUMP: {
2580	len_min = sizeof(info->header) + sizeof(info->dump);
2581
2582	if (info->header.len_max < len_min)
2583	return -EINVAL;
2584
2585	info->dump.dump_cpu_buffer_len = uv_info.guest_cpu_stor_len;
2586	info->dump.dump_config_mem_buffer_per_1m = uv_info.conf_dump_storage_state_len;
2587	info->dump.dump_config_finalize_len = uv_info.conf_dump_finalize_len;
2588	return len_min;
2589	}
2590	default:
2591	return -EINVAL;
2592	}
2593	}
2594
2595	static int kvm_s390_pv_dmp(struct kvm *kvm, struct kvm_pv_cmd *cmd,
2596	struct kvm_s390_pv_dmp dmp)
2597	{
2598	int r = -EINVAL;
2599	void __user *result_buff = (void __user *)dmp.buff_addr;
2600
2601	switch (dmp.subcmd) {
2602	case KVM_PV_DUMP_INIT: {
2603	if (kvm->arch.pv.dumping)
2604	break;
2605
2606	/*
2607	* Block SIE entry as concurrent dump UVCs could lead
2608	* to validities.
2609	*/
2610	kvm_s390_vcpu_block_all(kvm);
2611
2612	r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
2613	UVC_CMD_DUMP_INIT, &cmd->rc, &cmd->rrc);
2614	KVM_UV_EVENT(kvm, 3, "PROTVIRT DUMP INIT: rc %x rrc %x",
2615	cmd->rc, cmd->rrc);
2616	if (!r) {
2617	kvm->arch.pv.dumping = true;
2618	} else {
2619	kvm_s390_vcpu_unblock_all(kvm);
2620	r = -EINVAL;
2621	}
2622	break;
2623	}
2624	case KVM_PV_DUMP_CONFIG_STOR_STATE: {
2625	if (!kvm->arch.pv.dumping)
2626	break;
2627
2628	/*
2629	* gaddr is an output parameter since we might stop
2630	* early. As dmp will be copied back in our caller, we
2631	* don't need to do it ourselves.
2632	*/
2633	r = kvm_s390_pv_dump_stor_state(kvm, buff_user: result_buff, gaddr: &dmp.gaddr, buff_user_len: dmp.buff_len,
2634	rc: &cmd->rc, rrc: &cmd->rrc);
2635	break;
2636	}
2637	case KVM_PV_DUMP_COMPLETE: {
2638	if (!kvm->arch.pv.dumping)
2639	break;
2640
2641	r = -EINVAL;
2642	if (dmp.buff_len < uv_info.conf_dump_finalize_len)
2643	break;
2644
2645	r = kvm_s390_pv_dump_complete(kvm, buff_user: result_buff,
2646	rc: &cmd->rc, rrc: &cmd->rrc);
2647	break;
2648	}
2649	default:
2650	r = -ENOTTY;
2651	break;
2652	}
2653
2654	return r;
2655	}
2656
2657	static int kvm_s390_handle_pv(struct kvm *kvm, struct kvm_pv_cmd *cmd)
2658	{
2659	const bool need_lock = (cmd->cmd != KVM_PV_ASYNC_CLEANUP_PERFORM);
2660	void __user *argp = (void __user *)cmd->data;
2661	int r = 0;
2662	u16 dummy;
2663
2664	if (need_lock)
2665	mutex_lock(&kvm->lock);
2666
2667	switch (cmd->cmd) {
2668	case KVM_PV_ENABLE: {
2669	r = -EINVAL;
2670	if (kvm_s390_pv_is_protected(kvm))
2671	break;
2672
2673	mmap_write_lock(mm: kvm->mm);
2674	r = gmap_helper_disable_cow_sharing();
2675	mmap_write_unlock(mm: kvm->mm);
2676	if (r)
2677	break;
2678
2679	r = kvm_s390_pv_init_vm(kvm, rc: &cmd->rc, rrc: &cmd->rrc);
2680	if (r)
2681	break;
2682
2683	r = kvm_s390_cpus_to_pv(kvm, rc: &cmd->rc, rrc: &cmd->rrc);
2684	if (r)
2685	kvm_s390_pv_deinit_vm(kvm, rc: &dummy, rrc: &dummy);
2686
2687	/* we need to block service interrupts from now on */
2688	set_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs);
2689	break;
2690	}
2691	case KVM_PV_ASYNC_CLEANUP_PREPARE:
2692	r = -EINVAL;
2693	if (!kvm_s390_pv_is_protected(kvm) || !async_destroy)
2694	break;
2695
2696	r = kvm_s390_cpus_from_pv(kvm, rc: &cmd->rc, rrc: &cmd->rrc);
2697	/*
2698	* If a CPU could not be destroyed, destroy VM will also fail.
2699	* There is no point in trying to destroy it. Instead return
2700	* the rc and rrc from the first CPU that failed destroying.
2701	*/
2702	if (r)
2703	break;
2704	r = kvm_s390_pv_set_aside(kvm, rc: &cmd->rc, rrc: &cmd->rrc);
2705
2706	/* no need to block service interrupts any more */
2707	clear_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs);
2708	break;
2709	case KVM_PV_ASYNC_CLEANUP_PERFORM:
2710	r = -EINVAL;
2711	if (!async_destroy)
2712	break;
2713	/* kvm->lock must not be held; this is asserted inside the function. */
2714	r = kvm_s390_pv_deinit_aside_vm(kvm, rc: &cmd->rc, rrc: &cmd->rrc);
2715	break;
2716	case KVM_PV_DISABLE: {
2717	r = -EINVAL;
2718	if (!kvm_s390_pv_is_protected(kvm))
2719	break;
2720
2721	r = kvm_s390_cpus_from_pv(kvm, rc: &cmd->rc, rrc: &cmd->rrc);
2722	/*
2723	* If a CPU could not be destroyed, destroy VM will also fail.
2724	* There is no point in trying to destroy it. Instead return
2725	* the rc and rrc from the first CPU that failed destroying.
2726	*/
2727	if (r)
2728	break;
2729	r = kvm_s390_pv_deinit_cleanup_all(kvm, rc: &cmd->rc, rrc: &cmd->rrc);
2730
2731	/* no need to block service interrupts any more */
2732	clear_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs);
2733	break;
2734	}
2735	case KVM_PV_SET_SEC_PARMS: {
2736	struct kvm_s390_pv_sec_parm parms = {};
2737	void *hdr;
2738
2739	r = -EINVAL;
2740	if (!kvm_s390_pv_is_protected(kvm))
2741	break;
2742
2743	r = -EFAULT;
2744	if (copy_from_user(to: &parms, from: argp, n: sizeof(parms)))
2745	break;
2746
2747	/* Currently restricted to 8KB */
2748	r = -EINVAL;
2749	if (parms.length > PAGE_SIZE * 2)
2750	break;
2751
2752	r = -ENOMEM;
2753	hdr = vmalloc(parms.length);
2754	if (!hdr)
2755	break;
2756
2757	r = -EFAULT;
2758	if (!copy_from_user(to: hdr, from: (void __user *)parms.origin,
2759	n: parms.length))
2760	r = kvm_s390_pv_set_sec_parms(kvm, hdr, length: parms.length,
2761	rc: &cmd->rc, rrc: &cmd->rrc);
2762
2763	vfree(addr: hdr);
2764	break;
2765	}
2766	case KVM_PV_UNPACK: {
2767	struct kvm_s390_pv_unp unp = {};
2768
2769	r = -EINVAL;
2770	if (!kvm_s390_pv_is_protected(kvm) || !mm_is_protected(kvm->mm))
2771	break;
2772
2773	r = -EFAULT;
2774	if (copy_from_user(to: &unp, from: argp, n: sizeof(unp)))
2775	break;
2776
2777	r = kvm_s390_pv_unpack(kvm, addr: unp.addr, size: unp.size, tweak: unp.tweak,
2778	rc: &cmd->rc, rrc: &cmd->rrc);
2779	break;
2780	}
2781	case KVM_PV_VERIFY: {
2782	r = -EINVAL;
2783	if (!kvm_s390_pv_is_protected(kvm))
2784	break;
2785
2786	r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
2787	UVC_CMD_VERIFY_IMG, &cmd->rc, &cmd->rrc);
2788	KVM_UV_EVENT(kvm, 3, "PROTVIRT VERIFY: rc %x rrc %x", cmd->rc,
2789	cmd->rrc);
2790	break;
2791	}
2792	case KVM_PV_PREP_RESET: {
2793	r = -EINVAL;
2794	if (!kvm_s390_pv_is_protected(kvm))
2795	break;
2796
2797	r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
2798	UVC_CMD_PREPARE_RESET, &cmd->rc, &cmd->rrc);
2799	KVM_UV_EVENT(kvm, 3, "PROTVIRT PREP RESET: rc %x rrc %x",
2800	cmd->rc, cmd->rrc);
2801	break;
2802	}
2803	case KVM_PV_UNSHARE_ALL: {
2804	r = -EINVAL;
2805	if (!kvm_s390_pv_is_protected(kvm))
2806	break;
2807
2808	r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
2809	UVC_CMD_SET_UNSHARE_ALL, &cmd->rc, &cmd->rrc);
2810	KVM_UV_EVENT(kvm, 3, "PROTVIRT UNSHARE: rc %x rrc %x",
2811	cmd->rc, cmd->rrc);
2812	break;
2813	}
2814	case KVM_PV_INFO: {
2815	struct kvm_s390_pv_info info = {};
2816	ssize_t data_len;
2817
2818	/*
2819	* No need to check the VM protection here.
2820	*
2821	* Maybe user space wants to query some of the data
2822	* when the VM is still unprotected. If we see the
2823	* need to fence a new data command we can still
2824	* return an error in the info handler.
2825	*/
2826
2827	r = -EFAULT;
2828	if (copy_from_user(to: &info, from: argp, n: sizeof(info.header)))
2829	break;
2830
2831	r = -EINVAL;
2832	if (info.header.len_max < sizeof(info.header))
2833	break;
2834
2835	data_len = kvm_s390_handle_pv_info(info: &info);
2836	if (data_len < 0) {
2837	r = data_len;
2838	break;
2839	}
2840	/*
2841	* If a data command struct is extended (multiple
2842	* times) this can be used to determine how much of it
2843	* is valid.
2844	*/
2845	info.header.len_written = data_len;
2846
2847	r = -EFAULT;
2848	if (copy_to_user(to: argp, from: &info, n: data_len))
2849	break;
2850
2851	r = 0;
2852	break;
2853	}
2854	case KVM_PV_DUMP: {
2855	struct kvm_s390_pv_dmp dmp;
2856
2857	r = -EINVAL;
2858	if (!kvm_s390_pv_is_protected(kvm))
2859	break;
2860
2861	r = -EFAULT;
2862	if (copy_from_user(to: &dmp, from: argp, n: sizeof(dmp)))
2863	break;
2864
2865	r = kvm_s390_pv_dmp(kvm, cmd, dmp: dmp);
2866	if (r)
2867	break;
2868
2869	if (copy_to_user(to: argp, from: &dmp, n: sizeof(dmp))) {
2870	r = -EFAULT;
2871	break;
2872	}
2873
2874	break;
2875	}
2876	default:
2877	r = -ENOTTY;
2878	}
2879	if (need_lock)
2880	mutex_unlock(lock: &kvm->lock);
2881
2882	return r;
2883	}
2884
2885	static int mem_op_validate_common(struct kvm_s390_mem_op *mop, u64 supported_flags)
2886	{
2887	if (mop->flags & ~supported_flags || !mop->size)
2888	return -EINVAL;
2889	if (mop->size > MEM_OP_MAX_SIZE)
2890	return -E2BIG;
2891	if (mop->flags & KVM_S390_MEMOP_F_SKEY_PROTECTION) {
2892	if (mop->key > 0xf)
2893	return -EINVAL;
2894	} else {
2895	mop->key = 0;
2896	}
2897	return 0;
2898	}
2899
2900	static int kvm_s390_vm_mem_op_abs(struct kvm *kvm, struct kvm_s390_mem_op *mop)
2901	{
2902	void __user *uaddr = (void __user *)mop->buf;
2903	enum gacc_mode acc_mode;
2904	void *tmpbuf = NULL;
2905	int r, srcu_idx;
2906
2907	r = mem_op_validate_common(mop, KVM_S390_MEMOP_F_SKEY_PROTECTION |
2908	KVM_S390_MEMOP_F_CHECK_ONLY);
2909	if (r)
2910	return r;
2911
2912	if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2913	tmpbuf = vmalloc(mop->size);
2914	if (!tmpbuf)
2915	return -ENOMEM;
2916	}
2917
2918	srcu_idx = srcu_read_lock(ssp: &kvm->srcu);
2919
2920	if (!kvm_is_gpa_in_memslot(kvm, gpa: mop->gaddr)) {
2921	r = PGM_ADDRESSING;
2922	goto out_unlock;
2923	}
2924
2925	acc_mode = mop->op == KVM_S390_MEMOP_ABSOLUTE_READ ? GACC_FETCH : GACC_STORE;
2926	if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2927	r = check_gpa_range(kvm, gpa: mop->gaddr, length: mop->size, mode: acc_mode, access_key: mop->key);
2928	goto out_unlock;
2929	}
2930	if (acc_mode == GACC_FETCH) {
2931	r = access_guest_abs_with_key(kvm, gpa: mop->gaddr, data: tmpbuf,
2932	len: mop->size, mode: GACC_FETCH, access_key: mop->key);
2933	if (r)
2934	goto out_unlock;
2935	if (copy_to_user(to: uaddr, from: tmpbuf, n: mop->size))
2936	r = -EFAULT;
2937	} else {
2938	if (copy_from_user(to: tmpbuf, from: uaddr, n: mop->size)) {
2939	r = -EFAULT;
2940	goto out_unlock;
2941	}
2942	r = access_guest_abs_with_key(kvm, gpa: mop->gaddr, data: tmpbuf,
2943	len: mop->size, mode: GACC_STORE, access_key: mop->key);
2944	}
2945
2946	out_unlock:
2947	srcu_read_unlock(ssp: &kvm->srcu, idx: srcu_idx);
2948
2949	vfree(addr: tmpbuf);
2950	return r;
2951	}
2952
2953	static int kvm_s390_vm_mem_op_cmpxchg(struct kvm *kvm, struct kvm_s390_mem_op *mop)
2954	{
2955	void __user *uaddr = (void __user *)mop->buf;
2956	void __user *old_addr = (void __user *)mop->old_addr;
2957	union {
2958	__uint128_t quad;
2959	char raw[sizeof(__uint128_t)];
2960	} old = { .quad = 0}, new = { .quad = 0 };
2961	unsigned int off_in_quad = sizeof(new) - mop->size;
2962	int r, srcu_idx;
2963	bool success;
2964
2965	r = mem_op_validate_common(mop, KVM_S390_MEMOP_F_SKEY_PROTECTION);
2966	if (r)
2967	return r;
2968	/*
2969	* This validates off_in_quad. Checking that size is a power
2970	* of two is not necessary, as cmpxchg_guest_abs_with_key
2971	* takes care of that
2972	*/
2973	if (mop->size > sizeof(new))
2974	return -EINVAL;
2975	if (copy_from_user(to: &new.raw[off_in_quad], from: uaddr, n: mop->size))
2976	return -EFAULT;
2977	if (copy_from_user(to: &old.raw[off_in_quad], from: old_addr, n: mop->size))
2978	return -EFAULT;
2979
2980	srcu_idx = srcu_read_lock(ssp: &kvm->srcu);
2981
2982	if (!kvm_is_gpa_in_memslot(kvm, gpa: mop->gaddr)) {
2983	r = PGM_ADDRESSING;
2984	goto out_unlock;
2985	}
2986
2987	r = cmpxchg_guest_abs_with_key(kvm, gpa: mop->gaddr, len: mop->size, old: &old.quad,
2988	new: new.quad, access_key: mop->key, success: &success);
2989	if (!success && copy_to_user(to: old_addr, from: &old.raw[off_in_quad], n: mop->size))
2990	r = -EFAULT;
2991
2992	out_unlock:
2993	srcu_read_unlock(ssp: &kvm->srcu, idx: srcu_idx);
2994	return r;
2995	}
2996
2997	static int kvm_s390_vm_mem_op(struct kvm *kvm, struct kvm_s390_mem_op *mop)
2998	{
2999	/*
3000	* This is technically a heuristic only, if the kvm->lock is not
3001	* taken, it is not guaranteed that the vm is/remains non-protected.
3002	* This is ok from a kernel perspective, wrongdoing is detected
3003	* on the access, -EFAULT is returned and the vm may crash the
3004	* next time it accesses the memory in question.
3005	* There is no sane usecase to do switching and a memop on two
3006	* different CPUs at the same time.
3007	*/
3008	if (kvm_s390_pv_get_handle(kvm))
3009	return -EINVAL;
3010
3011	switch (mop->op) {
3012	case KVM_S390_MEMOP_ABSOLUTE_READ:
3013	case KVM_S390_MEMOP_ABSOLUTE_WRITE:
3014	return kvm_s390_vm_mem_op_abs(kvm, mop);
3015	case KVM_S390_MEMOP_ABSOLUTE_CMPXCHG:
3016	return kvm_s390_vm_mem_op_cmpxchg(kvm, mop);
3017	default:
3018	return -EINVAL;
3019	}
3020	}
3021
3022	int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
3023	{
3024	struct kvm *kvm = filp->private_data;
3025	void __user *argp = (void __user *)arg;
3026	struct kvm_device_attr attr;
3027	int r;
3028
3029	switch (ioctl) {
3030	case KVM_S390_INTERRUPT: {
3031	struct kvm_s390_interrupt s390int;
3032
3033	r = -EFAULT;
3034	if (copy_from_user(to: &s390int, from: argp, n: sizeof(s390int)))
3035	break;
3036	r = kvm_s390_inject_vm(kvm, s390int: &s390int);
3037	break;
3038	}
3039	case KVM_CREATE_IRQCHIP: {
3040	r = -EINVAL;
3041	if (kvm->arch.use_irqchip)
3042	r = 0;
3043	break;
3044	}
3045	case KVM_SET_DEVICE_ATTR: {
3046	r = -EFAULT;
3047	if (copy_from_user(to: &attr, from: (void __user *)arg, n: sizeof(attr)))
3048	break;
3049	r = kvm_s390_vm_set_attr(kvm, attr: &attr);
3050	break;
3051	}
3052	case KVM_GET_DEVICE_ATTR: {
3053	r = -EFAULT;
3054	if (copy_from_user(to: &attr, from: (void __user *)arg, n: sizeof(attr)))
3055	break;
3056	r = kvm_s390_vm_get_attr(kvm, attr: &attr);
3057	break;
3058	}
3059	case KVM_HAS_DEVICE_ATTR: {
3060	r = -EFAULT;
3061	if (copy_from_user(to: &attr, from: (void __user *)arg, n: sizeof(attr)))
3062	break;
3063	r = kvm_s390_vm_has_attr(kvm, attr: &attr);
3064	break;
3065	}
3066	case KVM_S390_GET_SKEYS: {
3067	struct kvm_s390_skeys args;
3068
3069	r = -EFAULT;
3070	if (copy_from_user(&args, argp,
3071	sizeof(struct kvm_s390_skeys)))
3072	break;
3073	r = kvm_s390_get_skeys(kvm, args: &args);
3074	break;
3075	}
3076	case KVM_S390_SET_SKEYS: {
3077	struct kvm_s390_skeys args;
3078
3079	r = -EFAULT;
3080	if (copy_from_user(&args, argp,
3081	sizeof(struct kvm_s390_skeys)))
3082	break;
3083	r = kvm_s390_set_skeys(kvm, args: &args);
3084	break;
3085	}
3086	case KVM_S390_GET_CMMA_BITS: {
3087	struct kvm_s390_cmma_log args;
3088
3089	r = -EFAULT;
3090	if (copy_from_user(to: &args, from: argp, n: sizeof(args)))
3091	break;
3092	mutex_lock(&kvm->slots_lock);
3093	r = kvm_s390_get_cmma_bits(kvm, args: &args);
3094	mutex_unlock(lock: &kvm->slots_lock);
3095	if (!r) {
3096	r = copy_to_user(to: argp, from: &args, n: sizeof(args));
3097	if (r)
3098	r = -EFAULT;
3099	}
3100	break;
3101	}
3102	case KVM_S390_SET_CMMA_BITS: {
3103	struct kvm_s390_cmma_log args;
3104
3105	r = -EFAULT;
3106	if (copy_from_user(to: &args, from: argp, n: sizeof(args)))
3107	break;
3108	mutex_lock(&kvm->slots_lock);
3109	r = kvm_s390_set_cmma_bits(kvm, args: &args);
3110	mutex_unlock(lock: &kvm->slots_lock);
3111	break;
3112	}
3113	case KVM_S390_PV_COMMAND: {
3114	struct kvm_pv_cmd args;
3115
3116	/* protvirt means user cpu state */
3117	kvm_s390_set_user_cpu_state_ctrl(kvm);
3118	r = 0;
3119	if (!is_prot_virt_host()) {
3120	r = -EINVAL;
3121	break;
3122	}
3123	if (copy_from_user(to: &args, from: argp, n: sizeof(args))) {
3124	r = -EFAULT;
3125	break;
3126	}
3127	if (args.flags) {
3128	r = -EINVAL;
3129	break;
3130	}
3131	/* must be called without kvm->lock */
3132	r = kvm_s390_handle_pv(kvm, cmd: &args);
3133	if (copy_to_user(to: argp, from: &args, n: sizeof(args))) {
3134	r = -EFAULT;
3135	break;
3136	}
3137	break;
3138	}
3139	case KVM_S390_MEM_OP: {
3140	struct kvm_s390_mem_op mem_op;
3141
3142	if (copy_from_user(to: &mem_op, from: argp, n: sizeof(mem_op)) == 0)
3143	r = kvm_s390_vm_mem_op(kvm, mop: &mem_op);
3144	else
3145	r = -EFAULT;
3146	break;
3147	}
3148	case KVM_S390_ZPCI_OP: {
3149	struct kvm_s390_zpci_op args;
3150
3151	r = -EINVAL;
3152	if (!IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM))
3153	break;
3154	if (copy_from_user(to: &args, from: argp, n: sizeof(args))) {
3155	r = -EFAULT;
3156	break;
3157	}
3158	r = kvm_s390_pci_zpci_op(kvm, args: &args);
3159	break;
3160	}
3161	default:
3162	r = -ENOTTY;
3163	}
3164
3165	return r;
3166	}
3167
3168	static int kvm_s390_apxa_installed(void)
3169	{
3170	struct ap_config_info info;
3171
3172	if (ap_instructions_available()) {
3173	if (ap_qci(&info) == 0)
3174	return info.apxa;
3175	}
3176
3177	return 0;
3178	}
3179
3180	/*
3181	* The format of the crypto control block (CRYCB) is specified in the 3 low
3182	* order bits of the CRYCB designation (CRYCBD) field as follows:
3183	* Format 0: Neither the message security assist extension 3 (MSAX3) nor the
3184	* AP extended addressing (APXA) facility are installed.
3185	* Format 1: The APXA facility is not installed but the MSAX3 facility is.
3186	* Format 2: Both the APXA and MSAX3 facilities are installed
3187	*/
3188	static void kvm_s390_set_crycb_format(struct kvm *kvm)
3189	{
3190	kvm->arch.crypto.crycbd = virt_to_phys(address: kvm->arch.crypto.crycb);
3191
3192	/* Clear the CRYCB format bits - i.e., set format 0 by default */
3193	kvm->arch.crypto.crycbd &= ~(CRYCB_FORMAT_MASK);
3194
3195	/* Check whether MSAX3 is installed */
3196	if (!test_kvm_facility(kvm, nr: 76))
3197	return;
3198
3199	if (kvm_s390_apxa_installed())
3200	kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
3201	else
3202	kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
3203	}
3204
3205	/*
3206	* kvm_arch_crypto_set_masks
3207	*
3208	* @kvm: pointer to the target guest's KVM struct containing the crypto masks
3209	* to be set.
3210	* @apm: the mask identifying the accessible AP adapters
3211	* @aqm: the mask identifying the accessible AP domains
3212	* @adm: the mask identifying the accessible AP control domains
3213	*
3214	* Set the masks that identify the adapters, domains and control domains to
3215	* which the KVM guest is granted access.
3216	*
3217	* Note: The kvm->lock mutex must be locked by the caller before invoking this
3218	* function.
3219	*/
3220	void kvm_arch_crypto_set_masks(struct kvm *kvm, unsigned long *apm,
3221	unsigned long *aqm, unsigned long *adm)
3222	{
3223	struct kvm_s390_crypto_cb *crycb = kvm->arch.crypto.crycb;
3224
3225	kvm_s390_vcpu_block_all(kvm);
3226
3227	switch (kvm->arch.crypto.crycbd & CRYCB_FORMAT_MASK) {
3228	case CRYCB_FORMAT2: /* APCB1 use 256 bits */
3229	memcpy(crycb->apcb1.apm, apm, 32);
3230	VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx %016lx %016lx %016lx",
3231	apm[0], apm[1], apm[2], apm[3]);
3232	memcpy(crycb->apcb1.aqm, aqm, 32);
3233	VM_EVENT(kvm, 3, "SET CRYCB: aqm %016lx %016lx %016lx %016lx",
3234	aqm[0], aqm[1], aqm[2], aqm[3]);
3235	memcpy(crycb->apcb1.adm, adm, 32);
3236	VM_EVENT(kvm, 3, "SET CRYCB: adm %016lx %016lx %016lx %016lx",
3237	adm[0], adm[1], adm[2], adm[3]);
3238	break;
3239	case CRYCB_FORMAT1:
3240	case CRYCB_FORMAT0: /* Fall through both use APCB0 */
3241	memcpy(crycb->apcb0.apm, apm, 8);
3242	memcpy(crycb->apcb0.aqm, aqm, 2);
3243	memcpy(crycb->apcb0.adm, adm, 2);
3244	VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx aqm %04x adm %04x",
3245	apm[0], *((unsigned short *)aqm),
3246	*((unsigned short *)adm));
3247	break;
3248	default: /* Can not happen */
3249	break;
3250	}
3251
3252	/* recreate the shadow crycb for each vcpu */
3253	kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
3254	kvm_s390_vcpu_unblock_all(kvm);
3255	}
3256	EXPORT_SYMBOL_GPL(kvm_arch_crypto_set_masks);
3257
3258	/*
3259	* kvm_arch_crypto_clear_masks
3260	*
3261	* @kvm: pointer to the target guest's KVM struct containing the crypto masks
3262	* to be cleared.
3263	*
3264	* Clear the masks that identify the adapters, domains and control domains to
3265	* which the KVM guest is granted access.
3266	*
3267	* Note: The kvm->lock mutex must be locked by the caller before invoking this
3268	* function.
3269	*/
3270	void kvm_arch_crypto_clear_masks(struct kvm *kvm)
3271	{
3272	kvm_s390_vcpu_block_all(kvm);
3273
3274	memset(&kvm->arch.crypto.crycb->apcb0, 0,
3275	sizeof(kvm->arch.crypto.crycb->apcb0));
3276	memset(&kvm->arch.crypto.crycb->apcb1, 0,
3277	sizeof(kvm->arch.crypto.crycb->apcb1));
3278
3279	VM_EVENT(kvm, 3, "%s", "CLR CRYCB:");
3280	/* recreate the shadow crycb for each vcpu */
3281	kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
3282	kvm_s390_vcpu_unblock_all(kvm);
3283	}
3284	EXPORT_SYMBOL_GPL(kvm_arch_crypto_clear_masks);
3285
3286	static u64 kvm_s390_get_initial_cpuid(void)
3287	{
3288	struct cpuid cpuid;
3289
3290	get_cpu_id(&cpuid);
3291	cpuid.version = 0xff;
3292	return *((u64 *) &cpuid);
3293	}
3294
3295	static void kvm_s390_crypto_init(struct kvm *kvm)
3296	{
3297	kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
3298	kvm_s390_set_crycb_format(kvm);
3299	init_rwsem(&kvm->arch.crypto.pqap_hook_rwsem);
3300
3301	if (!test_kvm_facility(kvm, nr: 76))
3302	return;
3303
3304	/* Enable AES/DEA protected key functions by default */
3305	kvm->arch.crypto.aes_kw = 1;
3306	kvm->arch.crypto.dea_kw = 1;
3307	get_random_bytes(buf: kvm->arch.crypto.crycb->aes_wrapping_key_mask,
3308	len: sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
3309	get_random_bytes(buf: kvm->arch.crypto.crycb->dea_wrapping_key_mask,
3310	len: sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
3311	}
3312
3313	static void sca_dispose(struct kvm *kvm)
3314	{
3315	free_pages_exact(virt: kvm->arch.sca, size: sizeof(*kvm->arch.sca));
3316	kvm->arch.sca = NULL;
3317	}
3318
3319	void kvm_arch_free_vm(struct kvm *kvm)
3320	{
3321	if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM))
3322	kvm_s390_pci_clear_list(kvm);
3323
3324	__kvm_arch_free_vm(kvm);
3325	}
3326
3327	int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
3328	{
3329	gfp_t alloc_flags = GFP_KERNEL_ACCOUNT | __GFP_ZERO;
3330	char debug_name[16];
3331	int i, rc;
3332
3333	rc = -EINVAL;
3334	#ifdef CONFIG_KVM_S390_UCONTROL
3335	if (type & ~KVM_VM_S390_UCONTROL)
3336	goto out_err;
3337	if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
3338	goto out_err;
3339	#else
3340	if (type)
3341	goto out_err;
3342	#endif
3343
3344	rc = s390_enable_sie();
3345	if (rc)
3346	goto out_err;
3347
3348	rc = -ENOMEM;
3349
3350	if (!sclp.has_64bscao)
3351	alloc_flags |= GFP_DMA;
3352	mutex_lock(&kvm_lock);
3353
3354	kvm->arch.sca = alloc_pages_exact(sizeof(*kvm->arch.sca), alloc_flags);
3355	mutex_unlock(lock: &kvm_lock);
3356	if (!kvm->arch.sca)
3357	goto out_err;
3358
3359	snprintf(buf: debug_name, size: sizeof(debug_name), fmt: "kvm-%u", current->pid);
3360
3361	kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
3362	if (!kvm->arch.dbf)
3363	goto out_err;
3364
3365	BUILD_BUG_ON(sizeof(struct sie_page2) != 4096);
3366	kvm->arch.sie_page2 =
3367	(struct sie_page2 *) get_zeroed_page(GFP_KERNEL_ACCOUNT | GFP_DMA);
3368	if (!kvm->arch.sie_page2)
3369	goto out_err;
3370
3371	kvm->arch.sie_page2->kvm = kvm;
3372	kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
3373
3374	for (i = 0; i < kvm_s390_fac_size(); i++) {
3375	kvm->arch.model.fac_mask[i] = stfle_fac_list[i] &
3376	(kvm_s390_fac_base[i] |
3377	kvm_s390_fac_ext[i]);
3378	kvm->arch.model.fac_list[i] = stfle_fac_list[i] &
3379	kvm_s390_fac_base[i];
3380	}
3381	kvm->arch.model.subfuncs = kvm_s390_available_subfunc;
3382
3383	/* we are always in czam mode - even on pre z14 machines */
3384	set_kvm_facility(fac_list: kvm->arch.model.fac_mask, nr: 138);
3385	set_kvm_facility(fac_list: kvm->arch.model.fac_list, nr: 138);
3386	/* we emulate STHYI in kvm */
3387	set_kvm_facility(fac_list: kvm->arch.model.fac_mask, nr: 74);
3388	set_kvm_facility(fac_list: kvm->arch.model.fac_list, nr: 74);
3389	if (machine_has_tlb_guest()) {
3390	set_kvm_facility(fac_list: kvm->arch.model.fac_mask, nr: 147);
3391	set_kvm_facility(fac_list: kvm->arch.model.fac_list, nr: 147);
3392	}
3393
3394	if (css_general_characteristics.aiv && test_facility(65))
3395	set_kvm_facility(fac_list: kvm->arch.model.fac_mask, nr: 65);
3396
3397	kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
3398	kvm->arch.model.ibc = sclp.ibc & 0x0fff;
3399
3400	kvm->arch.model.uv_feat_guest.feat = 0;
3401
3402	kvm_s390_crypto_init(kvm);
3403
3404	if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM)) {
3405	mutex_lock(&kvm->lock);
3406	kvm_s390_pci_init_list(kvm);
3407	kvm_s390_vcpu_pci_enable_interp(kvm);
3408	mutex_unlock(lock: &kvm->lock);
3409	}
3410
3411	mutex_init(&kvm->arch.float_int.ais_lock);
3412	spin_lock_init(&kvm->arch.float_int.lock);
3413	for (i = 0; i < FIRQ_LIST_COUNT; i++)
3414	INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
3415	init_waitqueue_head(&kvm->arch.ipte_wq);
3416	mutex_init(&kvm->arch.ipte_mutex);
3417
3418	debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
3419	VM_EVENT(kvm, 3, "vm created with type %lu", type);
3420
3421	if (type & KVM_VM_S390_UCONTROL) {
3422	struct kvm_userspace_memory_region2 fake_memslot = {
3423	.slot = KVM_S390_UCONTROL_MEMSLOT,
3424	.guest_phys_addr = 0,
3425	.userspace_addr = 0,
3426	.memory_size = ALIGN_DOWN(TASK_SIZE, _SEGMENT_SIZE),
3427	.flags = 0,
3428	};
3429
3430	kvm->arch.gmap = NULL;
3431	kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
3432	/* one flat fake memslot covering the whole address-space */
3433	mutex_lock(&kvm->slots_lock);
3434	KVM_BUG_ON(kvm_set_internal_memslot(kvm, &fake_memslot), kvm);
3435	mutex_unlock(lock: &kvm->slots_lock);
3436	} else {
3437	if (sclp.hamax == U64_MAX)
3438	kvm->arch.mem_limit = TASK_SIZE_MAX;
3439	else
3440	kvm->arch.mem_limit = min_t(unsigned long, TASK_SIZE_MAX,
3441	sclp.hamax + 1);
3442	kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1);
3443	if (!kvm->arch.gmap)
3444	goto out_err;
3445	kvm->arch.gmap->private = kvm;
3446	kvm->arch.gmap->pfault_enabled = 0;
3447	}
3448
3449	kvm->arch.use_pfmfi = sclp.has_pfmfi;
3450	kvm->arch.use_skf = sclp.has_skey;
3451	spin_lock_init(&kvm->arch.start_stop_lock);
3452	kvm_s390_vsie_init(kvm);
3453	if (use_gisa)
3454	kvm_s390_gisa_init(kvm);
3455	INIT_LIST_HEAD(list: &kvm->arch.pv.need_cleanup);
3456	kvm->arch.pv.set_aside = NULL;
3457	KVM_EVENT(3, "vm 0x%p created by pid %u", kvm, current->pid);
3458
3459	return 0;
3460	out_err:
3461	free_page((unsigned long)kvm->arch.sie_page2);
3462	debug_unregister(kvm->arch.dbf);
3463	sca_dispose(kvm);
3464	KVM_EVENT(3, "creation of vm failed: %d", rc);
3465	return rc;
3466	}
3467
3468	void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
3469	{
3470	u16 rc, rrc;
3471
3472	VCPU_EVENT(vcpu, 3, "%s", "free cpu");
3473	trace_kvm_s390_destroy_vcpu(id: vcpu->vcpu_id);
3474	kvm_s390_clear_local_irqs(vcpu);
3475	kvm_clear_async_pf_completion_queue(vcpu);
3476	if (!kvm_is_ucontrol(kvm: vcpu->kvm))
3477	sca_del_vcpu(vcpu);
3478	kvm_s390_update_topology_change_report(kvm: vcpu->kvm, val: 1);
3479
3480	if (kvm_is_ucontrol(kvm: vcpu->kvm))
3481	gmap_remove(vcpu->arch.gmap);
3482
3483	if (vcpu->kvm->arch.use_cmma)
3484	kvm_s390_vcpu_unsetup_cmma(vcpu);
3485	/* We can not hold the vcpu mutex here, we are already dying */
3486	if (kvm_s390_pv_cpu_get_handle(vcpu))
3487	kvm_s390_pv_destroy_cpu(vcpu, rc: &rc, rrc: &rrc);
3488	free_page((unsigned long)(vcpu->arch.sie_block));
3489	}
3490
3491	void kvm_arch_destroy_vm(struct kvm *kvm)
3492	{
3493	u16 rc, rrc;
3494
3495	kvm_destroy_vcpus(kvm);
3496	sca_dispose(kvm);
3497	kvm_s390_gisa_destroy(kvm);
3498	/*
3499	* We are already at the end of life and kvm->lock is not taken.
3500	* This is ok as the file descriptor is closed by now and nobody
3501	* can mess with the pv state.
3502	*/
3503	kvm_s390_pv_deinit_cleanup_all(kvm, rc: &rc, rrc: &rrc);
3504	/*
3505	* Remove the mmu notifier only when the whole KVM VM is torn down,
3506	* and only if one was registered to begin with. If the VM is
3507	* currently not protected, but has been previously been protected,
3508	* then it's possible that the notifier is still registered.
3509	*/
3510	if (kvm->arch.pv.mmu_notifier.ops)
3511	mmu_notifier_unregister(subscription: &kvm->arch.pv.mmu_notifier, mm: kvm->mm);
3512
3513	debug_unregister(kvm->arch.dbf);
3514	free_page((unsigned long)kvm->arch.sie_page2);
3515	if (!kvm_is_ucontrol(kvm))
3516	gmap_remove(kvm->arch.gmap);
3517	kvm_s390_destroy_adapters(kvm);
3518	kvm_s390_clear_float_irqs(kvm);
3519	kvm_s390_vsie_destroy(kvm);
3520	KVM_EVENT(3, "vm 0x%p destroyed", kvm);
3521	}
3522
3523	/* Section: vcpu related */
3524	static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
3525	{
3526	vcpu->arch.gmap = gmap_create(current->mm, -1UL);
3527	if (!vcpu->arch.gmap)
3528	return -ENOMEM;
3529	vcpu->arch.gmap->private = vcpu->kvm;
3530
3531	return 0;
3532	}
3533
3534	static void sca_del_vcpu(struct kvm_vcpu *vcpu)
3535	{
3536	struct esca_block *sca = vcpu->kvm->arch.sca;
3537
3538	if (!kvm_s390_use_sca_entries())
3539	return;
3540
3541	clear_bit_inv(vcpu->vcpu_id, (unsigned long *)sca->mcn);
3542	sca->cpu[vcpu->vcpu_id].sda = 0;
3543	}
3544
3545	static void sca_add_vcpu(struct kvm_vcpu *vcpu)
3546	{
3547	struct esca_block *sca = vcpu->kvm->arch.sca;
3548	phys_addr_t sca_phys = virt_to_phys(address: sca);
3549
3550	/* we still need the sca header for the ipte control */
3551	vcpu->arch.sie_block->scaoh = sca_phys >> 32;
3552	vcpu->arch.sie_block->scaol = sca_phys & ESCA_SCAOL_MASK;
3553	vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
3554
3555	if (!kvm_s390_use_sca_entries())
3556	return;
3557
3558	set_bit_inv(vcpu->vcpu_id, (unsigned long *)sca->mcn);
3559	sca->cpu[vcpu->vcpu_id].sda = virt_to_phys(address: vcpu->arch.sie_block);
3560	}
3561
3562	static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
3563	{
3564	if (!kvm_s390_use_sca_entries())
3565	return id < KVM_MAX_VCPUS;
3566
3567	return id < KVM_S390_ESCA_CPU_SLOTS;
3568	}
3569
3570	/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
3571	static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3572	{
3573	WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
3574	raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
3575	vcpu->arch.cputm_start = get_tod_clock_fast();
3576	raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
3577	}
3578
3579	/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
3580	static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3581	{
3582	WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
3583	raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
3584	vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
3585	vcpu->arch.cputm_start = 0;
3586	raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
3587	}
3588
3589	/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
3590	static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3591	{
3592	WARN_ON_ONCE(vcpu->arch.cputm_enabled);
3593	vcpu->arch.cputm_enabled = true;
3594	__start_cpu_timer_accounting(vcpu);
3595	}
3596
3597	/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
3598	static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3599	{
3600	WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
3601	__stop_cpu_timer_accounting(vcpu);
3602	vcpu->arch.cputm_enabled = false;
3603	}
3604
3605	static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3606	{
3607	preempt_disable(); /* protect from TOD sync and vcpu_load/put */
3608	__enable_cpu_timer_accounting(vcpu);
3609	preempt_enable();
3610	}
3611
3612	static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3613	{
3614	preempt_disable(); /* protect from TOD sync and vcpu_load/put */
3615	__disable_cpu_timer_accounting(vcpu);
3616	preempt_enable();
3617	}
3618
3619	/* set the cpu timer - may only be called from the VCPU thread itself */
3620	void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
3621	{
3622	preempt_disable(); /* protect from TOD sync and vcpu_load/put */
3623	raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
3624	if (vcpu->arch.cputm_enabled)
3625	vcpu->arch.cputm_start = get_tod_clock_fast();
3626	vcpu->arch.sie_block->cputm = cputm;
3627	raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
3628	preempt_enable();
3629	}
3630
3631	/* update and get the cpu timer - can also be called from other VCPU threads */
3632	__u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
3633	{
3634	unsigned int seq;
3635	__u64 value;
3636
3637	if (unlikely(!vcpu->arch.cputm_enabled))
3638	return vcpu->arch.sie_block->cputm;
3639
3640	preempt_disable(); /* protect from TOD sync and vcpu_load/put */
3641	do {
3642	seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
3643	/*
3644	* If the writer would ever execute a read in the critical
3645	* section, e.g. in irq context, we have a deadlock.
3646	*/
3647	WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
3648	value = vcpu->arch.sie_block->cputm;
3649	/* if cputm_start is 0, accounting is being started/stopped */
3650	if (likely(vcpu->arch.cputm_start))
3651	value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
3652	} while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
3653	preempt_enable();
3654	return value;
3655	}
3656
3657	void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
3658	{
3659
3660	kvm_s390_set_cpuflags(vcpu, CPUSTAT_RUNNING);
3661	if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
3662	__start_cpu_timer_accounting(vcpu);
3663	vcpu->cpu = cpu;
3664	}
3665
3666	void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
3667	{
3668	vcpu->cpu = -1;
3669	if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
3670	__stop_cpu_timer_accounting(vcpu);
3671	kvm_s390_clear_cpuflags(vcpu, CPUSTAT_RUNNING);
3672
3673	}
3674
3675	void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
3676	{
3677	mutex_lock(&vcpu->kvm->lock);
3678	preempt_disable();
3679	vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
3680	vcpu->arch.sie_block->epdx = vcpu->kvm->arch.epdx;
3681	preempt_enable();
3682	mutex_unlock(lock: &vcpu->kvm->lock);
3683	if (!kvm_is_ucontrol(kvm: vcpu->kvm)) {
3684	vcpu->arch.gmap = vcpu->kvm->arch.gmap;
3685	sca_add_vcpu(vcpu);
3686	}
3687	if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0)
3688	vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
3689	}
3690
3691	static bool kvm_has_pckmo_subfunc(struct kvm *kvm, unsigned long nr)
3692	{
3693	if (test_bit_inv(nr, (unsigned long *)&kvm->arch.model.subfuncs.pckmo) &&
3694	test_bit_inv(nr, (unsigned long *)&kvm_s390_available_subfunc.pckmo))
3695	return true;
3696	return false;
3697	}
3698
3699	static bool kvm_has_pckmo_ecc(struct kvm *kvm)
3700	{
3701	/* At least one ECC subfunction must be present */
3702	return kvm_has_pckmo_subfunc(kvm, nr: 32) ||
3703	kvm_has_pckmo_subfunc(kvm, nr: 33) ||
3704	kvm_has_pckmo_subfunc(kvm, nr: 34) ||
3705	kvm_has_pckmo_subfunc(kvm, nr: 40) ||
3706	kvm_has_pckmo_subfunc(kvm, nr: 41);
3707
3708	}
3709
3710	static bool kvm_has_pckmo_hmac(struct kvm *kvm)
3711	{
3712	/* At least one HMAC subfunction must be present */
3713	return kvm_has_pckmo_subfunc(kvm, nr: 118) ||
3714	kvm_has_pckmo_subfunc(kvm, nr: 122);
3715	}
3716
3717	static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
3718	{
3719	/*
3720	* If the AP instructions are not being interpreted and the MSAX3
3721	* facility is not configured for the guest, there is nothing to set up.
3722	*/
3723	if (!vcpu->kvm->arch.crypto.apie && !test_kvm_facility(kvm: vcpu->kvm, nr: 76))
3724	return;
3725
3726	vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
3727	vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
3728	vcpu->arch.sie_block->eca &= ~ECA_APIE;
3729	vcpu->arch.sie_block->ecd &= ~(ECD_ECC | ECD_HMAC);
3730
3731	if (vcpu->kvm->arch.crypto.apie)
3732	vcpu->arch.sie_block->eca |= ECA_APIE;
3733
3734	/* Set up protected key support */
3735	if (vcpu->kvm->arch.crypto.aes_kw) {
3736	vcpu->arch.sie_block->ecb3 |= ECB3_AES;
3737	/* ecc/hmac is also wrapped with AES key */
3738	if (kvm_has_pckmo_ecc(vcpu->kvm))
3739	vcpu->arch.sie_block->ecd |= ECD_ECC;
3740	if (kvm_has_pckmo_hmac(vcpu->kvm))
3741	vcpu->arch.sie_block->ecd |= ECD_HMAC;
3742	}
3743
3744	if (vcpu->kvm->arch.crypto.dea_kw)
3745	vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
3746	}
3747
3748	void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
3749	{
3750	free_page((unsigned long)phys_to_virt(vcpu->arch.sie_block->cbrlo));
3751	vcpu->arch.sie_block->cbrlo = 0;
3752	}
3753
3754	int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
3755	{
3756	void *cbrlo_page = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
3757
3758	if (!cbrlo_page)
3759	return -ENOMEM;
3760
3761	vcpu->arch.sie_block->cbrlo = virt_to_phys(address: cbrlo_page);
3762	return 0;
3763	}
3764
3765	static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
3766	{
3767	struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
3768
3769	vcpu->arch.sie_block->ibc = model->ibc;
3770	if (test_kvm_facility(kvm: vcpu->kvm, nr: 7))
3771	vcpu->arch.sie_block->fac = virt_to_phys(address: model->fac_list);
3772	}
3773
3774	static int kvm_s390_vcpu_setup(struct kvm_vcpu *vcpu)
3775	{
3776	int rc = 0;
3777	u16 uvrc, uvrrc;
3778
3779	atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
3780	CPUSTAT_SM |
3781	CPUSTAT_STOPPED);
3782
3783	if (test_kvm_facility(vcpu->kvm, 78))
3784	kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED2);
3785	else if (test_kvm_facility(vcpu->kvm, 8))
3786	kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED);
3787
3788	kvm_s390_vcpu_setup_model(vcpu);
3789
3790	/* pgste_set_pte has special handling for !machine_has_esop() */
3791	if (machine_has_esop())
3792	vcpu->arch.sie_block->ecb |= ECB_HOSTPROTINT;
3793	if (test_kvm_facility(vcpu->kvm, 9))
3794	vcpu->arch.sie_block->ecb |= ECB_SRSI;
3795	if (test_kvm_facility(vcpu->kvm, 11))
3796	vcpu->arch.sie_block->ecb |= ECB_PTF;
3797	if (test_kvm_facility(vcpu->kvm, 73))
3798	vcpu->arch.sie_block->ecb |= ECB_TE;
3799	if (!kvm_is_ucontrol(vcpu->kvm))
3800	vcpu->arch.sie_block->ecb |= ECB_SPECI;
3801
3802	if (test_kvm_facility(vcpu->kvm, 8) && vcpu->kvm->arch.use_pfmfi)
3803	vcpu->arch.sie_block->ecb2 |= ECB2_PFMFI;
3804	if (test_kvm_facility(vcpu->kvm, 130))
3805	vcpu->arch.sie_block->ecb2 |= ECB2_IEP;
3806	vcpu->arch.sie_block->eca = ECA_MVPGI | ECA_PROTEXCI;
3807	if (sclp.has_cei)
3808	vcpu->arch.sie_block->eca |= ECA_CEI;
3809	if (sclp.has_ib)
3810	vcpu->arch.sie_block->eca |= ECA_IB;
3811	if (sclp.has_siif)
3812	vcpu->arch.sie_block->eca |= ECA_SII;
3813	if (kvm_s390_use_sca_entries())
3814	vcpu->arch.sie_block->eca |= ECA_SIGPI;
3815	if (test_kvm_facility(kvm: vcpu->kvm, nr: 129)) {
3816	vcpu->arch.sie_block->eca |= ECA_VX;
3817	vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
3818	}
3819	if (test_kvm_facility(vcpu->kvm, 139))
3820	vcpu->arch.sie_block->ecd |= ECD_MEF;
3821	if (test_kvm_facility(vcpu->kvm, 156))
3822	vcpu->arch.sie_block->ecd |= ECD_ETOKENF;
3823	if (vcpu->arch.sie_block->gd) {
3824	vcpu->arch.sie_block->eca |= ECA_AIV;
3825	VCPU_EVENT(vcpu, 3, "AIV gisa format-%u enabled for cpu %03u",
3826	vcpu->arch.sie_block->gd & 0x3, vcpu->vcpu_id);
3827	}
3828	vcpu->arch.sie_block->sdnxo = virt_to_phys(&vcpu->run->s.regs.sdnx) | SDNXC;
3829	vcpu->arch.sie_block->riccbd = virt_to_phys(address: &vcpu->run->s.regs.riccb);
3830
3831	if (sclp.has_kss)
3832	kvm_s390_set_cpuflags(vcpu, CPUSTAT_KSS);
3833	else
3834	vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
3835
3836	if (vcpu->kvm->arch.use_cmma) {
3837	rc = kvm_s390_vcpu_setup_cmma(vcpu);
3838	if (rc)
3839	return rc;
3840	}
3841	hrtimer_setup(timer: &vcpu->arch.ckc_timer, function: kvm_s390_idle_wakeup, CLOCK_MONOTONIC,
3842	mode: HRTIMER_MODE_REL);
3843
3844	vcpu->arch.sie_block->hpid = HPID_KVM;
3845
3846	kvm_s390_vcpu_crypto_setup(vcpu);
3847
3848	kvm_s390_vcpu_pci_setup(vcpu);
3849
3850	mutex_lock(&vcpu->kvm->lock);
3851	if (kvm_s390_pv_is_protected(vcpu->kvm)) {
3852	rc = kvm_s390_pv_create_cpu(vcpu, rc: &uvrc, rrc: &uvrrc);
3853	if (rc)
3854	kvm_s390_vcpu_unsetup_cmma(vcpu);
3855	}
3856	mutex_unlock(lock: &vcpu->kvm->lock);
3857
3858	return rc;
3859	}
3860
3861	int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
3862	{
3863	if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
3864	return -EINVAL;
3865	return 0;
3866	}
3867
3868	int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
3869	{
3870	struct sie_page *sie_page;
3871	int rc;
3872
3873	BUILD_BUG_ON(sizeof(struct sie_page) != 4096);
3874	sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL_ACCOUNT);
3875	if (!sie_page)
3876	return -ENOMEM;
3877
3878	vcpu->arch.sie_block = &sie_page->sie_block;
3879	vcpu->arch.sie_block->itdba = virt_to_phys(address: &sie_page->itdb);
3880
3881	/* the real guest size will always be smaller than msl */
3882	vcpu->arch.sie_block->mso = 0;
3883	vcpu->arch.sie_block->msl = sclp.hamax;
3884
3885	vcpu->arch.sie_block->icpua = vcpu->vcpu_id;
3886	spin_lock_init(&vcpu->arch.local_int.lock);
3887	vcpu->arch.sie_block->gd = kvm_s390_get_gisa_desc(kvm: vcpu->kvm);
3888	seqcount_init(&vcpu->arch.cputm_seqcount);
3889
3890	vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
3891	kvm_clear_async_pf_completion_queue(vcpu);
3892	vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
3893	KVM_SYNC_GPRS |
3894	KVM_SYNC_ACRS |
3895	KVM_SYNC_CRS |
3896	KVM_SYNC_ARCH0 |
3897	KVM_SYNC_PFAULT |
3898	KVM_SYNC_DIAG318;
3899	vcpu->arch.acrs_loaded = false;
3900	kvm_s390_set_prefix(vcpu, prefix: 0);
3901	if (test_kvm_facility(vcpu->kvm, 64))
3902	vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
3903	if (test_kvm_facility(vcpu->kvm, 82))
3904	vcpu->run->kvm_valid_regs |= KVM_SYNC_BPBC;
3905	if (test_kvm_facility(vcpu->kvm, 133))
3906	vcpu->run->kvm_valid_regs |= KVM_SYNC_GSCB;
3907	if (test_kvm_facility(vcpu->kvm, 156))
3908	vcpu->run->kvm_valid_regs |= KVM_SYNC_ETOKEN;
3909	/* fprs can be synchronized via vrs, even if the guest has no vx. With
3910	* cpu_has_vx(), (load|store)_fpu_regs() will work with vrs format.
3911	*/
3912	if (cpu_has_vx())
3913	vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
3914	else
3915	vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
3916
3917	if (kvm_is_ucontrol(kvm: vcpu->kvm)) {
3918	rc = __kvm_ucontrol_vcpu_init(vcpu);
3919	if (rc)
3920	goto out_free_sie_block;
3921	}
3922
3923	VM_EVENT(vcpu->kvm, 3, "create cpu %d at 0x%p, sie block at 0x%p",
3924	vcpu->vcpu_id, vcpu, vcpu->arch.sie_block);
3925	trace_kvm_s390_create_vcpu(id: vcpu->vcpu_id, vcpu, sie_block: vcpu->arch.sie_block);
3926
3927	rc = kvm_s390_vcpu_setup(vcpu);
3928	if (rc)
3929	goto out_ucontrol_uninit;
3930
3931	kvm_s390_update_topology_change_report(kvm: vcpu->kvm, val: 1);
3932	return 0;
3933
3934	out_ucontrol_uninit:
3935	if (kvm_is_ucontrol(kvm: vcpu->kvm))
3936	gmap_remove(vcpu->arch.gmap);
3937	out_free_sie_block:
3938	free_page((unsigned long)(vcpu->arch.sie_block));
3939	return rc;
3940	}
3941
3942	int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
3943	{
3944	clear_bit(nr: vcpu->vcpu_idx, addr: vcpu->kvm->arch.gisa_int.kicked_mask);
3945	return kvm_s390_vcpu_has_irq(vcpu, exclude_stop: 0);
3946	}
3947
3948	bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
3949	{
3950	return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE);
3951	}
3952
3953	void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
3954	{
3955	atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
3956	exit_sie(vcpu);
3957	}
3958
3959	void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
3960	{
3961	atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
3962	}
3963
3964	static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
3965	{
3966	atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
3967	exit_sie(vcpu);
3968	}
3969
3970	bool kvm_s390_vcpu_sie_inhibited(struct kvm_vcpu *vcpu)
3971	{
3972	return atomic_read(&vcpu->arch.sie_block->prog20) &
3973	(PROG_BLOCK_SIE | PROG_REQUEST);
3974	}
3975
3976	static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
3977	{
3978	atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
3979	}
3980
3981	/*
3982	* Kick a guest cpu out of (v)SIE and wait until (v)SIE is not running.
3983	* If the CPU is not running (e.g. waiting as idle) the function will
3984	* return immediately. */
3985	void exit_sie(struct kvm_vcpu *vcpu)
3986	{
3987	kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
3988	kvm_s390_vsie_kick(vcpu);
3989	while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
3990	cpu_relax();
3991	}
3992
3993	/* Kick a guest cpu out of SIE to process a request synchronously */
3994	void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
3995	{
3996	__kvm_make_request(req, vcpu);
3997	kvm_s390_vcpu_request(vcpu);
3998	}
3999
4000	static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
4001	unsigned long end)
4002	{
4003	struct kvm *kvm = gmap->private;
4004	struct kvm_vcpu *vcpu;
4005	unsigned long prefix;
4006	unsigned long i;
4007
4008	trace_kvm_s390_gmap_notifier(start, end, shadow: gmap_is_shadow(gmap));
4009
4010	if (gmap_is_shadow(gmap))
4011	return;
4012	if (start >= 1UL << 31)
4013	/* We are only interested in prefix pages */
4014	return;
4015	kvm_for_each_vcpu(i, vcpu, kvm) {
4016	/* match against both prefix pages */
4017	prefix = kvm_s390_get_prefix(vcpu);
4018	if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) {
4019	VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx",
4020	start, end);
4021	kvm_s390_sync_request(KVM_REQ_REFRESH_GUEST_PREFIX, vcpu);
4022	}
4023	}
4024	}
4025
4026	bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
4027	{
4028	/* do not poll with more than halt_poll_max_steal percent of steal time */
4029	if (get_lowcore()->avg_steal_timer * 100 / (TICK_USEC << 12) >=
4030	READ_ONCE(halt_poll_max_steal)) {
4031	vcpu->stat.halt_no_poll_steal++;
4032	return true;
4033	}
4034	return false;
4035	}
4036
4037	int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
4038	{
4039	/* kvm common code refers to this, but never calls it */
4040	BUG();
4041	return 0;
4042	}
4043
4044	static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
4045	struct kvm_one_reg *reg)
4046	{
4047	int r = -EINVAL;
4048
4049	switch (reg->id) {
4050	case KVM_REG_S390_TODPR:
4051	r = put_user(vcpu->arch.sie_block->todpr,
4052	(u32 __user *)reg->addr);
4053	break;
4054	case KVM_REG_S390_EPOCHDIFF:
4055	r = put_user(vcpu->arch.sie_block->epoch,
4056	(u64 __user *)reg->addr);
4057	break;
4058	case KVM_REG_S390_CPU_TIMER:
4059	r = put_user(kvm_s390_get_cpu_timer(vcpu),
4060	(u64 __user *)reg->addr);
4061	break;
4062	case KVM_REG_S390_CLOCK_COMP:
4063	r = put_user(vcpu->arch.sie_block->ckc,
4064	(u64 __user *)reg->addr);
4065	break;
4066	case KVM_REG_S390_PFTOKEN:
4067	r = put_user(vcpu->arch.pfault_token,
4068	(u64 __user *)reg->addr);
4069	break;
4070	case KVM_REG_S390_PFCOMPARE:
4071	r = put_user(vcpu->arch.pfault_compare,
4072	(u64 __user *)reg->addr);
4073	break;
4074	case KVM_REG_S390_PFSELECT:
4075	r = put_user(vcpu->arch.pfault_select,
4076	(u64 __user *)reg->addr);
4077	break;
4078	case KVM_REG_S390_PP:
4079	r = put_user(vcpu->arch.sie_block->pp,
4080	(u64 __user *)reg->addr);
4081	break;
4082	case KVM_REG_S390_GBEA:
4083	r = put_user(vcpu->arch.sie_block->gbea,
4084	(u64 __user *)reg->addr);
4085	break;
4086	default:
4087	break;
4088	}
4089
4090	return r;
4091	}
4092
4093	static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
4094	struct kvm_one_reg *reg)
4095	{
4096	int r = -EINVAL;
4097	__u64 val;
4098
4099	switch (reg->id) {
4100	case KVM_REG_S390_TODPR:
4101	r = get_user(vcpu->arch.sie_block->todpr,
4102	(u32 __user *)reg->addr);
4103	break;
4104	case KVM_REG_S390_EPOCHDIFF:
4105	r = get_user(vcpu->arch.sie_block->epoch,
4106	(u64 __user *)reg->addr);
4107	break;
4108	case KVM_REG_S390_CPU_TIMER:
4109	r = get_user(val, (u64 __user *)reg->addr);
4110	if (!r)
4111	kvm_s390_set_cpu_timer(vcpu, cputm: val);
4112	break;
4113	case KVM_REG_S390_CLOCK_COMP:
4114	r = get_user(vcpu->arch.sie_block->ckc,
4115	(u64 __user *)reg->addr);
4116	break;
4117	case KVM_REG_S390_PFTOKEN:
4118	r = get_user(vcpu->arch.pfault_token,
4119	(u64 __user *)reg->addr);
4120	if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
4121	kvm_clear_async_pf_completion_queue(vcpu);
4122	break;
4123	case KVM_REG_S390_PFCOMPARE:
4124	r = get_user(vcpu->arch.pfault_compare,
4125	(u64 __user *)reg->addr);
4126	break;
4127	case KVM_REG_S390_PFSELECT:
4128	r = get_user(vcpu->arch.pfault_select,
4129	(u64 __user *)reg->addr);
4130	break;
4131	case KVM_REG_S390_PP:
4132	r = get_user(vcpu->arch.sie_block->pp,
4133	(u64 __user *)reg->addr);
4134	break;
4135	case KVM_REG_S390_GBEA:
4136	r = get_user(vcpu->arch.sie_block->gbea,
4137	(u64 __user *)reg->addr);
4138	break;
4139	default:
4140	break;
4141	}
4142
4143	return r;
4144	}
4145
4146	static void kvm_arch_vcpu_ioctl_normal_reset(struct kvm_vcpu *vcpu)
4147	{
4148	vcpu->arch.sie_block->gpsw.mask &= ~PSW_MASK_RI;
4149	vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
4150	memset(vcpu->run->s.regs.riccb, 0, sizeof(vcpu->run->s.regs.riccb));
4151
4152	kvm_clear_async_pf_completion_queue(vcpu);
4153	if (!kvm_s390_user_cpu_state_ctrl(kvm: vcpu->kvm))
4154	kvm_s390_vcpu_stop(vcpu);
4155	kvm_s390_clear_local_irqs(vcpu);
4156	}
4157
4158	static void kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
4159	{
4160	/* Initial reset is a superset of the normal reset */
4161	kvm_arch_vcpu_ioctl_normal_reset(vcpu);
4162
4163	/*
4164	* This equals initial cpu reset in pop, but we don't switch to ESA.
4165	* We do not only reset the internal data, but also ...
4166	*/
4167	vcpu->arch.sie_block->gpsw.mask = 0;
4168	vcpu->arch.sie_block->gpsw.addr = 0;
4169	kvm_s390_set_prefix(vcpu, prefix: 0);
4170	kvm_s390_set_cpu_timer(vcpu, cputm: 0);
4171	vcpu->arch.sie_block->ckc = 0;
4172	memset(vcpu->arch.sie_block->gcr, 0, sizeof(vcpu->arch.sie_block->gcr));
4173	vcpu->arch.sie_block->gcr[0] = CR0_INITIAL_MASK;
4174	vcpu->arch.sie_block->gcr[14] = CR14_INITIAL_MASK;
4175
4176	/* ... the data in sync regs */
4177	memset(vcpu->run->s.regs.crs, 0, sizeof(vcpu->run->s.regs.crs));
4178	vcpu->run->s.regs.ckc = 0;
4179	vcpu->run->s.regs.crs[0] = CR0_INITIAL_MASK;
4180	vcpu->run->s.regs.crs[14] = CR14_INITIAL_MASK;
4181	vcpu->run->psw_addr = 0;
4182	vcpu->run->psw_mask = 0;
4183	vcpu->run->s.regs.todpr = 0;
4184	vcpu->run->s.regs.cputm = 0;
4185	vcpu->run->s.regs.ckc = 0;
4186	vcpu->run->s.regs.pp = 0;
4187	vcpu->run->s.regs.gbea = 1;
4188	vcpu->run->s.regs.fpc = 0;
4189	/*
4190	* Do not reset these registers in the protected case, as some of
4191	* them are overlaid and they are not accessible in this case
4192	* anyway.
4193	*/
4194	if (!kvm_s390_pv_cpu_is_protected(vcpu)) {
4195	vcpu->arch.sie_block->gbea = 1;
4196	vcpu->arch.sie_block->pp = 0;
4197	vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
4198	vcpu->arch.sie_block->todpr = 0;
4199	}
4200	}
4201
4202	static void kvm_arch_vcpu_ioctl_clear_reset(struct kvm_vcpu *vcpu)
4203	{
4204	struct kvm_sync_regs *regs = &vcpu->run->s.regs;
4205
4206	/* Clear reset is a superset of the initial reset */
4207	kvm_arch_vcpu_ioctl_initial_reset(vcpu);
4208
4209	memset(&regs->gprs, 0, sizeof(regs->gprs));
4210	memset(&regs->vrs, 0, sizeof(regs->vrs));
4211	memset(&regs->acrs, 0, sizeof(regs->acrs));
4212	memset(&regs->gscb, 0, sizeof(regs->gscb));
4213
4214	regs->etoken = 0;
4215	regs->etoken_extension = 0;
4216	}
4217
4218	int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
4219	{
4220	vcpu_load(vcpu);
4221	memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
4222	vcpu_put(vcpu);
4223	return 0;
4224	}
4225
4226	int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
4227	{
4228	vcpu_load(vcpu);
4229	memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
4230	vcpu_put(vcpu);
4231	return 0;
4232	}
4233
4234	int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
4235	struct kvm_sregs *sregs)
4236	{
4237	vcpu_load(vcpu);
4238
4239	memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
4240	memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
4241
4242	vcpu_put(vcpu);
4243	return 0;
4244	}
4245
4246	int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
4247	struct kvm_sregs *sregs)
4248	{
4249	vcpu_load(vcpu);
4250
4251	memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
4252	memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
4253
4254	vcpu_put(vcpu);
4255	return 0;
4256	}
4257
4258	int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
4259	{
4260	vcpu_load(vcpu);
4261
4262	vcpu->run->s.regs.fpc = fpu->fpc;
4263	if (cpu_has_vx())
4264	convert_fp_to_vx((__vector128 *) vcpu->run->s.regs.vrs,
4265	(freg_t *) fpu->fprs);
4266	else
4267	memcpy(vcpu->run->s.regs.fprs, &fpu->fprs, sizeof(fpu->fprs));
4268
4269	vcpu_put(vcpu);
4270	return 0;
4271	}
4272
4273	int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
4274	{
4275	vcpu_load(vcpu);
4276
4277	if (cpu_has_vx())
4278	convert_vx_to_fp((freg_t *) fpu->fprs,
4279	(__vector128 *) vcpu->run->s.regs.vrs);
4280	else
4281	memcpy(fpu->fprs, vcpu->run->s.regs.fprs, sizeof(fpu->fprs));
4282	fpu->fpc = vcpu->run->s.regs.fpc;
4283
4284	vcpu_put(vcpu);
4285	return 0;
4286	}
4287
4288	static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
4289	{
4290	int rc = 0;
4291
4292	if (!is_vcpu_stopped(vcpu))
4293	rc = -EBUSY;
4294	else {
4295	vcpu->run->psw_mask = psw.mask;
4296	vcpu->run->psw_addr = psw.addr;
4297	}
4298	return rc;
4299	}
4300
4301	int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
4302	struct kvm_translation *tr)
4303	{
4304	return -EINVAL; /* not implemented yet */
4305	}
4306
4307	#define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
4308	KVM_GUESTDBG_USE_HW_BP | \
4309	KVM_GUESTDBG_ENABLE)
4310
4311	int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
4312	struct kvm_guest_debug *dbg)
4313	{
4314	int rc = 0;
4315
4316	vcpu_load(vcpu);
4317
4318	vcpu->guest_debug = 0;
4319	kvm_s390_clear_bp_data(vcpu);
4320
4321	if (dbg->control & ~VALID_GUESTDBG_FLAGS) {
4322	rc = -EINVAL;
4323	goto out;
4324	}
4325	if (!sclp.has_gpere) {
4326	rc = -EINVAL;
4327	goto out;
4328	}
4329
4330	if (dbg->control & KVM_GUESTDBG_ENABLE) {
4331	vcpu->guest_debug = dbg->control;
4332	/* enforce guest PER */
4333	kvm_s390_set_cpuflags(vcpu, CPUSTAT_P);
4334
4335	if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
4336	rc = kvm_s390_import_bp_data(vcpu, dbg);
4337	} else {
4338	kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
4339	vcpu->arch.guestdbg.last_bp = 0;
4340	}
4341
4342	if (rc) {
4343	vcpu->guest_debug = 0;
4344	kvm_s390_clear_bp_data(vcpu);
4345	kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
4346	}
4347
4348	out:
4349	vcpu_put(vcpu);
4350	return rc;
4351	}
4352
4353	int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
4354	struct kvm_mp_state *mp_state)
4355	{
4356	int ret;
4357
4358	vcpu_load(vcpu);
4359
4360	/* CHECK_STOP and LOAD are not supported yet */
4361	ret = is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
4362	KVM_MP_STATE_OPERATING;
4363
4364	vcpu_put(vcpu);
4365	return ret;
4366	}
4367
4368	int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
4369	struct kvm_mp_state *mp_state)
4370	{
4371	int rc = 0;
4372
4373	vcpu_load(vcpu);
4374
4375	/* user space knows about this interface - let it control the state */
4376	kvm_s390_set_user_cpu_state_ctrl(kvm: vcpu->kvm);
4377
4378	switch (mp_state->mp_state) {
4379	case KVM_MP_STATE_STOPPED:
4380	rc = kvm_s390_vcpu_stop(vcpu);
4381	break;
4382	case KVM_MP_STATE_OPERATING:
4383	rc = kvm_s390_vcpu_start(vcpu);
4384	break;
4385	case KVM_MP_STATE_LOAD:
4386	if (!kvm_s390_pv_cpu_is_protected(vcpu)) {
4387	rc = -ENXIO;
4388	break;
4389	}
4390	rc = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_OPR_LOAD);
4391	break;
4392	case KVM_MP_STATE_CHECK_STOP:
4393	fallthrough; /* CHECK_STOP and LOAD are not supported yet */
4394	default:
4395	rc = -ENXIO;
4396	}
4397
4398	vcpu_put(vcpu);
4399	return rc;
4400	}
4401
4402	static bool ibs_enabled(struct kvm_vcpu *vcpu)
4403	{
4404	return kvm_s390_test_cpuflags(vcpu, CPUSTAT_IBS);
4405	}
4406
4407	static int __kvm_s390_fixup_fault_sync(struct gmap *gmap, gpa_t gaddr, unsigned int flags)
4408	{
4409	struct kvm *kvm = gmap->private;
4410	gfn_t gfn = gpa_to_gfn(gpa: gaddr);
4411	bool unlocked;
4412	hva_t vmaddr;
4413	gpa_t tmp;
4414	int rc;
4415
4416	if (kvm_is_ucontrol(kvm)) {
4417	tmp = __gmap_translate(gmap, gaddr);
4418	gfn = gpa_to_gfn(gpa: tmp);
4419	}
4420
4421	vmaddr = gfn_to_hva(kvm, gfn);
4422	rc = fixup_user_fault(mm: gmap->mm, address: vmaddr, fault_flags: FAULT_FLAG_WRITE, unlocked: &unlocked);
4423	if (!rc)
4424	rc = __gmap_link(gmap, gaddr, vmaddr);
4425	return rc;
4426	}
4427
4428	/**
4429	* __kvm_s390_mprotect_many() - Apply specified protection to guest pages
4430	* @gmap: the gmap of the guest
4431	* @gpa: the starting guest address
4432	* @npages: how many pages to protect
4433	* @prot: indicates access rights: PROT_NONE, PROT_READ or PROT_WRITE
4434	* @bits: pgste notification bits to set
4435	*
4436	* Returns: 0 in case of success, < 0 in case of error - see gmap_protect_one()
4437	*
4438	* Context: kvm->srcu and gmap->mm need to be held in read mode
4439	*/
4440	int __kvm_s390_mprotect_many(struct gmap *gmap, gpa_t gpa, u8 npages, unsigned int prot,
4441	unsigned long bits)
4442	{
4443	unsigned int fault_flag = (prot & PROT_WRITE) ? FAULT_FLAG_WRITE : 0;
4444	gpa_t end = gpa + npages * PAGE_SIZE;
4445	int rc;
4446
4447	for (; gpa < end; gpa = ALIGN(gpa + 1, rc)) {
4448	rc = gmap_protect_one(gmap, gpa, prot, bits);
4449	if (rc == -EAGAIN) {
4450	__kvm_s390_fixup_fault_sync(gmap, gaddr: gpa, flags: fault_flag);
4451	rc = gmap_protect_one(gmap, gpa, prot, bits);
4452	}
4453	if (rc < 0)
4454	return rc;
4455	}
4456
4457	return 0;
4458	}
4459
4460	static int kvm_s390_mprotect_notify_prefix(struct kvm_vcpu *vcpu)
4461	{
4462	gpa_t gaddr = kvm_s390_get_prefix(vcpu);
4463	int idx, rc;
4464
4465	idx = srcu_read_lock(ssp: &vcpu->kvm->srcu);
4466	mmap_read_lock(mm: vcpu->arch.gmap->mm);
4467
4468	rc = __kvm_s390_mprotect_many(vcpu->arch.gmap, gaddr, 2, PROT_WRITE, GMAP_NOTIFY_MPROT);
4469
4470	mmap_read_unlock(mm: vcpu->arch.gmap->mm);
4471	srcu_read_unlock(ssp: &vcpu->kvm->srcu, idx);
4472
4473	return rc;
4474	}
4475
4476	static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
4477	{
4478	retry:
4479	kvm_s390_vcpu_request_handled(vcpu);
4480	if (!kvm_request_pending(vcpu))
4481	return 0;
4482	/*
4483	* If the guest prefix changed, re-arm the ipte notifier for the
4484	* guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
4485	* This ensures that the ipte instruction for this request has
4486	* already finished. We might race against a second unmapper that
4487	* wants to set the blocking bit. Lets just retry the request loop.
4488	*/
4489	if (kvm_check_request(KVM_REQ_REFRESH_GUEST_PREFIX, vcpu)) {
4490	int rc;
4491
4492	rc = kvm_s390_mprotect_notify_prefix(vcpu);
4493	if (rc) {
4494	kvm_make_request(KVM_REQ_REFRESH_GUEST_PREFIX, vcpu);
4495	return rc;
4496	}
4497	goto retry;
4498	}
4499
4500	if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
4501	vcpu->arch.sie_block->ihcpu = 0xffff;
4502	goto retry;
4503	}
4504
4505	if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
4506	if (!ibs_enabled(vcpu)) {
4507	trace_kvm_s390_enable_disable_ibs(id: vcpu->vcpu_id, state: 1);
4508	kvm_s390_set_cpuflags(vcpu, CPUSTAT_IBS);
4509	}
4510	goto retry;
4511	}
4512
4513	if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
4514	if (ibs_enabled(vcpu)) {
4515	trace_kvm_s390_enable_disable_ibs(id: vcpu->vcpu_id, state: 0);
4516	kvm_s390_clear_cpuflags(vcpu, CPUSTAT_IBS);
4517	}
4518	goto retry;
4519	}
4520
4521	if (kvm_check_request(KVM_REQ_ICPT_OPEREXC, vcpu)) {
4522	vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
4523	goto retry;
4524	}
4525
4526	if (kvm_check_request(KVM_REQ_START_MIGRATION, vcpu)) {
4527	/*
4528	* Disable CMM virtualization; we will emulate the ESSA
4529	* instruction manually, in order to provide additional
4530	* functionalities needed for live migration.
4531	*/
4532	vcpu->arch.sie_block->ecb2 &= ~ECB2_CMMA;
4533	goto retry;
4534	}
4535
4536	if (kvm_check_request(KVM_REQ_STOP_MIGRATION, vcpu)) {
4537	/*
4538	* Re-enable CMM virtualization if CMMA is available and
4539	* CMM has been used.
4540	*/
4541	if ((vcpu->kvm->arch.use_cmma) &&
4542	(vcpu->kvm->mm->context.uses_cmm))
4543	vcpu->arch.sie_block->ecb2 |= ECB2_CMMA;
4544	goto retry;
4545	}
4546
4547	/* we left the vsie handler, nothing to do, just clear the request */
4548	kvm_clear_request(KVM_REQ_VSIE_RESTART, vcpu);
4549
4550	return 0;
4551	}
4552
4553	static void __kvm_s390_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clock *gtod)
4554	{
4555	struct kvm_vcpu *vcpu;
4556	union tod_clock clk;
4557	unsigned long i;
4558
4559	preempt_disable();
4560
4561	store_tod_clock_ext(&clk);
4562
4563	kvm->arch.epoch = gtod->tod - clk.tod;
4564	kvm->arch.epdx = 0;
4565	if (test_kvm_facility(kvm, nr: 139)) {
4566	kvm->arch.epdx = gtod->epoch_idx - clk.ei;
4567	if (kvm->arch.epoch > gtod->tod)
4568	kvm->arch.epdx -= 1;
4569	}
4570
4571	kvm_s390_vcpu_block_all(kvm);
4572	kvm_for_each_vcpu(i, vcpu, kvm) {
4573	vcpu->arch.sie_block->epoch = kvm->arch.epoch;
4574	vcpu->arch.sie_block->epdx = kvm->arch.epdx;
4575	}
4576
4577	kvm_s390_vcpu_unblock_all(kvm);
4578	preempt_enable();
4579	}
4580
4581	int kvm_s390_try_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clock *gtod)
4582	{
4583	if (!mutex_trylock(&kvm->lock))
4584	return 0;
4585	__kvm_s390_set_tod_clock(kvm, gtod);
4586	mutex_unlock(lock: &kvm->lock);
4587	return 1;
4588	}
4589
4590	static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
4591	unsigned long token)
4592	{
4593	struct kvm_s390_interrupt inti;
4594	struct kvm_s390_irq irq;
4595
4596	if (start_token) {
4597	irq.u.ext.ext_params2 = token;
4598	irq.type = KVM_S390_INT_PFAULT_INIT;
4599	WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
4600	} else {
4601	inti.type = KVM_S390_INT_PFAULT_DONE;
4602	inti.parm64 = token;
4603	WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
4604	}
4605	}
4606
4607	bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
4608	struct kvm_async_pf *work)
4609	{
4610	trace_kvm_s390_pfault_init(vcpu, pfault_token: work->arch.pfault_token);
4611	__kvm_inject_pfault_token(vcpu, start_token: true, token: work->arch.pfault_token);
4612
4613	return true;
4614	}
4615
4616	void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
4617	struct kvm_async_pf *work)
4618	{
4619	trace_kvm_s390_pfault_done(vcpu, pfault_token: work->arch.pfault_token);
4620	__kvm_inject_pfault_token(vcpu, start_token: false, token: work->arch.pfault_token);
4621	}
4622
4623	void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
4624	struct kvm_async_pf *work)
4625	{
4626	/* s390 will always inject the page directly */
4627	}
4628
4629	bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu)
4630	{
4631	/*
4632	* s390 will always inject the page directly,
4633	* but we still want check_async_completion to cleanup
4634	*/
4635	return true;
4636	}
4637
4638	static bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
4639	{
4640	hva_t hva;
4641	struct kvm_arch_async_pf arch;
4642
4643	if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
4644	return false;
4645	if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
4646	vcpu->arch.pfault_compare)
4647	return false;
4648	if (psw_extint_disabled(vcpu))
4649	return false;
4650	if (kvm_s390_vcpu_has_irq(vcpu, exclude_stop: 0))
4651	return false;
4652	if (!(vcpu->arch.sie_block->gcr[0] & CR0_SERVICE_SIGNAL_SUBMASK))
4653	return false;
4654	if (!vcpu->arch.gmap->pfault_enabled)
4655	return false;
4656
4657	hva = gfn_to_hva(kvm: vcpu->kvm, current->thread.gmap_teid.addr);
4658	if (read_guest_real(vcpu, gra: vcpu->arch.pfault_token, data: &arch.pfault_token, len: 8))
4659	return false;
4660
4661	return kvm_setup_async_pf(vcpu, current->thread.gmap_teid.addr * PAGE_SIZE, hva, arch: &arch);
4662	}
4663
4664	static int vcpu_pre_run(struct kvm_vcpu *vcpu)
4665	{
4666	int rc, cpuflags;
4667
4668	/*
4669	* On s390 notifications for arriving pages will be delivered directly
4670	* to the guest but the house keeping for completed pfaults is
4671	* handled outside the worker.
4672	*/
4673	kvm_check_async_pf_completion(vcpu);
4674
4675	vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
4676	vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
4677
4678	if (!kvm_is_ucontrol(kvm: vcpu->kvm)) {
4679	rc = kvm_s390_deliver_pending_interrupts(vcpu);
4680	if (rc || guestdbg_exit_pending(vcpu))
4681	return rc;
4682	}
4683
4684	rc = kvm_s390_handle_requests(vcpu);
4685	if (rc)
4686	return rc;
4687
4688	if (guestdbg_enabled(vcpu)) {
4689	kvm_s390_backup_guest_per_regs(vcpu);
4690	kvm_s390_patch_guest_per_regs(vcpu);
4691	}
4692
4693	clear_bit(nr: vcpu->vcpu_idx, addr: vcpu->kvm->arch.gisa_int.kicked_mask);
4694
4695	vcpu->arch.sie_block->icptcode = 0;
4696	current->thread.gmap_int_code = 0;
4697	cpuflags = atomic_read(v: &vcpu->arch.sie_block->cpuflags);
4698	VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
4699	trace_kvm_s390_sie_enter(vcpu, cpuflags);
4700
4701	return 0;
4702	}
4703
4704	static int vcpu_post_run_addressing_exception(struct kvm_vcpu *vcpu)
4705	{
4706	struct kvm_s390_pgm_info pgm_info = {
4707	.code = PGM_ADDRESSING,
4708	};
4709	u8 opcode, ilen;
4710	int rc;
4711
4712	VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
4713	trace_kvm_s390_sie_fault(vcpu);
4714
4715	/*
4716	* We want to inject an addressing exception, which is defined as a
4717	* suppressing or terminating exception. However, since we came here
4718	* by a DAT access exception, the PSW still points to the faulting
4719	* instruction since DAT exceptions are nullifying. So we've got
4720	* to look up the current opcode to get the length of the instruction
4721	* to be able to forward the PSW.
4722	*/
4723	rc = read_guest_instr(vcpu, ga: vcpu->arch.sie_block->gpsw.addr, data: &opcode, len: 1);
4724	ilen = insn_length(opcode);
4725	if (rc < 0) {
4726	return rc;
4727	} else if (rc) {
4728	/* Instruction-Fetching Exceptions - we can't detect the ilen.
4729	* Forward by arbitrary ilc, injection will take care of
4730	* nullification if necessary.
4731	*/
4732	pgm_info = vcpu->arch.pgm;
4733	ilen = 4;
4734	}
4735	pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
4736	kvm_s390_forward_psw(vcpu, ilen);
4737	return kvm_s390_inject_prog_irq(vcpu, pgm_info: &pgm_info);
4738	}
4739
4740	static void kvm_s390_assert_primary_as(struct kvm_vcpu *vcpu)
4741	{
4742	KVM_BUG(current->thread.gmap_teid.as != PSW_BITS_AS_PRIMARY, vcpu->kvm,
4743	"Unexpected program interrupt 0x%x, TEID 0x%016lx",
4744	current->thread.gmap_int_code, current->thread.gmap_teid.val);
4745	}
4746
4747	/*
4748	* __kvm_s390_handle_dat_fault() - handle a dat fault for the gmap of a vcpu
4749	* @vcpu: the vCPU whose gmap is to be fixed up
4750	* @gfn: the guest frame number used for memslots (including fake memslots)
4751	* @gaddr: the gmap address, does not have to match @gfn for ucontrol gmaps
4752	* @foll: FOLL_* flags
4753	*
4754	* Return: 0 on success, < 0 in case of error.
4755	* Context: The mm lock must not be held before calling. May sleep.
4756	*/
4757	int __kvm_s390_handle_dat_fault(struct kvm_vcpu *vcpu, gfn_t gfn, gpa_t gaddr, unsigned int foll)
4758	{
4759	struct kvm_memory_slot *slot;
4760	unsigned int fault_flags;
4761	bool writable, unlocked;
4762	unsigned long vmaddr;
4763	struct page *page;
4764	kvm_pfn_t pfn;
4765	int rc;
4766
4767	slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
4768	if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
4769	return vcpu_post_run_addressing_exception(vcpu);
4770
4771	fault_flags = foll & FOLL_WRITE ? FAULT_FLAG_WRITE : 0;
4772	if (vcpu->arch.gmap->pfault_enabled)
4773	foll |= FOLL_NOWAIT;
4774	vmaddr = __gfn_to_hva_memslot(slot, gfn);
4775
4776	try_again:
4777	pfn = __kvm_faultin_pfn(slot, gfn, foll, writable: &writable, refcounted_page: &page);
4778
4779	/* Access outside memory, inject addressing exception */
4780	if (is_noslot_pfn(pfn))
4781	return vcpu_post_run_addressing_exception(vcpu);
4782	/* Signal pending: try again */
4783	if (pfn == KVM_PFN_ERR_SIGPENDING)
4784	return -EAGAIN;
4785
4786	/* Needs I/O, try to setup async pfault (only possible with FOLL_NOWAIT) */
4787	if (pfn == KVM_PFN_ERR_NEEDS_IO) {
4788	trace_kvm_s390_major_guest_pfault(vcpu);
4789	if (kvm_arch_setup_async_pf(vcpu))
4790	return 0;
4791	vcpu->stat.pfault_sync++;
4792	/* Could not setup async pfault, try again synchronously */
4793	foll &= ~FOLL_NOWAIT;
4794	goto try_again;
4795	}
4796	/* Any other error */
4797	if (is_error_pfn(pfn))
4798	return -EFAULT;
4799
4800	/* Success */
4801	mmap_read_lock(mm: vcpu->arch.gmap->mm);
4802	/* Mark the userspace PTEs as young and/or dirty, to avoid page fault loops */
4803	rc = fixup_user_fault(mm: vcpu->arch.gmap->mm, address: vmaddr, fault_flags, unlocked: &unlocked);
4804	if (!rc)
4805	rc = __gmap_link(vcpu->arch.gmap, gaddr, vmaddr);
4806	scoped_guard(spinlock, &vcpu->kvm->mmu_lock) {
4807	kvm_release_faultin_page(kvm: vcpu->kvm, page, unused: false, dirty: writable);
4808	}
4809	mmap_read_unlock(mm: vcpu->arch.gmap->mm);
4810	return rc;
4811	}
4812
4813	static int vcpu_dat_fault_handler(struct kvm_vcpu *vcpu, unsigned long gaddr, unsigned int foll)
4814	{
4815	unsigned long gaddr_tmp;
4816	gfn_t gfn;
4817
4818	gfn = gpa_to_gfn(gpa: gaddr);
4819	if (kvm_is_ucontrol(kvm: vcpu->kvm)) {
4820	/*
4821	* This translates the per-vCPU guest address into a
4822	* fake guest address, which can then be used with the
4823	* fake memslots that are identity mapping userspace.
4824	* This allows ucontrol VMs to use the normal fault
4825	* resolution path, like normal VMs.
4826	*/
4827	mmap_read_lock(mm: vcpu->arch.gmap->mm);
4828	gaddr_tmp = __gmap_translate(vcpu->arch.gmap, gaddr);
4829	mmap_read_unlock(mm: vcpu->arch.gmap->mm);
4830	if (gaddr_tmp == -EFAULT) {
4831	vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
4832	vcpu->run->s390_ucontrol.trans_exc_code = gaddr;
4833	vcpu->run->s390_ucontrol.pgm_code = PGM_SEGMENT_TRANSLATION;
4834	return -EREMOTE;
4835	}
4836	gfn = gpa_to_gfn(gpa: gaddr_tmp);
4837	}
4838	return __kvm_s390_handle_dat_fault(vcpu, gfn, gaddr, foll);
4839	}
4840
4841	static int vcpu_post_run_handle_fault(struct kvm_vcpu *vcpu)
4842	{
4843	unsigned int foll = 0;
4844	unsigned long gaddr;
4845	int rc;
4846
4847	gaddr = current->thread.gmap_teid.addr * PAGE_SIZE;
4848	if (kvm_s390_cur_gmap_fault_is_write())
4849	foll = FOLL_WRITE;
4850
4851	switch (current->thread.gmap_int_code & PGM_INT_CODE_MASK) {
4852	case 0:
4853	vcpu->stat.exit_null++;
4854	break;
4855	case PGM_SECURE_STORAGE_ACCESS:
4856	case PGM_SECURE_STORAGE_VIOLATION:
4857	kvm_s390_assert_primary_as(vcpu);
4858	/*
4859	* This can happen after a reboot with asynchronous teardown;
4860	* the new guest (normal or protected) will run on top of the
4861	* previous protected guest. The old pages need to be destroyed
4862	* so the new guest can use them.
4863	*/
4864	if (kvm_s390_pv_destroy_page(kvm: vcpu->kvm, gaddr)) {
4865	/*
4866	* Either KVM messed up the secure guest mapping or the
4867	* same page is mapped into multiple secure guests.
4868	*
4869	* This exception is only triggered when a guest 2 is
4870	* running and can therefore never occur in kernel
4871	* context.
4872	*/
4873	pr_warn_ratelimited("Secure storage violation (%x) in task: %s, pid %d\n",
4874	current->thread.gmap_int_code, current->comm,
4875	current->pid);
4876	send_sig(SIGSEGV, current, 0);
4877	}
4878	break;
4879	case PGM_NON_SECURE_STORAGE_ACCESS:
4880	kvm_s390_assert_primary_as(vcpu);
4881	/*
4882	* This is normal operation; a page belonging to a protected
4883	* guest has not been imported yet. Try to import the page into
4884	* the protected guest.
4885	*/
4886	rc = kvm_s390_pv_convert_to_secure(kvm: vcpu->kvm, gaddr);
4887	if (rc == -EINVAL)
4888	send_sig(SIGSEGV, current, 0);
4889	if (rc != -ENXIO)
4890	break;
4891	foll = FOLL_WRITE;
4892	fallthrough;
4893	case PGM_PROTECTION:
4894	case PGM_SEGMENT_TRANSLATION:
4895	case PGM_PAGE_TRANSLATION:
4896	case PGM_ASCE_TYPE:
4897	case PGM_REGION_FIRST_TRANS:
4898	case PGM_REGION_SECOND_TRANS:
4899	case PGM_REGION_THIRD_TRANS:
4900	kvm_s390_assert_primary_as(vcpu);
4901	return vcpu_dat_fault_handler(vcpu, gaddr, foll);
4902	default:
4903	KVM_BUG(1, vcpu->kvm, "Unexpected program interrupt 0x%x, TEID 0x%016lx",
4904	current->thread.gmap_int_code, current->thread.gmap_teid.val);
4905	send_sig(SIGSEGV, current, 0);
4906	break;
4907	}
4908	return 0;
4909	}
4910
4911	static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
4912	{
4913	struct mcck_volatile_info *mcck_info;
4914	struct sie_page *sie_page;
4915	int rc;
4916
4917	VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
4918	vcpu->arch.sie_block->icptcode);
4919	trace_kvm_s390_sie_exit(vcpu, icptcode: vcpu->arch.sie_block->icptcode);
4920
4921	if (guestdbg_enabled(vcpu))
4922	kvm_s390_restore_guest_per_regs(vcpu);
4923
4924	vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
4925	vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
4926
4927	if (exit_reason == -EINTR) {
4928	VCPU_EVENT(vcpu, 3, "%s", "machine check");
4929	sie_page = container_of(vcpu->arch.sie_block,
4930	struct sie_page, sie_block);
4931	mcck_info = &sie_page->mcck_info;
4932	kvm_s390_reinject_machine_check(vcpu, mcck_info);
4933	return 0;
4934	}
4935
4936	if (vcpu->arch.sie_block->icptcode > 0) {
4937	rc = kvm_handle_sie_intercept(vcpu);
4938
4939	if (rc != -EOPNOTSUPP)
4940	return rc;
4941	vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
4942	vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
4943	vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
4944	vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
4945	return -EREMOTE;
4946	}
4947
4948	return vcpu_post_run_handle_fault(vcpu);
4949	}
4950
4951	int noinstr kvm_s390_enter_exit_sie(struct kvm_s390_sie_block *scb,
4952	u64 *gprs, unsigned long gasce)
4953	{
4954	int ret;
4955
4956	guest_state_enter_irqoff();
4957
4958	/*
4959	* The guest_state_{enter,exit}_irqoff() functions inform lockdep and
4960	* tracing that entry to the guest will enable host IRQs, and exit from
4961	* the guest will disable host IRQs.
4962	*/
4963	ret = sie64a(scb, gprs, gasce);
4964
4965	guest_state_exit_irqoff();
4966
4967	return ret;
4968	}
4969
4970	#define PSW_INT_MASK (PSW_MASK_EXT | PSW_MASK_IO | PSW_MASK_MCHECK)
4971	static int __vcpu_run(struct kvm_vcpu *vcpu)
4972	{
4973	int rc, exit_reason;
4974	struct sie_page *sie_page = (struct sie_page *)vcpu->arch.sie_block;
4975
4976	/*
4977	* We try to hold kvm->srcu during most of vcpu_run (except when run-
4978	* ning the guest), so that memslots (and other stuff) are protected
4979	*/
4980	kvm_vcpu_srcu_read_lock(vcpu);
4981
4982	while (true) {
4983	rc = vcpu_pre_run(vcpu);
4984	kvm_vcpu_srcu_read_unlock(vcpu);
4985	if (rc || guestdbg_exit_pending(vcpu))
4986	break;
4987
4988	/*
4989	* As PF_VCPU will be used in fault handler, between
4990	* guest_timing_enter_irqoff and guest_timing_exit_irqoff
4991	* should be no uaccess.
4992	*/
4993	if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4994	memcpy(sie_page->pv_grregs,
4995	vcpu->run->s.regs.gprs,
4996	sizeof(sie_page->pv_grregs));
4997	}
4998
4999	xfer_to_guest_mode_check:
5000	local_irq_disable();
5001	xfer_to_guest_mode_prepare();
5002	if (xfer_to_guest_mode_work_pending()) {
5003	local_irq_enable();
5004	rc = kvm_xfer_to_guest_mode_handle_work(vcpu);
5005	if (rc)
5006	break;
5007	goto xfer_to_guest_mode_check;
5008	}
5009
5010	guest_timing_enter_irqoff();
5011	__disable_cpu_timer_accounting(vcpu);
5012
5013	exit_reason = kvm_s390_enter_exit_sie(scb: vcpu->arch.sie_block,
5014	gprs: vcpu->run->s.regs.gprs,
5015	gasce: vcpu->arch.gmap->asce);
5016
5017	__enable_cpu_timer_accounting(vcpu);
5018	guest_timing_exit_irqoff();
5019	local_irq_enable();
5020
5021	if (kvm_s390_pv_cpu_is_protected(vcpu)) {
5022	memcpy(vcpu->run->s.regs.gprs,
5023	sie_page->pv_grregs,
5024	sizeof(sie_page->pv_grregs));
5025	/*
5026	* We're not allowed to inject interrupts on intercepts
5027	* that leave the guest state in an "in-between" state
5028	* where the next SIE entry will do a continuation.
5029	* Fence interrupts in our "internal" PSW.
5030	*/
5031	if (vcpu->arch.sie_block->icptcode == ICPT_PV_INSTR ||
5032	vcpu->arch.sie_block->icptcode == ICPT_PV_PREF) {
5033	vcpu->arch.sie_block->gpsw.mask &= ~PSW_INT_MASK;
5034	}
5035	}
5036	kvm_vcpu_srcu_read_lock(vcpu);
5037
5038	rc = vcpu_post_run(vcpu, exit_reason);
5039	if (rc || guestdbg_exit_pending(vcpu)) {
5040	kvm_vcpu_srcu_read_unlock(vcpu);
5041	break;
5042	}
5043	}
5044
5045	return rc;
5046	}
5047
5048	static void sync_regs_fmt2(struct kvm_vcpu *vcpu)
5049	{
5050	struct kvm_run *kvm_run = vcpu->run;
5051	struct runtime_instr_cb *riccb;
5052	struct gs_cb *gscb;
5053
5054	riccb = (struct runtime_instr_cb *) &kvm_run->s.regs.riccb;
5055	gscb = (struct gs_cb *) &kvm_run->s.regs.gscb;
5056	vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
5057	vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
5058	if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
5059	vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
5060	vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
5061	vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
5062	}
5063	if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
5064	vcpu->arch.pfault_token = kvm_run->s.regs.pft;
5065	vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
5066	vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
5067	if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
5068	kvm_clear_async_pf_completion_queue(vcpu);
5069	}
5070	if (kvm_run->kvm_dirty_regs & KVM_SYNC_DIAG318) {
5071	vcpu->arch.diag318_info.val = kvm_run->s.regs.diag318;
5072	vcpu->arch.sie_block->cpnc = vcpu->arch.diag318_info.cpnc;
5073	VCPU_EVENT(vcpu, 3, "setting cpnc to %d", vcpu->arch.diag318_info.cpnc);
5074	}
5075	/*
5076	* If userspace sets the riccb (e.g. after migration) to a valid state,
5077	* we should enable RI here instead of doing the lazy enablement.
5078	*/
5079	if ((kvm_run->kvm_dirty_regs & KVM_SYNC_RICCB) &&
5080	test_kvm_facility(vcpu->kvm, 64) &&
5081	riccb->v &&
5082	!(vcpu->arch.sie_block->ecb3 & ECB3_RI)) {
5083	VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (sync_regs)");
5084	vcpu->arch.sie_block->ecb3 |= ECB3_RI;
5085	}
5086	/*
5087	* If userspace sets the gscb (e.g. after migration) to non-zero,
5088	* we should enable GS here instead of doing the lazy enablement.
5089	*/
5090	if ((kvm_run->kvm_dirty_regs & KVM_SYNC_GSCB) &&
5091	test_kvm_facility(vcpu->kvm, 133) &&
5092	gscb->gssm &&
5093	!vcpu->arch.gs_enabled) {
5094	VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (sync_regs)");
5095	vcpu->arch.sie_block->ecb |= ECB_GS;
5096	vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
5097	vcpu->arch.gs_enabled = 1;
5098	}
5099	if ((kvm_run->kvm_dirty_regs & KVM_SYNC_BPBC) &&
5100	test_kvm_facility(vcpu->kvm, 82)) {
5101	vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
5102	vcpu->arch.sie_block->fpf |= kvm_run->s.regs.bpbc ? FPF_BPBC : 0;
5103	}
5104	if (cpu_has_gs()) {
5105	preempt_disable();
5106	local_ctl_set_bit(2, CR2_GUARDED_STORAGE_BIT);
5107	if (current->thread.gs_cb) {
5108	vcpu->arch.host_gscb = current->thread.gs_cb;
5109	save_gs_cb(vcpu->arch.host_gscb);
5110	}
5111	if (vcpu->arch.gs_enabled) {
5112	current->thread.gs_cb = (struct gs_cb *)
5113	&vcpu->run->s.regs.gscb;
5114	restore_gs_cb(current->thread.gs_cb);
5115	}
5116	preempt_enable();
5117	}
5118	/* SIE will load etoken directly from SDNX and therefore kvm_run */
5119	}
5120
5121	static void sync_regs(struct kvm_vcpu *vcpu)
5122	{
5123	struct kvm_run *kvm_run = vcpu->run;
5124
5125	if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
5126	kvm_s390_set_prefix(vcpu, prefix: kvm_run->s.regs.prefix);
5127	if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
5128	memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
5129	/* some control register changes require a tlb flush */
5130	kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
5131	}
5132	if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
5133	kvm_s390_set_cpu_timer(vcpu, cputm: kvm_run->s.regs.cputm);
5134	vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
5135	}
5136	save_access_regs(vcpu->arch.host_acrs);
5137	restore_access_regs(vcpu->run->s.regs.acrs);
5138	vcpu->arch.acrs_loaded = true;
5139	kvm_s390_fpu_load(run: vcpu->run);
5140	/* Sync fmt2 only data */
5141	if (likely(!kvm_s390_pv_cpu_is_protected(vcpu))) {
5142	sync_regs_fmt2(vcpu);
5143	} else {
5144	/*
5145	* In several places we have to modify our internal view to
5146	* not do things that are disallowed by the ultravisor. For
5147	* example we must not inject interrupts after specific exits
5148	* (e.g. 112 prefix page not secure). We do this by turning
5149	* off the machine check, external and I/O interrupt bits
5150	* of our PSW copy. To avoid getting validity intercepts, we
5151	* do only accept the condition code from userspace.
5152	*/
5153	vcpu->arch.sie_block->gpsw.mask &= ~PSW_MASK_CC;
5154	vcpu->arch.sie_block->gpsw.mask |= kvm_run->psw_mask &
5155	PSW_MASK_CC;
5156	}
5157
5158	kvm_run->kvm_dirty_regs = 0;
5159	}
5160
5161	static void store_regs_fmt2(struct kvm_vcpu *vcpu)
5162	{
5163	struct kvm_run *kvm_run = vcpu->run;
5164
5165	kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
5166	kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
5167	kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
5168	kvm_run->s.regs.bpbc = (vcpu->arch.sie_block->fpf & FPF_BPBC) == FPF_BPBC;
5169	kvm_run->s.regs.diag318 = vcpu->arch.diag318_info.val;
5170	if (cpu_has_gs()) {
5171	preempt_disable();
5172	local_ctl_set_bit(2, CR2_GUARDED_STORAGE_BIT);
5173	if (vcpu->arch.gs_enabled)
5174	save_gs_cb(current->thread.gs_cb);
5175	current->thread.gs_cb = vcpu->arch.host_gscb;
5176	restore_gs_cb(vcpu->arch.host_gscb);
5177	if (!vcpu->arch.host_gscb)
5178	local_ctl_clear_bit(2, CR2_GUARDED_STORAGE_BIT);
5179	vcpu->arch.host_gscb = NULL;
5180	preempt_enable();
5181	}
5182	/* SIE will save etoken directly into SDNX and therefore kvm_run */
5183	}
5184
5185	static void store_regs(struct kvm_vcpu *vcpu)
5186	{
5187	struct kvm_run *kvm_run = vcpu->run;
5188
5189	kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
5190	kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
5191	kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
5192	memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
5193	kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
5194	kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
5195	kvm_run->s.regs.pft = vcpu->arch.pfault_token;
5196	kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
5197	kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
5198	save_access_regs(vcpu->run->s.regs.acrs);
5199	restore_access_regs(vcpu->arch.host_acrs);
5200	vcpu->arch.acrs_loaded = false;
5201	kvm_s390_fpu_store(run: vcpu->run);
5202	if (likely(!kvm_s390_pv_cpu_is_protected(vcpu)))
5203	store_regs_fmt2(vcpu);
5204	}
5205
5206	int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
5207	{
5208	struct kvm_run *kvm_run = vcpu->run;
5209	DECLARE_KERNEL_FPU_ONSTACK32(fpu);
5210	int rc;
5211
5212	/*
5213	* Running a VM while dumping always has the potential to
5214	* produce inconsistent dump data. But for PV vcpus a SIE
5215	* entry while dumping could also lead to a fatal validity
5216	* intercept which we absolutely want to avoid.
5217	*/
5218	if (vcpu->kvm->arch.pv.dumping)
5219	return -EINVAL;
5220
5221	if (!vcpu->wants_to_run)
5222	return -EINTR;
5223
5224	if (kvm_run->kvm_valid_regs & ~KVM_SYNC_S390_VALID_FIELDS ||
5225	kvm_run->kvm_dirty_regs & ~KVM_SYNC_S390_VALID_FIELDS)
5226	return -EINVAL;
5227
5228	vcpu_load(vcpu);
5229
5230	if (guestdbg_exit_pending(vcpu)) {
5231	kvm_s390_prepare_debug_exit(vcpu);
5232	rc = 0;
5233	goto out;
5234	}
5235
5236	kvm_sigset_activate(vcpu);
5237
5238	/*
5239	* no need to check the return value of vcpu_start as it can only have
5240	* an error for protvirt, but protvirt means user cpu state
5241	*/
5242	if (!kvm_s390_user_cpu_state_ctrl(kvm: vcpu->kvm)) {
5243	kvm_s390_vcpu_start(vcpu);
5244	} else if (is_vcpu_stopped(vcpu)) {
5245	pr_err_ratelimited("can't run stopped vcpu %d\n",
5246	vcpu->vcpu_id);
5247	rc = -EINVAL;
5248	goto out;
5249	}
5250
5251	kernel_fpu_begin(&fpu, KERNEL_FPC | KERNEL_VXR);
5252	sync_regs(vcpu);
5253	enable_cpu_timer_accounting(vcpu);
5254
5255	might_fault();
5256	rc = __vcpu_run(vcpu);
5257
5258	if (signal_pending(current) && !rc) {
5259	kvm_run->exit_reason = KVM_EXIT_INTR;
5260	vcpu->stat.signal_exits++;
5261	rc = -EINTR;
5262	}
5263
5264	if (guestdbg_exit_pending(vcpu) && !rc) {
5265	kvm_s390_prepare_debug_exit(vcpu);
5266	rc = 0;
5267	}
5268
5269	if (rc == -EREMOTE) {
5270	/* userspace support is needed, kvm_run has been prepared */
5271	rc = 0;
5272	}
5273
5274	disable_cpu_timer_accounting(vcpu);
5275	store_regs(vcpu);
5276	kernel_fpu_end(&fpu, KERNEL_FPC | KERNEL_VXR);
5277
5278	kvm_sigset_deactivate(vcpu);
5279
5280	vcpu->stat.exit_userspace++;
5281	out:
5282	vcpu_put(vcpu);
5283	return rc;
5284	}
5285
5286	/*
5287	* store status at address
5288	* we use have two special cases:
5289	* KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
5290	* KVM_S390_STORE_STATUS_PREFIXED: -> prefix
5291	*/
5292	int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
5293	{
5294	unsigned char archmode = 1;
5295	freg_t fprs[NUM_FPRS];
5296	unsigned int px;
5297	u64 clkcomp, cputm;
5298	int rc;
5299
5300	px = kvm_s390_get_prefix(vcpu);
5301	if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
5302	if (write_guest_abs(vcpu, gpa: 163, data: &archmode, len: 1))
5303	return -EFAULT;
5304	gpa = 0;
5305	} else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
5306	if (write_guest_real(vcpu, gra: 163, data: &archmode, len: 1))
5307	return -EFAULT;
5308	gpa = px;
5309	} else
5310	gpa -= __LC_FPREGS_SAVE_AREA;
5311
5312	/* manually convert vector registers if necessary */
5313	if (cpu_has_vx()) {
5314	convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
5315	rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
5316	fprs, 128);
5317	} else {
5318	rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
5319	vcpu->run->s.regs.fprs, 128);
5320	}
5321	rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
5322	vcpu->run->s.regs.gprs, 128);
5323	rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
5324	&vcpu->arch.sie_block->gpsw, 16);
5325	rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
5326	&px, 4);
5327	rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
5328	&vcpu->run->s.regs.fpc, 4);
5329	rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
5330	&vcpu->arch.sie_block->todpr, 4);
5331	cputm = kvm_s390_get_cpu_timer(vcpu);
5332	rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
5333	&cputm, 8);
5334	clkcomp = vcpu->arch.sie_block->ckc >> 8;
5335	rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
5336	&clkcomp, 8);
5337	rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
5338	&vcpu->run->s.regs.acrs, 64);
5339	rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
5340	&vcpu->arch.sie_block->gcr, 128);
5341	return rc ? -EFAULT : 0;
5342	}
5343
5344	int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
5345	{
5346	/*
5347	* The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
5348	* switch in the run ioctl. Let's update our copies before we save
5349	* it into the save area
5350	*/
5351	kvm_s390_fpu_store(run: vcpu->run);
5352	save_access_regs(vcpu->run->s.regs.acrs);
5353
5354	return kvm_s390_store_status_unloaded(vcpu, gpa: addr);
5355	}
5356
5357	static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
5358	{
5359	kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
5360	kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
5361	}
5362
5363	static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
5364	{
5365	unsigned long i;
5366	struct kvm_vcpu *vcpu;
5367
5368	kvm_for_each_vcpu(i, vcpu, kvm) {
5369	__disable_ibs_on_vcpu(vcpu);
5370	}
5371	}
5372
5373	static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
5374	{
5375	if (!sclp.has_ibs)
5376	return;
5377	kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
5378	kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
5379	}
5380
5381	int kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
5382	{
5383	int i, online_vcpus, r = 0, started_vcpus = 0;
5384
5385	if (!is_vcpu_stopped(vcpu))
5386	return 0;
5387
5388	trace_kvm_s390_vcpu_start_stop(id: vcpu->vcpu_id, state: 1);
5389	/* Only one cpu at a time may enter/leave the STOPPED state. */
5390	spin_lock(lock: &vcpu->kvm->arch.start_stop_lock);
5391	online_vcpus = atomic_read(v: &vcpu->kvm->online_vcpus);
5392
5393	/* Let's tell the UV that we want to change into the operating state */
5394	if (kvm_s390_pv_cpu_is_protected(vcpu)) {
5395	r = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_OPR);
5396	if (r) {
5397	spin_unlock(lock: &vcpu->kvm->arch.start_stop_lock);
5398	return r;
5399	}
5400	}
5401
5402	for (i = 0; i < online_vcpus; i++) {
5403	if (!is_vcpu_stopped(vcpu: kvm_get_vcpu(kvm: vcpu->kvm, i)))
5404	started_vcpus++;
5405	}
5406
5407	if (started_vcpus == 0) {
5408	/* we're the only active VCPU -> speed it up */
5409	__enable_ibs_on_vcpu(vcpu);
5410	} else if (started_vcpus == 1) {
5411	/*
5412	* As we are starting a second VCPU, we have to disable
5413	* the IBS facility on all VCPUs to remove potentially
5414	* outstanding ENABLE requests.
5415	*/
5416	__disable_ibs_on_all_vcpus(kvm: vcpu->kvm);
5417	}
5418
5419	kvm_s390_clear_cpuflags(vcpu, CPUSTAT_STOPPED);
5420	/*
5421	* The real PSW might have changed due to a RESTART interpreted by the
5422	* ultravisor. We block all interrupts and let the next sie exit
5423	* refresh our view.
5424	*/
5425	if (kvm_s390_pv_cpu_is_protected(vcpu))
5426	vcpu->arch.sie_block->gpsw.mask &= ~PSW_INT_MASK;
5427	/*
5428	* Another VCPU might have used IBS while we were offline.
5429	* Let's play safe and flush the VCPU at startup.
5430	*/
5431	kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
5432	spin_unlock(lock: &vcpu->kvm->arch.start_stop_lock);
5433	return 0;
5434	}
5435
5436	int kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
5437	{
5438	int i, online_vcpus, r = 0, started_vcpus = 0;
5439	struct kvm_vcpu *started_vcpu = NULL;
5440
5441	if (is_vcpu_stopped(vcpu))
5442	return 0;
5443
5444	trace_kvm_s390_vcpu_start_stop(id: vcpu->vcpu_id, state: 0);
5445	/* Only one cpu at a time may enter/leave the STOPPED state. */
5446	spin_lock(lock: &vcpu->kvm->arch.start_stop_lock);
5447	online_vcpus = atomic_read(v: &vcpu->kvm->online_vcpus);
5448
5449	/* Let's tell the UV that we want to change into the stopped state */
5450	if (kvm_s390_pv_cpu_is_protected(vcpu)) {
5451	r = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_STP);
5452	if (r) {
5453	spin_unlock(lock: &vcpu->kvm->arch.start_stop_lock);
5454	return r;
5455	}
5456	}
5457
5458	/*
5459	* Set the VCPU to STOPPED and THEN clear the interrupt flag,
5460	* now that the SIGP STOP and SIGP STOP AND STORE STATUS orders
5461	* have been fully processed. This will ensure that the VCPU
5462	* is kept BUSY if another VCPU is inquiring with SIGP SENSE.
5463	*/
5464	kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOPPED);
5465	kvm_s390_clear_stop_irq(vcpu);
5466
5467	__disable_ibs_on_vcpu(vcpu);
5468
5469	for (i = 0; i < online_vcpus; i++) {
5470	struct kvm_vcpu *tmp = kvm_get_vcpu(kvm: vcpu->kvm, i);
5471
5472	if (!is_vcpu_stopped(vcpu: tmp)) {
5473	started_vcpus++;
5474	started_vcpu = tmp;
5475	}
5476	}
5477
5478	if (started_vcpus == 1) {
5479	/*
5480	* As we only have one VCPU left, we want to enable the
5481	* IBS facility for that VCPU to speed it up.
5482	*/
5483	__enable_ibs_on_vcpu(vcpu: started_vcpu);
5484	}
5485
5486	spin_unlock(lock: &vcpu->kvm->arch.start_stop_lock);
5487	return 0;
5488	}
5489
5490	static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
5491	struct kvm_enable_cap *cap)
5492	{
5493	int r;
5494
5495	if (cap->flags)
5496	return -EINVAL;
5497
5498	switch (cap->cap) {
5499	case KVM_CAP_S390_CSS_SUPPORT:
5500	if (!vcpu->kvm->arch.css_support) {
5501	vcpu->kvm->arch.css_support = 1;
5502	VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
5503	trace_kvm_s390_enable_css(kvm: vcpu->kvm);
5504	}
5505	r = 0;
5506	break;
5507	default:
5508	r = -EINVAL;
5509	break;
5510	}
5511	return r;
5512	}
5513
5514	static long kvm_s390_vcpu_sida_op(struct kvm_vcpu *vcpu,
5515	struct kvm_s390_mem_op *mop)
5516	{
5517	void __user *uaddr = (void __user *)mop->buf;
5518	void *sida_addr;
5519	int r = 0;
5520
5521	if (mop->flags || !mop->size)
5522	return -EINVAL;
5523	if (mop->size + mop->sida_offset < mop->size)
5524	return -EINVAL;
5525	if (mop->size + mop->sida_offset > sida_size(vcpu->arch.sie_block))
5526	return -E2BIG;
5527	if (!kvm_s390_pv_cpu_is_protected(vcpu))
5528	return -EINVAL;
5529
5530	sida_addr = (char *)sida_addr(vcpu->arch.sie_block) + mop->sida_offset;
5531
5532	switch (mop->op) {
5533	case KVM_S390_MEMOP_SIDA_READ:
5534	if (copy_to_user(to: uaddr, from: sida_addr, n: mop->size))
5535	r = -EFAULT;
5536
5537	break;
5538	case KVM_S390_MEMOP_SIDA_WRITE:
5539	if (copy_from_user(to: sida_addr, from: uaddr, n: mop->size))
5540	r = -EFAULT;
5541	break;
5542	}
5543	return r;
5544	}
5545
5546	static long kvm_s390_vcpu_mem_op(struct kvm_vcpu *vcpu,
5547	struct kvm_s390_mem_op *mop)
5548	{
5549	void __user *uaddr = (void __user *)mop->buf;
5550	enum gacc_mode acc_mode;
5551	void *tmpbuf = NULL;
5552	int r;
5553
5554	r = mem_op_validate_common(mop, KVM_S390_MEMOP_F_INJECT_EXCEPTION |
5555	KVM_S390_MEMOP_F_CHECK_ONLY |
5556	KVM_S390_MEMOP_F_SKEY_PROTECTION);
5557	if (r)
5558	return r;
5559	if (mop->ar >= NUM_ACRS)
5560	return -EINVAL;
5561	if (kvm_s390_pv_cpu_is_protected(vcpu))
5562	return -EINVAL;
5563	if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
5564	tmpbuf = vmalloc(mop->size);
5565	if (!tmpbuf)
5566	return -ENOMEM;
5567	}
5568
5569	acc_mode = mop->op == KVM_S390_MEMOP_LOGICAL_READ ? GACC_FETCH : GACC_STORE;
5570	if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
5571	r = check_gva_range(vcpu, gva: mop->gaddr, ar: mop->ar, length: mop->size,
5572	mode: acc_mode, access_key: mop->key);
5573	goto out_inject;
5574	}
5575	if (acc_mode == GACC_FETCH) {
5576	r = read_guest_with_key(vcpu, ga: mop->gaddr, ar: mop->ar, data: tmpbuf,
5577	len: mop->size, access_key: mop->key);
5578	if (r)
5579	goto out_inject;
5580	if (copy_to_user(to: uaddr, from: tmpbuf, n: mop->size)) {
5581	r = -EFAULT;
5582	goto out_free;
5583	}
5584	} else {
5585	if (copy_from_user(to: tmpbuf, from: uaddr, n: mop->size)) {
5586	r = -EFAULT;
5587	goto out_free;
5588	}
5589	r = write_guest_with_key(vcpu, ga: mop->gaddr, ar: mop->ar, data: tmpbuf,
5590	len: mop->size, access_key: mop->key);
5591	}
5592
5593	out_inject:
5594	if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
5595	kvm_s390_inject_prog_irq(vcpu, pgm_info: &vcpu->arch.pgm);
5596
5597	out_free:
5598	vfree(addr: tmpbuf);
5599	return r;
5600	}
5601
5602	static long kvm_s390_vcpu_memsida_op(struct kvm_vcpu *vcpu,
5603	struct kvm_s390_mem_op *mop)
5604	{
5605	int r, srcu_idx;
5606
5607	srcu_idx = srcu_read_lock(ssp: &vcpu->kvm->srcu);
5608
5609	switch (mop->op) {
5610	case KVM_S390_MEMOP_LOGICAL_READ:
5611	case KVM_S390_MEMOP_LOGICAL_WRITE:
5612	r = kvm_s390_vcpu_mem_op(vcpu, mop);
5613	break;
5614	case KVM_S390_MEMOP_SIDA_READ:
5615	case KVM_S390_MEMOP_SIDA_WRITE:
5616	/* we are locked against sida going away by the vcpu->mutex */
5617	r = kvm_s390_vcpu_sida_op(vcpu, mop);
5618	break;
5619	default:
5620	r = -EINVAL;
5621	}
5622
5623	srcu_read_unlock(ssp: &vcpu->kvm->srcu, idx: srcu_idx);
5624	return r;
5625	}
5626
5627	long kvm_arch_vcpu_unlocked_ioctl(struct file *filp, unsigned int ioctl,
5628	unsigned long arg)
5629	{
5630	struct kvm_vcpu *vcpu = filp->private_data;
5631	void __user *argp = (void __user *)arg;
5632	int rc;
5633
5634	switch (ioctl) {
5635	case KVM_S390_IRQ: {
5636	struct kvm_s390_irq s390irq;
5637
5638	if (copy_from_user(to: &s390irq, from: argp, n: sizeof(s390irq)))
5639	return -EFAULT;
5640	rc = kvm_s390_inject_vcpu(vcpu, irq: &s390irq);
5641	break;
5642	}
5643	case KVM_S390_INTERRUPT: {
5644	struct kvm_s390_interrupt s390int;
5645	struct kvm_s390_irq s390irq = {};
5646
5647	if (copy_from_user(to: &s390int, from: argp, n: sizeof(s390int)))
5648	return -EFAULT;
5649	if (s390int_to_s390irq(s390int: &s390int, s390irq: &s390irq))
5650	return -EINVAL;
5651	rc = kvm_s390_inject_vcpu(vcpu, irq: &s390irq);
5652	break;
5653	}
5654	default:
5655	rc = -ENOIOCTLCMD;
5656	break;
5657	}
5658
5659	/*
5660	* To simplify single stepping of userspace-emulated instructions,
5661	* KVM_EXIT_S390_SIEIC exit sets KVM_GUESTDBG_EXIT_PENDING (see
5662	* should_handle_per_ifetch()). However, if userspace emulation injects
5663	* an interrupt, it needs to be cleared, so that KVM_EXIT_DEBUG happens
5664	* after (and not before) the interrupt delivery.
5665	*/
5666	if (!rc)
5667	vcpu->guest_debug &= ~KVM_GUESTDBG_EXIT_PENDING;
5668
5669	return rc;
5670	}
5671
5672	static int kvm_s390_handle_pv_vcpu_dump(struct kvm_vcpu *vcpu,
5673	struct kvm_pv_cmd *cmd)
5674	{
5675	struct kvm_s390_pv_dmp dmp;
5676	void *data;
5677	int ret;
5678
5679	/* Dump initialization is a prerequisite */
5680	if (!vcpu->kvm->arch.pv.dumping)
5681	return -EINVAL;
5682
5683	if (copy_from_user(to: &dmp, from: (__u8 __user *)cmd->data, n: sizeof(dmp)))
5684	return -EFAULT;
5685
5686	/* We only handle this subcmd right now */
5687	if (dmp.subcmd != KVM_PV_DUMP_CPU)
5688	return -EINVAL;
5689
5690	/* CPU dump length is the same as create cpu storage donation. */
5691	if (dmp.buff_len != uv_info.guest_cpu_stor_len)
5692	return -EINVAL;
5693
5694	data = kvzalloc(uv_info.guest_cpu_stor_len, GFP_KERNEL);
5695	if (!data)
5696	return -ENOMEM;
5697
5698	ret = kvm_s390_pv_dump_cpu(vcpu, buff: data, rc: &cmd->rc, rrc: &cmd->rrc);
5699
5700	VCPU_EVENT(vcpu, 3, "PROTVIRT DUMP CPU %d rc %x rrc %x",
5701	vcpu->vcpu_id, cmd->rc, cmd->rrc);
5702
5703	if (ret)
5704	ret = -EINVAL;
5705
5706	/* On success copy over the dump data */
5707	if (!ret && copy_to_user((__u8 __user *)dmp.buff_addr, data, uv_info.guest_cpu_stor_len))
5708	ret = -EFAULT;
5709
5710	kvfree(addr: data);
5711	return ret;
5712	}
5713
5714	long kvm_arch_vcpu_ioctl(struct file *filp,
5715	unsigned int ioctl, unsigned long arg)
5716	{
5717	struct kvm_vcpu *vcpu = filp->private_data;
5718	void __user *argp = (void __user *)arg;
5719	int idx;
5720	long r;
5721	u16 rc, rrc;
5722
5723	vcpu_load(vcpu);
5724
5725	switch (ioctl) {
5726	case KVM_S390_STORE_STATUS:
5727	idx = srcu_read_lock(ssp: &vcpu->kvm->srcu);
5728	r = kvm_s390_store_status_unloaded(vcpu, gpa: arg);
5729	srcu_read_unlock(ssp: &vcpu->kvm->srcu, idx);
5730	break;
5731	case KVM_S390_SET_INITIAL_PSW: {
5732	psw_t psw;
5733
5734	r = -EFAULT;
5735	if (copy_from_user(&psw, argp, sizeof(psw)))
5736	break;
5737	r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
5738	break;
5739	}
5740	case KVM_S390_CLEAR_RESET:
5741	r = 0;
5742	kvm_arch_vcpu_ioctl_clear_reset(vcpu);
5743	if (kvm_s390_pv_cpu_is_protected(vcpu)) {
5744	r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu),
5745	UVC_CMD_CPU_RESET_CLEAR, &rc, &rrc);
5746	VCPU_EVENT(vcpu, 3, "PROTVIRT RESET CLEAR VCPU: rc %x rrc %x",
5747	rc, rrc);
5748	}
5749	break;
5750	case KVM_S390_INITIAL_RESET:
5751	r = 0;
5752	kvm_arch_vcpu_ioctl_initial_reset(vcpu);
5753	if (kvm_s390_pv_cpu_is_protected(vcpu)) {
5754	r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu),
5755	UVC_CMD_CPU_RESET_INITIAL,
5756	&rc, &rrc);
5757	VCPU_EVENT(vcpu, 3, "PROTVIRT RESET INITIAL VCPU: rc %x rrc %x",
5758	rc, rrc);
5759	}
5760	break;
5761	case KVM_S390_NORMAL_RESET:
5762	r = 0;
5763	kvm_arch_vcpu_ioctl_normal_reset(vcpu);
5764	if (kvm_s390_pv_cpu_is_protected(vcpu)) {
5765	r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu),
5766	UVC_CMD_CPU_RESET, &rc, &rrc);
5767	VCPU_EVENT(vcpu, 3, "PROTVIRT RESET NORMAL VCPU: rc %x rrc %x",
5768	rc, rrc);
5769	}
5770	break;
5771	case KVM_SET_ONE_REG:
5772	case KVM_GET_ONE_REG: {
5773	struct kvm_one_reg reg;
5774	r = -EINVAL;
5775	if (kvm_s390_pv_cpu_is_protected(vcpu))
5776	break;
5777	r = -EFAULT;
5778	if (copy_from_user(to: &reg, from: argp, n: sizeof(reg)))
5779	break;
5780	if (ioctl == KVM_SET_ONE_REG)
5781	r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, reg: &reg);
5782	else
5783	r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, reg: &reg);
5784	break;
5785	}
5786	#ifdef CONFIG_KVM_S390_UCONTROL
5787	case KVM_S390_UCAS_MAP: {
5788	struct kvm_s390_ucas_mapping ucasmap;
5789
5790	if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
5791	r = -EFAULT;
5792	break;
5793	}
5794
5795	if (!kvm_is_ucontrol(vcpu->kvm)) {
5796	r = -EINVAL;
5797	break;
5798	}
5799
5800	r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
5801	ucasmap.vcpu_addr, ucasmap.length);
5802	break;
5803	}
5804	case KVM_S390_UCAS_UNMAP: {
5805	struct kvm_s390_ucas_mapping ucasmap;
5806
5807	if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
5808	r = -EFAULT;
5809	break;
5810	}
5811
5812	if (!kvm_is_ucontrol(vcpu->kvm)) {
5813	r = -EINVAL;
5814	break;
5815	}
5816
5817	r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
5818	ucasmap.length);
5819	break;
5820	}
5821	#endif
5822	case KVM_S390_VCPU_FAULT: {
5823	idx = srcu_read_lock(ssp: &vcpu->kvm->srcu);
5824	r = vcpu_dat_fault_handler(vcpu, gaddr: arg, foll: 0);
5825	srcu_read_unlock(ssp: &vcpu->kvm->srcu, idx);
5826	break;
5827	}
5828	case KVM_ENABLE_CAP:
5829	{
5830	struct kvm_enable_cap cap;
5831	r = -EFAULT;
5832	if (copy_from_user(to: &cap, from: argp, n: sizeof(cap)))
5833	break;
5834	r = kvm_vcpu_ioctl_enable_cap(vcpu, cap: &cap);
5835	break;
5836	}
5837	case KVM_S390_MEM_OP: {
5838	struct kvm_s390_mem_op mem_op;
5839
5840	if (copy_from_user(to: &mem_op, from: argp, n: sizeof(mem_op)) == 0)
5841	r = kvm_s390_vcpu_memsida_op(vcpu, mop: &mem_op);
5842	else
5843	r = -EFAULT;
5844	break;
5845	}
5846	case KVM_S390_SET_IRQ_STATE: {
5847	struct kvm_s390_irq_state irq_state;
5848
5849	r = -EFAULT;
5850	if (copy_from_user(to: &irq_state, from: argp, n: sizeof(irq_state)))
5851	break;
5852	if (irq_state.len > VCPU_IRQS_MAX_BUF ||
5853	irq_state.len == 0 ||
5854	irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
5855	r = -EINVAL;
5856	break;
5857	}
5858	/* do not use irq_state.flags, it will break old QEMUs */
5859	r = kvm_s390_set_irq_state(vcpu,
5860	buf: (void __user *) irq_state.buf,
5861	len: irq_state.len);
5862	break;
5863	}
5864	case KVM_S390_GET_IRQ_STATE: {
5865	struct kvm_s390_irq_state irq_state;
5866
5867	r = -EFAULT;
5868	if (copy_from_user(to: &irq_state, from: argp, n: sizeof(irq_state)))
5869	break;
5870	if (irq_state.len == 0) {
5871	r = -EINVAL;
5872	break;
5873	}
5874	/* do not use irq_state.flags, it will break old QEMUs */
5875	r = kvm_s390_get_irq_state(vcpu,
5876	buf: (__u8 __user *) irq_state.buf,
5877	len: irq_state.len);
5878	break;
5879	}
5880	case KVM_S390_PV_CPU_COMMAND: {
5881	struct kvm_pv_cmd cmd;
5882
5883	r = -EINVAL;
5884	if (!is_prot_virt_host())
5885	break;
5886
5887	r = -EFAULT;
5888	if (copy_from_user(to: &cmd, from: argp, n: sizeof(cmd)))
5889	break;
5890
5891	r = -EINVAL;
5892	if (cmd.flags)
5893	break;
5894
5895	/* We only handle this cmd right now */
5896	if (cmd.cmd != KVM_PV_DUMP)
5897	break;
5898
5899	r = kvm_s390_handle_pv_vcpu_dump(vcpu, cmd: &cmd);
5900
5901	/* Always copy over UV rc / rrc data */
5902	if (copy_to_user(to: (__u8 __user *)argp, from: &cmd.rc,
5903	n: sizeof(cmd.rc) + sizeof(cmd.rrc)))
5904	r = -EFAULT;
5905	break;
5906	}
5907	default:
5908	r = -ENOTTY;
5909	}
5910
5911	vcpu_put(vcpu);
5912	return r;
5913	}
5914
5915	vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
5916	{
5917	#ifdef CONFIG_KVM_S390_UCONTROL
5918	if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
5919	&& (kvm_is_ucontrol(vcpu->kvm))) {
5920	vmf->page = virt_to_page(vcpu->arch.sie_block);
5921	get_page(vmf->page);
5922	return 0;
5923	}
5924	#endif
5925	return VM_FAULT_SIGBUS;
5926	}
5927
5928	bool kvm_arch_irqchip_in_kernel(struct kvm *kvm)
5929	{
5930	return true;
5931	}
5932
5933	/* Section: memory related */
5934	int kvm_arch_prepare_memory_region(struct kvm *kvm,
5935	const struct kvm_memory_slot *old,
5936	struct kvm_memory_slot *new,
5937	enum kvm_mr_change change)
5938	{
5939	gpa_t size;
5940
5941	if (kvm_is_ucontrol(kvm) && new->id < KVM_USER_MEM_SLOTS)
5942	return -EINVAL;
5943
5944	/* When we are protected, we should not change the memory slots */
5945	if (kvm_s390_pv_get_handle(kvm))
5946	return -EINVAL;
5947
5948	if (change != KVM_MR_DELETE && change != KVM_MR_FLAGS_ONLY) {
5949	/*
5950	* A few sanity checks. We can have memory slots which have to be
5951	* located/ended at a segment boundary (1MB). The memory in userland is
5952	* ok to be fragmented into various different vmas. It is okay to mmap()
5953	* and munmap() stuff in this slot after doing this call at any time
5954	*/
5955
5956	if (new->userspace_addr & 0xffffful)
5957	return -EINVAL;
5958
5959	size = new->npages * PAGE_SIZE;
5960	if (size & 0xffffful)
5961	return -EINVAL;
5962
5963	if ((new->base_gfn * PAGE_SIZE) + size > kvm->arch.mem_limit)
5964	return -EINVAL;
5965	}
5966
5967	if (!kvm->arch.migration_mode)
5968	return 0;
5969
5970	/*
5971	* Turn off migration mode when:
5972	* - userspace creates a new memslot with dirty logging off,
5973	* - userspace modifies an existing memslot (MOVE or FLAGS_ONLY) and
5974	* dirty logging is turned off.
5975	* Migration mode expects dirty page logging being enabled to store
5976	* its dirty bitmap.
5977	*/
5978	if (change != KVM_MR_DELETE &&
5979	!(new->flags & KVM_MEM_LOG_DIRTY_PAGES))
5980	WARN(kvm_s390_vm_stop_migration(kvm),
5981	"Failed to stop migration mode");
5982
5983	return 0;
5984	}
5985
5986	void kvm_arch_commit_memory_region(struct kvm *kvm,
5987	struct kvm_memory_slot *old,
5988	const struct kvm_memory_slot *new,
5989	enum kvm_mr_change change)
5990	{
5991	int rc = 0;
5992
5993	if (kvm_is_ucontrol(kvm))
5994	return;
5995
5996	switch (change) {
5997	case KVM_MR_DELETE:
5998	rc = gmap_unmap_segment(kvm->arch.gmap, old->base_gfn * PAGE_SIZE,
5999	old->npages * PAGE_SIZE);
6000	break;
6001	case KVM_MR_MOVE:
6002	rc = gmap_unmap_segment(kvm->arch.gmap, old->base_gfn * PAGE_SIZE,
6003	old->npages * PAGE_SIZE);
6004	if (rc)
6005	break;
6006	fallthrough;
6007	case KVM_MR_CREATE:
6008	rc = gmap_map_segment(kvm->arch.gmap, new->userspace_addr,
6009	new->base_gfn * PAGE_SIZE,
6010	new->npages * PAGE_SIZE);
6011	break;
6012	case KVM_MR_FLAGS_ONLY:
6013	break;
6014	default:
6015	WARN(1, "Unknown KVM MR CHANGE: %d\n", change);
6016	}
6017	if (rc)
6018	pr_warn("failed to commit memory region\n");
6019	return;
6020	}
6021
6022	static inline unsigned long nonhyp_mask(int i)
6023	{
6024	unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
6025
6026	return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
6027	}
6028
6029	static int __init kvm_s390_init(void)
6030	{
6031	int i, r;
6032
6033	if (!sclp.has_sief2) {
6034	pr_info("SIE is not available\n");
6035	return -ENODEV;
6036	}
6037
6038	if (nested && hpage) {
6039	pr_info("A KVM host that supports nesting cannot back its KVM guests with huge pages\n");
6040	return -EINVAL;
6041	}
6042
6043	for (i = 0; i < 16; i++)
6044	kvm_s390_fac_base[i] |=
6045	stfle_fac_list[i] & nonhyp_mask(i);
6046
6047	r = __kvm_s390_init();
6048	if (r)
6049	return r;
6050
6051	r = kvm_init(vcpu_size: sizeof(struct kvm_vcpu), vcpu_align: 0, THIS_MODULE);
6052	if (r) {
6053	__kvm_s390_exit();
6054	return r;
6055	}
6056	return 0;
6057	}
6058
6059	static void __exit kvm_s390_exit(void)
6060	{
6061	kvm_exit();
6062
6063	__kvm_s390_exit();
6064	}
6065
6066	module_init(kvm_s390_init);
6067	module_exit(kvm_s390_exit);
6068
6069	/*
6070	* Enable autoloading of the kvm module.
6071	* Note that we add the module alias here instead of virt/kvm/kvm_main.c
6072	* since x86 takes a different approach.
6073	*/
6074	#include <linux/miscdevice.h>
6075	MODULE_ALIAS_MISCDEV(KVM_MINOR);
6076	MODULE_ALIAS("devname:kvm");
6077