1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* handling privileged instructions
4	*
5	* Copyright IBM Corp. 2008, 2020
6	*
7	* Author(s): Carsten Otte <cotte@de.ibm.com>
8	* Christian Borntraeger <borntraeger@de.ibm.com>
9	*/
10
11	#include <linux/kvm.h>
12	#include <linux/gfp.h>
13	#include <linux/errno.h>
14	#include <linux/mm_types.h>
15	#include <linux/pgtable.h>
16	#include <linux/io.h>
17	#include <asm/asm-offsets.h>
18	#include <asm/facility.h>
19	#include <asm/current.h>
20	#include <asm/debug.h>
21	#include <asm/ebcdic.h>
22	#include <asm/sysinfo.h>
23	#include <asm/page-states.h>
24	#include <asm/gmap.h>
25	#include <asm/ptrace.h>
26	#include <asm/sclp.h>
27	#include <asm/ap.h>
28	#include "gaccess.h"
29	#include "kvm-s390.h"
30	#include "trace.h"
31
32	static int handle_ri(struct kvm_vcpu *vcpu)
33	{
34	vcpu->stat.instruction_ri++;
35
36	if (test_kvm_facility(kvm: vcpu->kvm, nr: 64)) {
37	VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (lazy)");
38	vcpu->arch.sie_block->ecb3 |= ECB3_RI;
39	kvm_s390_retry_instr(vcpu);
40	return 0;
41	} else
42	return kvm_s390_inject_program_int(vcpu, code: PGM_OPERATION);
43	}
44
45	int kvm_s390_handle_aa(struct kvm_vcpu *vcpu)
46	{
47	if ((vcpu->arch.sie_block->ipa & 0xf) <= 4)
48	return handle_ri(vcpu);
49	else
50	return -EOPNOTSUPP;
51	}
52
53	static int handle_gs(struct kvm_vcpu *vcpu)
54	{
55	vcpu->stat.instruction_gs++;
56
57	if (test_kvm_facility(kvm: vcpu->kvm, nr: 133)) {
58	VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (lazy)");
59	preempt_disable();
60	local_ctl_set_bit(2, CR2_GUARDED_STORAGE_BIT);
61	current->thread.gs_cb = (struct gs_cb *)&vcpu->run->s.regs.gscb;
62	restore_gs_cb(current->thread.gs_cb);
63	preempt_enable();
64	vcpu->arch.sie_block->ecb |= ECB_GS;
65	vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
66	vcpu->arch.gs_enabled = 1;
67	kvm_s390_retry_instr(vcpu);
68	return 0;
69	} else
70	return kvm_s390_inject_program_int(vcpu, code: PGM_OPERATION);
71	}
72
73	int kvm_s390_handle_e3(struct kvm_vcpu *vcpu)
74	{
75	int code = vcpu->arch.sie_block->ipb & 0xff;
76
77	if (code == 0x49 || code == 0x4d)
78	return handle_gs(vcpu);
79	else
80	return -EOPNOTSUPP;
81	}
82	/* Handle SCK (SET CLOCK) interception */
83	static int handle_set_clock(struct kvm_vcpu *vcpu)
84	{
85	struct kvm_s390_vm_tod_clock gtod = { 0 };
86	int rc;
87	u8 ar;
88	u64 op2;
89
90	vcpu->stat.instruction_sck++;
91
92	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
93	return kvm_s390_inject_program_int(vcpu, code: PGM_PRIVILEGED_OP);
94
95	op2 = kvm_s390_get_base_disp_s(vcpu, ar: &ar);
96	if (op2 & 7) /* Operand must be on a doubleword boundary */
97	return kvm_s390_inject_program_int(vcpu, code: PGM_SPECIFICATION);
98	rc = read_guest(vcpu, ga: op2, ar, data: &gtod.tod, len: sizeof(gtod.tod));
99	if (rc)
100	return kvm_s390_inject_prog_cond(vcpu, rc);
101
102	VCPU_EVENT(vcpu, 3, "SCK: setting guest TOD to 0x%llx", gtod.tod);
103	/*
104	* To set the TOD clock the kvm lock must be taken, but the vcpu lock
105	* is already held in handle_set_clock. The usual lock order is the
106	* opposite. As SCK is deprecated and should not be used in several
107	* cases, for example when the multiple epoch facility or TOD clock
108	* steering facility is installed (see Principles of Operation), a
109	* slow path can be used. If the lock can not be taken via try_lock,
110	* the instruction will be retried via -EAGAIN at a later point in
111	* time.
112	*/
113	if (!kvm_s390_try_set_tod_clock(kvm: vcpu->kvm, gtod: &gtod)) {
114	kvm_s390_retry_instr(vcpu);
115	return -EAGAIN;
116	}
117
118	kvm_s390_set_psw_cc(vcpu, cc: 0);
119	return 0;
120	}
121
122	static int handle_set_prefix(struct kvm_vcpu *vcpu)
123	{
124	u64 operand2;
125	u32 address;
126	int rc;
127	u8 ar;
128
129	vcpu->stat.instruction_spx++;
130
131	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
132	return kvm_s390_inject_program_int(vcpu, code: PGM_PRIVILEGED_OP);
133
134	operand2 = kvm_s390_get_base_disp_s(vcpu, ar: &ar);
135
136	/* must be word boundary */
137	if (operand2 & 3)
138	return kvm_s390_inject_program_int(vcpu, code: PGM_SPECIFICATION);
139
140	/* get the value */
141	rc = read_guest(vcpu, ga: operand2, ar, data: &address, len: sizeof(address));
142	if (rc)
143	return kvm_s390_inject_prog_cond(vcpu, rc);
144
145	address &= 0x7fffe000u;
146
147	/*
148	* Make sure the new value is valid memory. We only need to check the
149	* first page, since address is 8k aligned and memory pieces are always
150	* at least 1MB aligned and have at least a size of 1MB.
151	*/
152	if (!kvm_is_gpa_in_memslot(vcpu->kvm, address))
153	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
154
155	kvm_s390_set_prefix(vcpu, prefix: address);
156	trace_kvm_s390_handle_prefix(vcpu, set: 1, address);
157	return 0;
158	}
159
160	static int handle_store_prefix(struct kvm_vcpu *vcpu)
161	{
162	u64 operand2;
163	u32 address;
164	int rc;
165	u8 ar;
166
167	vcpu->stat.instruction_stpx++;
168
169	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
170	return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
171
172	operand2 = kvm_s390_get_base_disp_s(vcpu, ar: &ar);
173
174	/* must be word boundary */
175	if (operand2 & 3)
176	return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
177
178	address = kvm_s390_get_prefix(vcpu);
179
180	/* get the value */
181	rc = write_guest(vcpu, ga: operand2, ar, data: &address, len: sizeof(address));
182	if (rc)
183	return kvm_s390_inject_prog_cond(vcpu, rc);
184
185	VCPU_EVENT(vcpu, 3, "STPX: storing prefix 0x%x into 0x%llx", address, operand2);
186	trace_kvm_s390_handle_prefix(vcpu, set: 0, address);
187	return 0;
188	}
189
190	static int handle_store_cpu_address(struct kvm_vcpu *vcpu)
191	{
192	u16 vcpu_id = vcpu->vcpu_id;
193	u64 ga;
194	int rc;
195	u8 ar;
196
197	vcpu->stat.instruction_stap++;
198
199	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
200	return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
201
202	ga = kvm_s390_get_base_disp_s(vcpu, ar: &ar);
203
204	if (ga & 1)
205	return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
206
207	rc = write_guest(vcpu, ga, ar, data: &vcpu_id, len: sizeof(vcpu_id));
208	if (rc)
209	return kvm_s390_inject_prog_cond(vcpu, rc);
210
211	VCPU_EVENT(vcpu, 3, "STAP: storing cpu address (%u) to 0x%llx", vcpu_id, ga);
212	trace_kvm_s390_handle_stap(vcpu, address: ga);
213	return 0;
214	}
215
216	int kvm_s390_skey_check_enable(struct kvm_vcpu *vcpu)
217	{
218	int rc;
219
220	trace_kvm_s390_skey_related_inst(vcpu);
221	/* Already enabled? */
222	if (vcpu->arch.skey_enabled)
223	return 0;
224
225	rc = s390_enable_skey();
226	VCPU_EVENT(vcpu, 3, "enabling storage keys for guest: %d", rc);
227	if (rc)
228	return rc;
229
230	if (kvm_s390_test_cpuflags(vcpu, CPUSTAT_KSS))
231	kvm_s390_clear_cpuflags(vcpu, CPUSTAT_KSS);
232	if (!vcpu->kvm->arch.use_skf)
233	vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
234	else
235	vcpu->arch.sie_block->ictl &= ~(ICTL_ISKE | ICTL_SSKE | ICTL_RRBE);
236	vcpu->arch.skey_enabled = true;
237	return 0;
238	}
239
240	static int try_handle_skey(struct kvm_vcpu *vcpu)
241	{
242	int rc;
243
244	rc = kvm_s390_skey_check_enable(vcpu);
245	if (rc)
246	return rc;
247	if (vcpu->kvm->arch.use_skf) {
248	/* with storage-key facility, SIE interprets it for us */
249	kvm_s390_retry_instr(vcpu);
250	VCPU_EVENT(vcpu, 4, "%s", "retrying storage key operation");
251	return -EAGAIN;
252	}
253	return 0;
254	}
255
256	static int handle_iske(struct kvm_vcpu *vcpu)
257	{
258	unsigned long gaddr, vmaddr;
259	unsigned char key;
260	int reg1, reg2;
261	bool unlocked;
262	int rc;
263
264	vcpu->stat.instruction_iske++;
265
266	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
267	return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
268
269	rc = try_handle_skey(vcpu);
270	if (rc)
271	return rc != -EAGAIN ? rc : 0;
272
273	kvm_s390_get_regs_rre(vcpu, r1: &reg1, r2: &reg2);
274
275	gaddr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK;
276	gaddr = kvm_s390_logical_to_effective(vcpu, ga: gaddr);
277	gaddr = kvm_s390_real_to_abs(vcpu, gra: gaddr);
278	vmaddr = gfn_to_hva(kvm: vcpu->kvm, gfn: gpa_to_gfn(gpa: gaddr));
279	if (kvm_is_error_hva(vmaddr))
280	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
281	retry:
282	unlocked = false;
283	mmap_read_lock(current->mm);
284	rc = get_guest_storage_key(current->mm, vmaddr, &key);
285
286	if (rc) {
287	rc = fixup_user_fault(current->mm, address: vmaddr,
288	fault_flags: FAULT_FLAG_WRITE, unlocked: &unlocked);
289	if (!rc) {
290	mmap_read_unlock(current->mm);
291	goto retry;
292	}
293	}
294	mmap_read_unlock(current->mm);
295	if (rc == -EFAULT)
296	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
297	if (rc < 0)
298	return rc;
299	vcpu->run->s.regs.gprs[reg1] &= ~0xff;
300	vcpu->run->s.regs.gprs[reg1] |= key;
301	return 0;
302	}
303
304	static int handle_rrbe(struct kvm_vcpu *vcpu)
305	{
306	unsigned long vmaddr, gaddr;
307	int reg1, reg2;
308	bool unlocked;
309	int rc;
310
311	vcpu->stat.instruction_rrbe++;
312
313	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
314	return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
315
316	rc = try_handle_skey(vcpu);
317	if (rc)
318	return rc != -EAGAIN ? rc : 0;
319
320	kvm_s390_get_regs_rre(vcpu, r1: &reg1, r2: &reg2);
321
322	gaddr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK;
323	gaddr = kvm_s390_logical_to_effective(vcpu, ga: gaddr);
324	gaddr = kvm_s390_real_to_abs(vcpu, gra: gaddr);
325	vmaddr = gfn_to_hva(kvm: vcpu->kvm, gfn: gpa_to_gfn(gpa: gaddr));
326	if (kvm_is_error_hva(vmaddr))
327	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
328	retry:
329	unlocked = false;
330	mmap_read_lock(current->mm);
331	rc = reset_guest_reference_bit(current->mm, vmaddr);
332	if (rc < 0) {
333	rc = fixup_user_fault(current->mm, address: vmaddr,
334	fault_flags: FAULT_FLAG_WRITE, unlocked: &unlocked);
335	if (!rc) {
336	mmap_read_unlock(current->mm);
337	goto retry;
338	}
339	}
340	mmap_read_unlock(current->mm);
341	if (rc == -EFAULT)
342	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
343	if (rc < 0)
344	return rc;
345	kvm_s390_set_psw_cc(vcpu, cc: rc);
346	return 0;
347	}
348
349	#define SSKE_NQ 0x8
350	#define SSKE_MR 0x4
351	#define SSKE_MC 0x2
352	#define SSKE_MB 0x1
353	static int handle_sske(struct kvm_vcpu *vcpu)
354	{
355	unsigned char m3 = vcpu->arch.sie_block->ipb >> 28;
356	unsigned long start, end;
357	unsigned char key, oldkey;
358	int reg1, reg2;
359	bool unlocked;
360	int rc;
361
362	vcpu->stat.instruction_sske++;
363
364	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
365	return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
366
367	rc = try_handle_skey(vcpu);
368	if (rc)
369	return rc != -EAGAIN ? rc : 0;
370
371	if (!test_kvm_facility(kvm: vcpu->kvm, nr: 8))
372	m3 &= ~SSKE_MB;
373	if (!test_kvm_facility(kvm: vcpu->kvm, nr: 10))
374	m3 &= ~(SSKE_MC | SSKE_MR);
375	if (!test_kvm_facility(kvm: vcpu->kvm, nr: 14))
376	m3 &= ~SSKE_NQ;
377
378	kvm_s390_get_regs_rre(vcpu, r1: &reg1, r2: &reg2);
379
380	key = vcpu->run->s.regs.gprs[reg1] & 0xfe;
381	start = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK;
382	start = kvm_s390_logical_to_effective(vcpu, ga: start);
383	if (m3 & SSKE_MB) {
384	/* start already designates an absolute address */
385	end = (start + _SEGMENT_SIZE) & ~(_SEGMENT_SIZE - 1);
386	} else {
387	start = kvm_s390_real_to_abs(vcpu, gra: start);
388	end = start + PAGE_SIZE;
389	}
390
391	while (start != end) {
392	unsigned long vmaddr = gfn_to_hva(kvm: vcpu->kvm, gfn: gpa_to_gfn(gpa: start));
393	unlocked = false;
394
395	if (kvm_is_error_hva(vmaddr))
396	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
397
398	mmap_read_lock(current->mm);
399	rc = cond_set_guest_storage_key(current->mm, vmaddr, key, &oldkey,
400	m3 & SSKE_NQ, m3 & SSKE_MR,
401	m3 & SSKE_MC);
402
403	if (rc < 0) {
404	rc = fixup_user_fault(current->mm, address: vmaddr,
405	fault_flags: FAULT_FLAG_WRITE, unlocked: &unlocked);
406	rc = !rc ? -EAGAIN : rc;
407	}
408	mmap_read_unlock(current->mm);
409	if (rc == -EFAULT)
410	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
411	if (rc == -EAGAIN)
412	continue;
413	if (rc < 0)
414	return rc;
415	start += PAGE_SIZE;
416	}
417
418	if (m3 & (SSKE_MC | SSKE_MR)) {
419	if (m3 & SSKE_MB) {
420	/* skey in reg1 is unpredictable */
421	kvm_s390_set_psw_cc(vcpu, cc: 3);
422	} else {
423	kvm_s390_set_psw_cc(vcpu, cc: rc);
424	vcpu->run->s.regs.gprs[reg1] &= ~0xff00UL;
425	vcpu->run->s.regs.gprs[reg1] |= (u64) oldkey << 8;
426	}
427	}
428	if (m3 & SSKE_MB) {
429	if (psw_bits(vcpu->arch.sie_block->gpsw).eaba == PSW_BITS_AMODE_64BIT)
430	vcpu->run->s.regs.gprs[reg2] &= ~PAGE_MASK;
431	else
432	vcpu->run->s.regs.gprs[reg2] &= ~0xfffff000UL;
433	end = kvm_s390_logical_to_effective(vcpu, ga: end);
434	vcpu->run->s.regs.gprs[reg2] |= end;
435	}
436	return 0;
437	}
438
439	static int handle_ipte_interlock(struct kvm_vcpu *vcpu)
440	{
441	vcpu->stat.instruction_ipte_interlock++;
442	if (psw_bits(vcpu->arch.sie_block->gpsw).pstate)
443	return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
444	wait_event(vcpu->kvm->arch.ipte_wq, !ipte_lock_held(vcpu->kvm));
445	kvm_s390_retry_instr(vcpu);
446	VCPU_EVENT(vcpu, 4, "%s", "retrying ipte interlock operation");
447	return 0;
448	}
449
450	static int handle_test_block(struct kvm_vcpu *vcpu)
451	{
452	gpa_t addr;
453	int reg2;
454
455	vcpu->stat.instruction_tb++;
456
457	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
458	return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
459
460	kvm_s390_get_regs_rre(vcpu, NULL, r2: &reg2);
461	addr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK;
462	addr = kvm_s390_logical_to_effective(vcpu, ga: addr);
463	if (kvm_s390_check_low_addr_prot_real(vcpu, gra: addr))
464	return kvm_s390_inject_prog_irq(vcpu, pgm_info: &vcpu->arch.pgm);
465	addr = kvm_s390_real_to_abs(vcpu, gra: addr);
466
467	if (!kvm_is_gpa_in_memslot(vcpu->kvm, addr))
468	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
469	/*
470	* We don't expect errors on modern systems, and do not care
471	* about storage keys (yet), so let's just clear the page.
472	*/
473	if (kvm_clear_guest(kvm: vcpu->kvm, gpa: addr, PAGE_SIZE))
474	return -EFAULT;
475	kvm_s390_set_psw_cc(vcpu, cc: 0);
476	vcpu->run->s.regs.gprs[0] = 0;
477	return 0;
478	}
479
480	static int handle_tpi(struct kvm_vcpu *vcpu)
481	{
482	struct kvm_s390_interrupt_info *inti;
483	unsigned long len;
484	u32 tpi_data[3];
485	int rc;
486	u64 addr;
487	u8 ar;
488
489	vcpu->stat.instruction_tpi++;
490
491	addr = kvm_s390_get_base_disp_s(vcpu, ar: &ar);
492	if (addr & 3)
493	return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
494
495	inti = kvm_s390_get_io_int(kvm: vcpu->kvm, isc_mask: vcpu->arch.sie_block->gcr[6], schid: 0);
496	if (!inti) {
497	kvm_s390_set_psw_cc(vcpu, cc: 0);
498	return 0;
499	}
500
501	tpi_data[0] = inti->io.subchannel_id << 16 | inti->io.subchannel_nr;
502	tpi_data[1] = inti->io.io_int_parm;
503	tpi_data[2] = inti->io.io_int_word;
504	if (addr) {
505	/*
506	* Store the two-word I/O interruption code into the
507	* provided area.
508	*/
509	len = sizeof(tpi_data) - 4;
510	rc = write_guest(vcpu, ga: addr, ar, data: &tpi_data, len);
511	if (rc) {
512	rc = kvm_s390_inject_prog_cond(vcpu, rc);
513	goto reinject_interrupt;
514	}
515	} else {
516	/*
517	* Store the three-word I/O interruption code into
518	* the appropriate lowcore area.
519	*/
520	len = sizeof(tpi_data);
521	if (write_guest_lc(vcpu, __LC_SUBCHANNEL_ID, &tpi_data, len)) {
522	/* failed writes to the low core are not recoverable */
523	rc = -EFAULT;
524	goto reinject_interrupt;
525	}
526	}
527
528	/* irq was successfully handed to the guest */
529	kfree(objp: inti);
530	kvm_s390_set_psw_cc(vcpu, cc: 1);
531	return 0;
532	reinject_interrupt:
533	/*
534	* If we encounter a problem storing the interruption code, the
535	* instruction is suppressed from the guest's view: reinject the
536	* interrupt.
537	*/
538	if (kvm_s390_reinject_io_int(kvm: vcpu->kvm, inti)) {
539	kfree(objp: inti);
540	rc = -EFAULT;
541	}
542	/* don't set the cc, a pgm irq was injected or we drop to user space */
543	return rc ? -EFAULT : 0;
544	}
545
546	static int handle_tsch(struct kvm_vcpu *vcpu)
547	{
548	struct kvm_s390_interrupt_info *inti = NULL;
549	const u64 isc_mask = 0xffUL << 24; /* all iscs set */
550
551	vcpu->stat.instruction_tsch++;
552
553	/* a valid schid has at least one bit set */
554	if (vcpu->run->s.regs.gprs[1])
555	inti = kvm_s390_get_io_int(kvm: vcpu->kvm, isc_mask,
556	schid: vcpu->run->s.regs.gprs[1]);
557
558	/*
559	* Prepare exit to userspace.
560	* We indicate whether we dequeued a pending I/O interrupt
561	* so that userspace can re-inject it if the instruction gets
562	* a program check. While this may re-order the pending I/O
563	* interrupts, this is no problem since the priority is kept
564	* intact.
565	*/
566	vcpu->run->exit_reason = KVM_EXIT_S390_TSCH;
567	vcpu->run->s390_tsch.dequeued = !!inti;
568	if (inti) {
569	vcpu->run->s390_tsch.subchannel_id = inti->io.subchannel_id;
570	vcpu->run->s390_tsch.subchannel_nr = inti->io.subchannel_nr;
571	vcpu->run->s390_tsch.io_int_parm = inti->io.io_int_parm;
572	vcpu->run->s390_tsch.io_int_word = inti->io.io_int_word;
573	}
574	vcpu->run->s390_tsch.ipb = vcpu->arch.sie_block->ipb;
575	kfree(objp: inti);
576	return -EREMOTE;
577	}
578
579	static int handle_io_inst(struct kvm_vcpu *vcpu)
580	{
581	VCPU_EVENT(vcpu, 4, "%s", "I/O instruction");
582
583	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
584	return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
585
586	if (vcpu->kvm->arch.css_support) {
587	/*
588	* Most I/O instructions will be handled by userspace.
589	* Exceptions are tpi and the interrupt portion of tsch.
590	*/
591	if (vcpu->arch.sie_block->ipa == 0xb236)
592	return handle_tpi(vcpu);
593	if (vcpu->arch.sie_block->ipa == 0xb235)
594	return handle_tsch(vcpu);
595	/* Handle in userspace. */
596	vcpu->stat.instruction_io_other++;
597	return -EOPNOTSUPP;
598	} else {
599	/*
600	* Set condition code 3 to stop the guest from issuing channel
601	* I/O instructions.
602	*/
603	kvm_s390_set_psw_cc(vcpu, cc: 3);
604	return 0;
605	}
606	}
607
608	/*
609	* handle_pqap: Handling pqap interception
610	* @vcpu: the vcpu having issue the pqap instruction
611	*
612	* We now support PQAP/AQIC instructions and we need to correctly
613	* answer the guest even if no dedicated driver's hook is available.
614	*
615	* The intercepting code calls a dedicated callback for this instruction
616	* if a driver did register one in the CRYPTO satellite of the
617	* SIE block.
618	*
619	* If no callback is available, the queues are not available, return this
620	* response code to the caller and set CC to 3.
621	* Else return the response code returned by the callback.
622	*/
623	static int handle_pqap(struct kvm_vcpu *vcpu)
624	{
625	struct ap_queue_status status = {};
626	crypto_hook pqap_hook;
627	unsigned long reg0;
628	int ret;
629	uint8_t fc;
630
631	/* Verify that the AP instruction are available */
632	if (!ap_instructions_available())
633	return -EOPNOTSUPP;
634	/* Verify that the guest is allowed to use AP instructions */
635	if (!(vcpu->arch.sie_block->eca & ECA_APIE))
636	return -EOPNOTSUPP;
637	/*
638	* The only possibly intercepted functions when AP instructions are
639	* available for the guest are AQIC and TAPQ with the t bit set
640	* since we do not set IC.3 (FIII) we currently will only intercept
641	* the AQIC function code.
642	* Note: running nested under z/VM can result in intercepts for other
643	* function codes, e.g. PQAP(QCI). We do not support this and bail out.
644	*/
645	reg0 = vcpu->run->s.regs.gprs[0];
646	fc = (reg0 >> 24) & 0xff;
647	if (fc != 0x03)
648	return -EOPNOTSUPP;
649
650	/* PQAP instruction is allowed for guest kernel only */
651	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
652	return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
653
654	/* Common PQAP instruction specification exceptions */
655	/* bits 41-47 must all be zeros */
656	if (reg0 & 0x007f0000UL)
657	return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
658	/* APFT not install and T bit set */
659	if (!test_kvm_facility(vcpu->kvm, 15) && (reg0 & 0x00800000UL))
660	return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
661	/* APXA not installed and APID greater 64 or APQI greater 16 */
662	if (!(vcpu->kvm->arch.crypto.crycbd & 0x02) && (reg0 & 0x0000c0f0UL))
663	return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
664
665	/* AQIC function code specific exception */
666	/* facility 65 not present for AQIC function code */
667	if (!test_kvm_facility(vcpu->kvm, 65))
668	return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
669
670	/*
671	* If the hook callback is registered, there will be a pointer to the
672	* hook function pointer in the kvm_s390_crypto structure. Lock the
673	* owner, retrieve the hook function pointer and call the hook.
674	*/
675	down_read(sem: &vcpu->kvm->arch.crypto.pqap_hook_rwsem);
676	if (vcpu->kvm->arch.crypto.pqap_hook) {
677	pqap_hook = *vcpu->kvm->arch.crypto.pqap_hook;
678	ret = pqap_hook(vcpu);
679	if (!ret) {
680	if (vcpu->run->s.regs.gprs[1] & 0x00ff0000)
681	kvm_s390_set_psw_cc(vcpu, cc: 3);
682	else
683	kvm_s390_set_psw_cc(vcpu, cc: 0);
684	}
685	up_read(sem: &vcpu->kvm->arch.crypto.pqap_hook_rwsem);
686	return ret;
687	}
688	up_read(sem: &vcpu->kvm->arch.crypto.pqap_hook_rwsem);
689	/*
690	* A vfio_driver must register a hook.
691	* No hook means no driver to enable the SIE CRYCB and no queues.
692	* We send this response to the guest.
693	*/
694	status.response_code = 0x01;
695	memcpy(&vcpu->run->s.regs.gprs[1], &status, sizeof(status));
696	kvm_s390_set_psw_cc(vcpu, cc: 3);
697	return 0;
698	}
699
700	static int handle_stfl(struct kvm_vcpu *vcpu)
701	{
702	int rc;
703	unsigned int fac;
704
705	vcpu->stat.instruction_stfl++;
706
707	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
708	return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
709
710	/*
711	* We need to shift the lower 32 facility bits (bit 0-31) from a u64
712	* into a u32 memory representation. They will remain bits 0-31.
713	*/
714	fac = *vcpu->kvm->arch.model.fac_list >> 32;
715	rc = write_guest_lc(vcpu, offsetof(struct lowcore, stfl_fac_list),
716	&fac, sizeof(fac));
717	if (rc)
718	return rc;
719	VCPU_EVENT(vcpu, 3, "STFL: store facility list 0x%x", fac);
720	trace_kvm_s390_handle_stfl(vcpu, facility_list: fac);
721	return 0;
722	}
723
724	#define PSW_MASK_ADDR_MODE (PSW_MASK_EA | PSW_MASK_BA)
725	#define PSW_MASK_UNASSIGNED 0xb80800fe7fffffffUL
726	#define PSW_ADDR_24 0x0000000000ffffffUL
727	#define PSW_ADDR_31 0x000000007fffffffUL
728
729	int is_valid_psw(psw_t *psw)
730	{
731	if (psw->mask & PSW_MASK_UNASSIGNED)
732	return 0;
733	if ((psw->mask & PSW_MASK_ADDR_MODE) == PSW_MASK_BA) {
734	if (psw->addr & ~PSW_ADDR_31)
735	return 0;
736	}
737	if (!(psw->mask & PSW_MASK_ADDR_MODE) && (psw->addr & ~PSW_ADDR_24))
738	return 0;
739	if ((psw->mask & PSW_MASK_ADDR_MODE) == PSW_MASK_EA)
740	return 0;
741	if (psw->addr & 1)
742	return 0;
743	return 1;
744	}
745
746	int kvm_s390_handle_lpsw(struct kvm_vcpu *vcpu)
747	{
748	psw_t *gpsw = &vcpu->arch.sie_block->gpsw;
749	psw_compat_t new_psw;
750	u64 addr;
751	int rc;
752	u8 ar;
753
754	vcpu->stat.instruction_lpsw++;
755
756	if (gpsw->mask & PSW_MASK_PSTATE)
757	return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
758
759	addr = kvm_s390_get_base_disp_s(vcpu, ar: &ar);
760	if (addr & 7)
761	return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
762
763	rc = read_guest(vcpu, addr, ar, &new_psw, sizeof(new_psw));
764	if (rc)
765	return kvm_s390_inject_prog_cond(vcpu, rc);
766	if (!(new_psw.mask & PSW32_MASK_BASE))
767	return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
768	gpsw->mask = (new_psw.mask & ~PSW32_MASK_BASE) << 32;
769	gpsw->mask |= new_psw.addr & PSW32_ADDR_AMODE;
770	gpsw->addr = new_psw.addr & ~PSW32_ADDR_AMODE;
771	if (!is_valid_psw(gpsw))
772	return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
773	return 0;
774	}
775
776	static int handle_lpswe(struct kvm_vcpu *vcpu)
777	{
778	psw_t new_psw;
779	u64 addr;
780	int rc;
781	u8 ar;
782
783	vcpu->stat.instruction_lpswe++;
784
785	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
786	return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
787
788	addr = kvm_s390_get_base_disp_s(vcpu, ar: &ar);
789	if (addr & 7)
790	return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
791	rc = read_guest(vcpu, addr, ar, &new_psw, sizeof(new_psw));
792	if (rc)
793	return kvm_s390_inject_prog_cond(vcpu, rc);
794	vcpu->arch.sie_block->gpsw = new_psw;
795	if (!is_valid_psw(&vcpu->arch.sie_block->gpsw))
796	return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
797	return 0;
798	}
799
800	static int handle_stidp(struct kvm_vcpu *vcpu)
801	{
802	u64 stidp_data = vcpu->kvm->arch.model.cpuid;
803	u64 operand2;
804	int rc;
805	u8 ar;
806
807	vcpu->stat.instruction_stidp++;
808
809	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
810	return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
811
812	operand2 = kvm_s390_get_base_disp_s(vcpu, ar: &ar);
813
814	if (operand2 & 7)
815	return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
816
817	rc = write_guest(vcpu, ga: operand2, ar, data: &stidp_data, len: sizeof(stidp_data));
818	if (rc)
819	return kvm_s390_inject_prog_cond(vcpu, rc);
820
821	VCPU_EVENT(vcpu, 3, "STIDP: store cpu id 0x%llx", stidp_data);
822	return 0;
823	}
824
825	static void handle_stsi_3_2_2(struct kvm_vcpu *vcpu, struct sysinfo_3_2_2 *mem)
826	{
827	int cpus = 0;
828	int n;
829
830	cpus = atomic_read(v: &vcpu->kvm->online_vcpus);
831
832	/* deal with other level 3 hypervisors */
833	if (stsi(mem, 3, 2, 2))
834	mem->count = 0;
835	if (mem->count < 8)
836	mem->count++;
837	for (n = mem->count - 1; n > 0 ; n--)
838	memcpy(&mem->vm[n], &mem->vm[n - 1], sizeof(mem->vm[0]));
839
840	memset(&mem->vm[0], 0, sizeof(mem->vm[0]));
841	mem->vm[0].cpus_total = cpus;
842	mem->vm[0].cpus_configured = cpus;
843	mem->vm[0].cpus_standby = 0;
844	mem->vm[0].cpus_reserved = 0;
845	mem->vm[0].caf = 1000;
846	memcpy(mem->vm[0].name, "KVMguest", 8);
847	ASCEBC(mem->vm[0].name, 8);
848	memcpy(mem->vm[0].cpi, "KVM/Linux ", 16);
849	ASCEBC(mem->vm[0].cpi, 16);
850	}
851
852	static void insert_stsi_usr_data(struct kvm_vcpu *vcpu, u64 addr, u8 ar,
853	u8 fc, u8 sel1, u16 sel2)
854	{
855	vcpu->run->exit_reason = KVM_EXIT_S390_STSI;
856	vcpu->run->s390_stsi.addr = addr;
857	vcpu->run->s390_stsi.ar = ar;
858	vcpu->run->s390_stsi.fc = fc;
859	vcpu->run->s390_stsi.sel1 = sel1;
860	vcpu->run->s390_stsi.sel2 = sel2;
861	}
862
863	static int handle_stsi(struct kvm_vcpu *vcpu)
864	{
865	int fc = (vcpu->run->s.regs.gprs[0] & 0xf0000000) >> 28;
866	int sel1 = vcpu->run->s.regs.gprs[0] & 0xff;
867	int sel2 = vcpu->run->s.regs.gprs[1] & 0xffff;
868	unsigned long mem = 0;
869	u64 operand2;
870	int rc = 0;
871	u8 ar;
872
873	vcpu->stat.instruction_stsi++;
874	VCPU_EVENT(vcpu, 3, "STSI: fc: %u sel1: %u sel2: %u", fc, sel1, sel2);
875
876	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
877	return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
878
879	/* Bailout forbidden function codes */
880	if (fc > 3 && fc != 15)
881	goto out_no_data;
882
883	/*
884	* fc 15 is provided only with
885	* - PTF/CPU topology support through facility 15
886	* - KVM_CAP_S390_USER_STSI
887	*/
888	if (fc == 15 && (!test_kvm_facility(kvm: vcpu->kvm, nr: 11) ||
889	!vcpu->kvm->arch.user_stsi))
890	goto out_no_data;
891
892	if (vcpu->run->s.regs.gprs[0] & 0x0fffff00
893	|| vcpu->run->s.regs.gprs[1] & 0xffff0000)
894	return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
895
896	if (fc == 0) {
897	vcpu->run->s.regs.gprs[0] = 3 << 28;
898	kvm_s390_set_psw_cc(vcpu, cc: 0);
899	return 0;
900	}
901
902	operand2 = kvm_s390_get_base_disp_s(vcpu, ar: &ar);
903
904	if (!kvm_s390_pv_cpu_is_protected(vcpu) && (operand2 & 0xfff))
905	return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
906
907	switch (fc) {
908	case 1: /* same handling for 1 and 2 */
909	case 2:
910	mem = get_zeroed_page(GFP_KERNEL_ACCOUNT);
911	if (!mem)
912	goto out_no_data;
913	if (stsi((void *) mem, fc, sel1, sel2))
914	goto out_no_data;
915	break;
916	case 3:
917	if (sel1 != 2 || sel2 != 2)
918	goto out_no_data;
919	mem = get_zeroed_page(GFP_KERNEL_ACCOUNT);
920	if (!mem)
921	goto out_no_data;
922	handle_stsi_3_2_2(vcpu, mem: (void *) mem);
923	break;
924	case 15: /* fc 15 is fully handled in userspace */
925	insert_stsi_usr_data(vcpu, addr: operand2, ar, fc, sel1, sel2);
926	trace_kvm_s390_handle_stsi(vcpu, fc, sel1, sel2, addr: operand2);
927	return -EREMOTE;
928	}
929	if (kvm_s390_pv_cpu_is_protected(vcpu)) {
930	memcpy(sida_addr(vcpu->arch.sie_block), (void *)mem, PAGE_SIZE);
931	rc = 0;
932	} else {
933	rc = write_guest(vcpu, ga: operand2, ar, data: (void *)mem, PAGE_SIZE);
934	}
935	if (rc) {
936	rc = kvm_s390_inject_prog_cond(vcpu, rc);
937	goto out;
938	}
939	if (vcpu->kvm->arch.user_stsi) {
940	insert_stsi_usr_data(vcpu, addr: operand2, ar, fc, sel1, sel2);
941	rc = -EREMOTE;
942	}
943	trace_kvm_s390_handle_stsi(vcpu, fc, sel1, sel2, addr: operand2);
944	free_page(mem);
945	kvm_s390_set_psw_cc(vcpu, cc: 0);
946	vcpu->run->s.regs.gprs[0] = 0;
947	return rc;
948	out_no_data:
949	kvm_s390_set_psw_cc(vcpu, cc: 3);
950	out:
951	free_page(mem);
952	return rc;
953	}
954
955	int kvm_s390_handle_b2(struct kvm_vcpu *vcpu)
956	{
957	switch (vcpu->arch.sie_block->ipa & 0x00ff) {
958	case 0x02:
959	return handle_stidp(vcpu);
960	case 0x04:
961	return handle_set_clock(vcpu);
962	case 0x10:
963	return handle_set_prefix(vcpu);
964	case 0x11:
965	return handle_store_prefix(vcpu);
966	case 0x12:
967	return handle_store_cpu_address(vcpu);
968	case 0x14:
969	return kvm_s390_handle_vsie(vcpu);
970	case 0x21:
971	case 0x50:
972	return handle_ipte_interlock(vcpu);
973	case 0x29:
974	return handle_iske(vcpu);
975	case 0x2a:
976	return handle_rrbe(vcpu);
977	case 0x2b:
978	return handle_sske(vcpu);
979	case 0x2c:
980	return handle_test_block(vcpu);
981	case 0x30:
982	case 0x31:
983	case 0x32:
984	case 0x33:
985	case 0x34:
986	case 0x35:
987	case 0x36:
988	case 0x37:
989	case 0x38:
990	case 0x39:
991	case 0x3a:
992	case 0x3b:
993	case 0x3c:
994	case 0x5f:
995	case 0x74:
996	case 0x76:
997	return handle_io_inst(vcpu);
998	case 0x56:
999	return handle_sthyi(vcpu);
1000	case 0x7d:
1001	return handle_stsi(vcpu);
1002	case 0xaf:
1003	return handle_pqap(vcpu);
1004	case 0xb1:
1005	return handle_stfl(vcpu);
1006	case 0xb2:
1007	return handle_lpswe(vcpu);
1008	default:
1009	return -EOPNOTSUPP;
1010	}
1011	}
1012
1013	static int handle_epsw(struct kvm_vcpu *vcpu)
1014	{
1015	int reg1, reg2;
1016
1017	vcpu->stat.instruction_epsw++;
1018
1019	kvm_s390_get_regs_rre(vcpu, r1: &reg1, r2: &reg2);
1020
1021	/* This basically extracts the mask half of the psw. */
1022	vcpu->run->s.regs.gprs[reg1] &= 0xffffffff00000000UL;
1023	vcpu->run->s.regs.gprs[reg1] |= vcpu->arch.sie_block->gpsw.mask >> 32;
1024	if (reg2) {
1025	vcpu->run->s.regs.gprs[reg2] &= 0xffffffff00000000UL;
1026	vcpu->run->s.regs.gprs[reg2] |=
1027	vcpu->arch.sie_block->gpsw.mask & 0x00000000ffffffffUL;
1028	}
1029	return 0;
1030	}
1031
1032	#define PFMF_RESERVED 0xfffc0101UL
1033	#define PFMF_SK 0x00020000UL
1034	#define PFMF_CF 0x00010000UL
1035	#define PFMF_UI 0x00008000UL
1036	#define PFMF_FSC 0x00007000UL
1037	#define PFMF_NQ 0x00000800UL
1038	#define PFMF_MR 0x00000400UL
1039	#define PFMF_MC 0x00000200UL
1040	#define PFMF_KEY 0x000000feUL
1041
1042	static int handle_pfmf(struct kvm_vcpu *vcpu)
1043	{
1044	bool mr = false, mc = false, nq;
1045	int reg1, reg2;
1046	unsigned long start, end;
1047	unsigned char key;
1048
1049	vcpu->stat.instruction_pfmf++;
1050
1051	kvm_s390_get_regs_rre(vcpu, r1: &reg1, r2: &reg2);
1052
1053	if (!test_kvm_facility(vcpu->kvm, 8))
1054	return kvm_s390_inject_program_int(vcpu, PGM_OPERATION);
1055
1056	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
1057	return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
1058
1059	if (vcpu->run->s.regs.gprs[reg1] & PFMF_RESERVED)
1060	return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
1061
1062	/* Only provide non-quiescing support if enabled for the guest */
1063	if (vcpu->run->s.regs.gprs[reg1] & PFMF_NQ &&
1064	!test_kvm_facility(vcpu->kvm, 14))
1065	return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
1066
1067	/* Only provide conditional-SSKE support if enabled for the guest */
1068	if (vcpu->run->s.regs.gprs[reg1] & PFMF_SK &&
1069	test_kvm_facility(kvm: vcpu->kvm, nr: 10)) {
1070	mr = vcpu->run->s.regs.gprs[reg1] & PFMF_MR;
1071	mc = vcpu->run->s.regs.gprs[reg1] & PFMF_MC;
1072	}
1073
1074	nq = vcpu->run->s.regs.gprs[reg1] & PFMF_NQ;
1075	key = vcpu->run->s.regs.gprs[reg1] & PFMF_KEY;
1076	start = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK;
1077	start = kvm_s390_logical_to_effective(vcpu, ga: start);
1078
1079	if (vcpu->run->s.regs.gprs[reg1] & PFMF_CF) {
1080	if (kvm_s390_check_low_addr_prot_real(vcpu, gra: start))
1081	return kvm_s390_inject_prog_irq(vcpu, pgm_info: &vcpu->arch.pgm);
1082	}
1083
1084	switch (vcpu->run->s.regs.gprs[reg1] & PFMF_FSC) {
1085	case 0x00000000:
1086	/* only 4k frames specify a real address */
1087	start = kvm_s390_real_to_abs(vcpu, gra: start);
1088	end = (start + PAGE_SIZE) & ~(PAGE_SIZE - 1);
1089	break;
1090	case 0x00001000:
1091	end = (start + _SEGMENT_SIZE) & ~(_SEGMENT_SIZE - 1);
1092	break;
1093	case 0x00002000:
1094	/* only support 2G frame size if EDAT2 is available and we are
1095	not in 24-bit addressing mode */
1096	if (!test_kvm_facility(vcpu->kvm, 78) ||
1097	psw_bits(vcpu->arch.sie_block->gpsw).eaba == PSW_BITS_AMODE_24BIT)
1098	return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
1099	end = (start + _REGION3_SIZE) & ~(_REGION3_SIZE - 1);
1100	break;
1101	default:
1102	return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
1103	}
1104
1105	while (start != end) {
1106	unsigned long vmaddr;
1107	bool unlocked = false;
1108
1109	/* Translate guest address to host address */
1110	vmaddr = gfn_to_hva(kvm: vcpu->kvm, gfn: gpa_to_gfn(gpa: start));
1111	if (kvm_is_error_hva(vmaddr))
1112	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
1113
1114	if (vcpu->run->s.regs.gprs[reg1] & PFMF_CF) {
1115	if (kvm_clear_guest(vcpu->kvm, start, PAGE_SIZE))
1116	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
1117	}
1118
1119	if (vcpu->run->s.regs.gprs[reg1] & PFMF_SK) {
1120	int rc = kvm_s390_skey_check_enable(vcpu);
1121
1122	if (rc)
1123	return rc;
1124	mmap_read_lock(current->mm);
1125	rc = cond_set_guest_storage_key(current->mm, vmaddr,
1126	key, NULL, nq, mr, mc);
1127	if (rc < 0) {
1128	rc = fixup_user_fault(current->mm, address: vmaddr,
1129	fault_flags: FAULT_FLAG_WRITE, unlocked: &unlocked);
1130	rc = !rc ? -EAGAIN : rc;
1131	}
1132	mmap_read_unlock(current->mm);
1133	if (rc == -EFAULT)
1134	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
1135	if (rc == -EAGAIN)
1136	continue;
1137	if (rc < 0)
1138	return rc;
1139	}
1140	start += PAGE_SIZE;
1141	}
1142	if (vcpu->run->s.regs.gprs[reg1] & PFMF_FSC) {
1143	if (psw_bits(vcpu->arch.sie_block->gpsw).eaba == PSW_BITS_AMODE_64BIT) {
1144	vcpu->run->s.regs.gprs[reg2] = end;
1145	} else {
1146	vcpu->run->s.regs.gprs[reg2] &= ~0xffffffffUL;
1147	end = kvm_s390_logical_to_effective(vcpu, ga: end);
1148	vcpu->run->s.regs.gprs[reg2] |= end;
1149	}
1150	}
1151	return 0;
1152	}
1153
1154	/*
1155	* Must be called with relevant read locks held (kvm->mm->mmap_lock, kvm->srcu)
1156	*/
1157	static inline int __do_essa(struct kvm_vcpu *vcpu, const int orc)
1158	{
1159	int r1, r2, nappended, entries;
1160	unsigned long gfn, hva, res, pgstev, ptev;
1161	unsigned long *cbrlo;
1162
1163	/*
1164	* We don't need to set SD.FPF.SK to 1 here, because if we have a
1165	* machine check here we either handle it or crash
1166	*/
1167
1168	kvm_s390_get_regs_rre(vcpu, r1: &r1, r2: &r2);
1169	gfn = vcpu->run->s.regs.gprs[r2] >> PAGE_SHIFT;
1170	hva = gfn_to_hva(kvm: vcpu->kvm, gfn);
1171	entries = (vcpu->arch.sie_block->cbrlo & ~PAGE_MASK) >> 3;
1172
1173	if (kvm_is_error_hva(hva))
1174	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
1175
1176	nappended = pgste_perform_essa(vcpu->kvm->mm, hva, orc, &ptev, &pgstev);
1177	if (nappended < 0) {
1178	res = orc ? 0x10 : 0;
1179	vcpu->run->s.regs.gprs[r1] = res; /* Exception Indication */
1180	return 0;
1181	}
1182	res = (pgstev & _PGSTE_GPS_USAGE_MASK) >> 22;
1183	/*
1184	* Set the block-content state part of the result. 0 means resident, so
1185	* nothing to do if the page is valid. 2 is for preserved pages
1186	* (non-present and non-zero), and 3 for zero pages (non-present and
1187	* zero).
1188	*/
1189	if (ptev & _PAGE_INVALID) {
1190	res |= 2;
1191	if (pgstev & _PGSTE_GPS_ZERO)
1192	res |= 1;
1193	}
1194	if (pgstev & _PGSTE_GPS_NODAT)
1195	res |= 0x20;
1196	vcpu->run->s.regs.gprs[r1] = res;
1197	/*
1198	* It is possible that all the normal 511 slots were full, in which case
1199	* we will now write in the 512th slot, which is reserved for host use.
1200	* In both cases we let the normal essa handling code process all the
1201	* slots, including the reserved one, if needed.
1202	*/
1203	if (nappended > 0) {
1204	cbrlo = phys_to_virt(address: vcpu->arch.sie_block->cbrlo & PAGE_MASK);
1205	cbrlo[entries] = gfn << PAGE_SHIFT;
1206	}
1207
1208	if (orc) {
1209	struct kvm_memory_slot *ms = gfn_to_memslot(kvm: vcpu->kvm, gfn);
1210
1211	/* Increment only if we are really flipping the bit */
1212	if (ms && !test_and_set_bit(nr: gfn - ms->base_gfn, addr: kvm_second_dirty_bitmap(memslot: ms)))
1213	atomic64_inc(v: &vcpu->kvm->arch.cmma_dirty_pages);
1214	}
1215
1216	return nappended;
1217	}
1218
1219	static int handle_essa(struct kvm_vcpu *vcpu)
1220	{
1221	/* entries expected to be 1FF */
1222	int entries = (vcpu->arch.sie_block->cbrlo & ~PAGE_MASK) >> 3;
1223	unsigned long *cbrlo;
1224	struct gmap *gmap;
1225	int i, orc;
1226
1227	VCPU_EVENT(vcpu, 4, "ESSA: release %d pages", entries);
1228	gmap = vcpu->arch.gmap;
1229	vcpu->stat.instruction_essa++;
1230	if (!vcpu->kvm->arch.use_cmma)
1231	return kvm_s390_inject_program_int(vcpu, PGM_OPERATION);
1232
1233	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
1234	return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
1235	/* Check for invalid operation request code */
1236	orc = (vcpu->arch.sie_block->ipb & 0xf0000000) >> 28;
1237	/* ORCs 0-6 are always valid */
1238	if (orc > (test_kvm_facility(vcpu->kvm, 147) ? ESSA_SET_STABLE_NODAT
1239	: ESSA_SET_STABLE_IF_RESIDENT))
1240	return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
1241
1242	if (!vcpu->kvm->arch.migration_mode) {
1243	/*
1244	* CMMA is enabled in the KVM settings, but is disabled in
1245	* the SIE block and in the mm_context, and we are not doing
1246	* a migration. Enable CMMA in the mm_context.
1247	* Since we need to take a write lock to write to the context
1248	* to avoid races with storage keys handling, we check if the
1249	* value really needs to be written to; if the value is
1250	* already correct, we do nothing and avoid the lock.
1251	*/
1252	if (vcpu->kvm->mm->context.uses_cmm == 0) {
1253	mmap_write_lock(mm: vcpu->kvm->mm);
1254	vcpu->kvm->mm->context.uses_cmm = 1;
1255	mmap_write_unlock(mm: vcpu->kvm->mm);
1256	}
1257	/*
1258	* If we are here, we are supposed to have CMMA enabled in
1259	* the SIE block. Enabling CMMA works on a per-CPU basis,
1260	* while the context use_cmma flag is per process.
1261	* It's possible that the context flag is enabled and the
1262	* SIE flag is not, so we set the flag always; if it was
1263	* already set, nothing changes, otherwise we enable it
1264	* on this CPU too.
1265	*/
1266	vcpu->arch.sie_block->ecb2 |= ECB2_CMMA;
1267	/* Retry the ESSA instruction */
1268	kvm_s390_retry_instr(vcpu);
1269	} else {
1270	int srcu_idx;
1271
1272	mmap_read_lock(mm: vcpu->kvm->mm);
1273	srcu_idx = srcu_read_lock(ssp: &vcpu->kvm->srcu);
1274	i = __do_essa(vcpu, orc);
1275	srcu_read_unlock(ssp: &vcpu->kvm->srcu, idx: srcu_idx);
1276	mmap_read_unlock(mm: vcpu->kvm->mm);
1277	if (i < 0)
1278	return i;
1279	/* Account for the possible extra cbrl entry */
1280	entries += i;
1281	}
1282	vcpu->arch.sie_block->cbrlo &= PAGE_MASK; /* reset nceo */
1283	cbrlo = phys_to_virt(address: vcpu->arch.sie_block->cbrlo);
1284	mmap_read_lock(mm: gmap->mm);
1285	for (i = 0; i < entries; ++i)
1286	__gmap_zap(gmap, cbrlo[i]);
1287	mmap_read_unlock(mm: gmap->mm);
1288	return 0;
1289	}
1290
1291	int kvm_s390_handle_b9(struct kvm_vcpu *vcpu)
1292	{
1293	switch (vcpu->arch.sie_block->ipa & 0x00ff) {
1294	case 0x8a:
1295	case 0x8e:
1296	case 0x8f:
1297	return handle_ipte_interlock(vcpu);
1298	case 0x8d:
1299	return handle_epsw(vcpu);
1300	case 0xab:
1301	return handle_essa(vcpu);
1302	case 0xaf:
1303	return handle_pfmf(vcpu);
1304	default:
1305	return -EOPNOTSUPP;
1306	}
1307	}
1308
1309	int kvm_s390_handle_lctl(struct kvm_vcpu *vcpu)
1310	{
1311	int reg1 = (vcpu->arch.sie_block->ipa & 0x00f0) >> 4;
1312	int reg3 = vcpu->arch.sie_block->ipa & 0x000f;
1313	int reg, rc, nr_regs;
1314	u32 ctl_array[16];
1315	u64 ga;
1316	u8 ar;
1317
1318	vcpu->stat.instruction_lctl++;
1319
1320	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
1321	return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
1322
1323	ga = kvm_s390_get_base_disp_rs(vcpu, ar: &ar);
1324
1325	if (ga & 3)
1326	return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
1327
1328	VCPU_EVENT(vcpu, 4, "LCTL: r1:%d, r3:%d, addr: 0x%llx", reg1, reg3, ga);
1329	trace_kvm_s390_handle_lctl(vcpu, g: 0, reg1, reg3, addr: ga);
1330
1331	nr_regs = ((reg3 - reg1) & 0xf) + 1;
1332	rc = read_guest(vcpu, ga, ar, data: ctl_array, len: nr_regs * sizeof(u32));
1333	if (rc)
1334	return kvm_s390_inject_prog_cond(vcpu, rc);
1335	reg = reg1;
1336	nr_regs = 0;
1337	do {
1338	vcpu->arch.sie_block->gcr[reg] &= 0xffffffff00000000ul;
1339	vcpu->arch.sie_block->gcr[reg] |= ctl_array[nr_regs++];
1340	if (reg == reg3)
1341	break;
1342	reg = (reg + 1) % 16;
1343	} while (1);
1344	kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
1345	return 0;
1346	}
1347
1348	int kvm_s390_handle_stctl(struct kvm_vcpu *vcpu)
1349	{
1350	int reg1 = (vcpu->arch.sie_block->ipa & 0x00f0) >> 4;
1351	int reg3 = vcpu->arch.sie_block->ipa & 0x000f;
1352	int reg, rc, nr_regs;
1353	u32 ctl_array[16];
1354	u64 ga;
1355	u8 ar;
1356
1357	vcpu->stat.instruction_stctl++;
1358
1359	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
1360	return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
1361
1362	ga = kvm_s390_get_base_disp_rs(vcpu, ar: &ar);
1363
1364	if (ga & 3)
1365	return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
1366
1367	VCPU_EVENT(vcpu, 4, "STCTL r1:%d, r3:%d, addr: 0x%llx", reg1, reg3, ga);
1368	trace_kvm_s390_handle_stctl(vcpu, g: 0, reg1, reg3, addr: ga);
1369
1370	reg = reg1;
1371	nr_regs = 0;
1372	do {
1373	ctl_array[nr_regs++] = vcpu->arch.sie_block->gcr[reg];
1374	if (reg == reg3)
1375	break;
1376	reg = (reg + 1) % 16;
1377	} while (1);
1378	rc = write_guest(vcpu, ga, ar, data: ctl_array, len: nr_regs * sizeof(u32));
1379	return rc ? kvm_s390_inject_prog_cond(vcpu, rc) : 0;
1380	}
1381
1382	static int handle_lctlg(struct kvm_vcpu *vcpu)
1383	{
1384	int reg1 = (vcpu->arch.sie_block->ipa & 0x00f0) >> 4;
1385	int reg3 = vcpu->arch.sie_block->ipa & 0x000f;
1386	int reg, rc, nr_regs;
1387	u64 ctl_array[16];
1388	u64 ga;
1389	u8 ar;
1390
1391	vcpu->stat.instruction_lctlg++;
1392
1393	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
1394	return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
1395
1396	ga = kvm_s390_get_base_disp_rsy(vcpu, ar: &ar);
1397
1398	if (ga & 7)
1399	return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
1400
1401	VCPU_EVENT(vcpu, 4, "LCTLG: r1:%d, r3:%d, addr: 0x%llx", reg1, reg3, ga);
1402	trace_kvm_s390_handle_lctl(vcpu, g: 1, reg1, reg3, addr: ga);
1403
1404	nr_regs = ((reg3 - reg1) & 0xf) + 1;
1405	rc = read_guest(vcpu, ga, ar, data: ctl_array, len: nr_regs * sizeof(u64));
1406	if (rc)
1407	return kvm_s390_inject_prog_cond(vcpu, rc);
1408	reg = reg1;
1409	nr_regs = 0;
1410	do {
1411	vcpu->arch.sie_block->gcr[reg] = ctl_array[nr_regs++];
1412	if (reg == reg3)
1413	break;
1414	reg = (reg + 1) % 16;
1415	} while (1);
1416	kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
1417	return 0;
1418	}
1419
1420	static int handle_stctg(struct kvm_vcpu *vcpu)
1421	{
1422	int reg1 = (vcpu->arch.sie_block->ipa & 0x00f0) >> 4;
1423	int reg3 = vcpu->arch.sie_block->ipa & 0x000f;
1424	int reg, rc, nr_regs;
1425	u64 ctl_array[16];
1426	u64 ga;
1427	u8 ar;
1428
1429	vcpu->stat.instruction_stctg++;
1430
1431	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
1432	return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
1433
1434	ga = kvm_s390_get_base_disp_rsy(vcpu, ar: &ar);
1435
1436	if (ga & 7)
1437	return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
1438
1439	VCPU_EVENT(vcpu, 4, "STCTG r1:%d, r3:%d, addr: 0x%llx", reg1, reg3, ga);
1440	trace_kvm_s390_handle_stctl(vcpu, g: 1, reg1, reg3, addr: ga);
1441
1442	reg = reg1;
1443	nr_regs = 0;
1444	do {
1445	ctl_array[nr_regs++] = vcpu->arch.sie_block->gcr[reg];
1446	if (reg == reg3)
1447	break;
1448	reg = (reg + 1) % 16;
1449	} while (1);
1450	rc = write_guest(vcpu, ga, ar, data: ctl_array, len: nr_regs * sizeof(u64));
1451	return rc ? kvm_s390_inject_prog_cond(vcpu, rc) : 0;
1452	}
1453
1454	int kvm_s390_handle_eb(struct kvm_vcpu *vcpu)
1455	{
1456	switch (vcpu->arch.sie_block->ipb & 0x000000ff) {
1457	case 0x25:
1458	return handle_stctg(vcpu);
1459	case 0x2f:
1460	return handle_lctlg(vcpu);
1461	case 0x60:
1462	case 0x61:
1463	case 0x62:
1464	return handle_ri(vcpu);
1465	default:
1466	return -EOPNOTSUPP;
1467	}
1468	}
1469
1470	static int handle_tprot(struct kvm_vcpu *vcpu)
1471	{
1472	u64 address, operand2;
1473	unsigned long gpa;
1474	u8 access_key;
1475	bool writable;
1476	int ret, cc;
1477	u8 ar;
1478
1479	vcpu->stat.instruction_tprot++;
1480
1481	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
1482	return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
1483
1484	kvm_s390_get_base_disp_sse(vcpu, address1: &address, address2: &operand2, ar_b1: &ar, NULL);
1485	access_key = (operand2 & 0xf0) >> 4;
1486
1487	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_DAT)
1488	ipte_lock(kvm: vcpu->kvm);
1489
1490	ret = guest_translate_address_with_key(vcpu, gva: address, ar, gpa: &gpa,
1491	mode: GACC_STORE, access_key);
1492	if (ret == 0) {
1493	gfn_to_hva_prot(kvm: vcpu->kvm, gfn: gpa_to_gfn(gpa), writable: &writable);
1494	} else if (ret == PGM_PROTECTION) {
1495	writable = false;
1496	/* Write protected? Try again with read-only... */
1497	ret = guest_translate_address_with_key(vcpu, gva: address, ar, gpa: &gpa,
1498	mode: GACC_FETCH, access_key);
1499	}
1500	if (ret >= 0) {
1501	cc = -1;
1502
1503	/* Fetching permitted; storing permitted */
1504	if (ret == 0 && writable)
1505	cc = 0;
1506	/* Fetching permitted; storing not permitted */
1507	else if (ret == 0 && !writable)
1508	cc = 1;
1509	/* Fetching not permitted; storing not permitted */
1510	else if (ret == PGM_PROTECTION)
1511	cc = 2;
1512	/* Translation not available */
1513	else if (ret != PGM_ADDRESSING && ret != PGM_TRANSLATION_SPEC)
1514	cc = 3;
1515
1516	if (cc != -1) {
1517	kvm_s390_set_psw_cc(vcpu, cc);
1518	ret = 0;
1519	} else {
1520	ret = kvm_s390_inject_program_int(vcpu, code: ret);
1521	}
1522	}
1523
1524	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_DAT)
1525	ipte_unlock(kvm: vcpu->kvm);
1526	return ret;
1527	}
1528
1529	int kvm_s390_handle_e5(struct kvm_vcpu *vcpu)
1530	{
1531	switch (vcpu->arch.sie_block->ipa & 0x00ff) {
1532	case 0x01:
1533	return handle_tprot(vcpu);
1534	default:
1535	return -EOPNOTSUPP;
1536	}
1537	}
1538
1539	static int handle_sckpf(struct kvm_vcpu *vcpu)
1540	{
1541	u32 value;
1542
1543	vcpu->stat.instruction_sckpf++;
1544
1545	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
1546	return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
1547
1548	if (vcpu->run->s.regs.gprs[0] & 0x00000000ffff0000)
1549	return kvm_s390_inject_program_int(vcpu,
1550	PGM_SPECIFICATION);
1551
1552	value = vcpu->run->s.regs.gprs[0] & 0x000000000000ffff;
1553	vcpu->arch.sie_block->todpr = value;
1554
1555	return 0;
1556	}
1557
1558	static int handle_ptff(struct kvm_vcpu *vcpu)
1559	{
1560	vcpu->stat.instruction_ptff++;
1561
1562	/* we don't emulate any control instructions yet */
1563	kvm_s390_set_psw_cc(vcpu, cc: 3);
1564	return 0;
1565	}
1566
1567	int kvm_s390_handle_01(struct kvm_vcpu *vcpu)
1568	{
1569	switch (vcpu->arch.sie_block->ipa & 0x00ff) {
1570	case 0x04:
1571	return handle_ptff(vcpu);
1572	case 0x07:
1573	return handle_sckpf(vcpu);
1574	default:
1575	return -EOPNOTSUPP;
1576	}
1577	}
1578