1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* handling kvm guest interrupts
4	*
5	* Copyright IBM Corp. 2008, 2020
6	*
7	* Author(s): Carsten Otte <cotte@de.ibm.com>
8	*/
9
10	#define KMSG_COMPONENT "kvm-s390"
11	#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
12
13	#include <linux/interrupt.h>
14	#include <linux/kvm_host.h>
15	#include <linux/hrtimer.h>
16	#include <linux/mmu_context.h>
17	#include <linux/nospec.h>
18	#include <linux/signal.h>
19	#include <linux/slab.h>
20	#include <linux/bitmap.h>
21	#include <linux/vmalloc.h>
22	#include <asm/access-regs.h>
23	#include <asm/asm-offsets.h>
24	#include <asm/dis.h>
25	#include <linux/uaccess.h>
26	#include <asm/sclp.h>
27	#include <asm/isc.h>
28	#include <asm/gmap.h>
29	#include <asm/nmi.h>
30	#include <asm/airq.h>
31	#include <asm/tpi.h>
32	#include "kvm-s390.h"
33	#include "gaccess.h"
34	#include "trace-s390.h"
35	#include "pci.h"
36
37	#define PFAULT_INIT 0x0600
38	#define PFAULT_DONE 0x0680
39	#define VIRTIO_PARAM 0x0d00
40
41	static struct kvm_s390_gib *gib;
42
43	/* handle external calls via sigp interpretation facility */
44	static int sca_ext_call_pending(struct kvm_vcpu *vcpu, int *src_id)
45	{
46	int c, scn;
47
48	if (!kvm_s390_test_cpuflags(vcpu, flags: CPUSTAT_ECALL_PEND))
49	return 0;
50
51	BUG_ON(!kvm_s390_use_sca_entries());
52	read_lock(&vcpu->kvm->arch.sca_lock);
53	if (vcpu->kvm->arch.use_esca) {
54	struct esca_block *sca = vcpu->kvm->arch.sca;
55	union esca_sigp_ctrl sigp_ctrl =
56	sca->cpu[vcpu->vcpu_id].sigp_ctrl;
57
58	c = sigp_ctrl.c;
59	scn = sigp_ctrl.scn;
60	} else {
61	struct bsca_block *sca = vcpu->kvm->arch.sca;
62	union bsca_sigp_ctrl sigp_ctrl =
63	sca->cpu[vcpu->vcpu_id].sigp_ctrl;
64
65	c = sigp_ctrl.c;
66	scn = sigp_ctrl.scn;
67	}
68	read_unlock(&vcpu->kvm->arch.sca_lock);
69
70	if (src_id)
71	*src_id = scn;
72
73	return c;
74	}
75
76	static int sca_inject_ext_call(struct kvm_vcpu *vcpu, int src_id)
77	{
78	int expect, rc;
79
80	BUG_ON(!kvm_s390_use_sca_entries());
81	read_lock(&vcpu->kvm->arch.sca_lock);
82	if (vcpu->kvm->arch.use_esca) {
83	struct esca_block *sca = vcpu->kvm->arch.sca;
84	union esca_sigp_ctrl *sigp_ctrl =
85	&(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
86	union esca_sigp_ctrl new_val = {0}, old_val;
87
88	old_val = READ_ONCE(*sigp_ctrl);
89	new_val.scn = src_id;
90	new_val.c = 1;
91	old_val.c = 0;
92
93	expect = old_val.value;
94	rc = cmpxchg(&sigp_ctrl->value, old_val.value, new_val.value);
95	} else {
96	struct bsca_block *sca = vcpu->kvm->arch.sca;
97	union bsca_sigp_ctrl *sigp_ctrl =
98	&(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
99	union bsca_sigp_ctrl new_val = {0}, old_val;
100
101	old_val = READ_ONCE(*sigp_ctrl);
102	new_val.scn = src_id;
103	new_val.c = 1;
104	old_val.c = 0;
105
106	expect = old_val.value;
107	rc = cmpxchg(&sigp_ctrl->value, old_val.value, new_val.value);
108	}
109	read_unlock(&vcpu->kvm->arch.sca_lock);
110
111	if (rc != expect) {
112	/* another external call is pending */
113	return -EBUSY;
114	}
115	kvm_s390_set_cpuflags(vcpu, flags: CPUSTAT_ECALL_PEND);
116	return 0;
117	}
118
119	static void sca_clear_ext_call(struct kvm_vcpu *vcpu)
120	{
121	int rc, expect;
122
123	if (!kvm_s390_use_sca_entries())
124	return;
125	kvm_s390_clear_cpuflags(vcpu, flags: CPUSTAT_ECALL_PEND);
126	read_lock(&vcpu->kvm->arch.sca_lock);
127	if (vcpu->kvm->arch.use_esca) {
128	struct esca_block *sca = vcpu->kvm->arch.sca;
129	union esca_sigp_ctrl *sigp_ctrl =
130	&(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
131	union esca_sigp_ctrl old;
132
133	old = READ_ONCE(*sigp_ctrl);
134	expect = old.value;
135	rc = cmpxchg(&sigp_ctrl->value, old.value, 0);
136	} else {
137	struct bsca_block *sca = vcpu->kvm->arch.sca;
138	union bsca_sigp_ctrl *sigp_ctrl =
139	&(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
140	union bsca_sigp_ctrl old;
141
142	old = READ_ONCE(*sigp_ctrl);
143	expect = old.value;
144	rc = cmpxchg(&sigp_ctrl->value, old.value, 0);
145	}
146	read_unlock(&vcpu->kvm->arch.sca_lock);
147	WARN_ON(rc != expect); /* cannot clear? */
148	}
149
150	int psw_extint_disabled(struct kvm_vcpu *vcpu)
151	{
152	return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_EXT);
153	}
154
155	static int psw_ioint_disabled(struct kvm_vcpu *vcpu)
156	{
157	return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_IO);
158	}
159
160	static int psw_mchk_disabled(struct kvm_vcpu *vcpu)
161	{
162	return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_MCHECK);
163	}
164
165	static int psw_interrupts_disabled(struct kvm_vcpu *vcpu)
166	{
167	return psw_extint_disabled(vcpu) &&
168	psw_ioint_disabled(vcpu) &&
169	psw_mchk_disabled(vcpu);
170	}
171
172	static int ckc_interrupts_enabled(struct kvm_vcpu *vcpu)
173	{
174	if (psw_extint_disabled(vcpu) ||
175	!(vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SUBMASK))
176	return 0;
177	if (guestdbg_enabled(vcpu) && guestdbg_sstep_enabled(vcpu))
178	/* No timer interrupts when single stepping */
179	return 0;
180	return 1;
181	}
182
183	static int ckc_irq_pending(struct kvm_vcpu *vcpu)
184	{
185	const u64 now = kvm_s390_get_tod_clock_fast(kvm: vcpu->kvm);
186	const u64 ckc = vcpu->arch.sie_block->ckc;
187
188	if (vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SIGN) {
189	if ((s64)ckc >= (s64)now)
190	return 0;
191	} else if (ckc >= now) {
192	return 0;
193	}
194	return ckc_interrupts_enabled(vcpu);
195	}
196
197	static int cpu_timer_interrupts_enabled(struct kvm_vcpu *vcpu)
198	{
199	return !psw_extint_disabled(vcpu) &&
200	(vcpu->arch.sie_block->gcr[0] & CR0_CPU_TIMER_SUBMASK);
201	}
202
203	static int cpu_timer_irq_pending(struct kvm_vcpu *vcpu)
204	{
205	if (!cpu_timer_interrupts_enabled(vcpu))
206	return 0;
207	return kvm_s390_get_cpu_timer(vcpu) >> 63;
208	}
209
210	static uint64_t isc_to_isc_bits(int isc)
211	{
212	return (0x80 >> isc) << 24;
213	}
214
215	static inline u32 isc_to_int_word(u8 isc)
216	{
217	return ((u32)isc << 27) | 0x80000000;
218	}
219
220	static inline u8 int_word_to_isc(u32 int_word)
221	{
222	return (int_word & 0x38000000) >> 27;
223	}
224
225	/*
226	* To use atomic bitmap functions, we have to provide a bitmap address
227	* that is u64 aligned. However, the ipm might be u32 aligned.
228	* Therefore, we logically start the bitmap at the very beginning of the
229	* struct and fixup the bit number.
230	*/
231	#define IPM_BIT_OFFSET (offsetof(struct kvm_s390_gisa, ipm) * BITS_PER_BYTE)
232
233	/**
234	* gisa_set_iam - change the GISA interruption alert mask
235	*
236	* @gisa: gisa to operate on
237	* @iam: new IAM value to use
238	*
239	* Change the IAM atomically with the next alert address and the IPM
240	* of the GISA if the GISA is not part of the GIB alert list. All three
241	* fields are located in the first long word of the GISA.
242	*
243	* Returns: 0 on success
244	* -EBUSY in case the gisa is part of the alert list
245	*/
246	static inline int gisa_set_iam(struct kvm_s390_gisa *gisa, u8 iam)
247	{
248	u64 word, _word;
249
250	do {
251	word = READ_ONCE(gisa->u64.word[0]);
252	if ((u64)gisa != word >> 32)
253	return -EBUSY;
254	_word = (word & ~0xffUL) | iam;
255	} while (cmpxchg(&gisa->u64.word[0], word, _word) != word);
256
257	return 0;
258	}
259
260	/**
261	* gisa_clear_ipm - clear the GISA interruption pending mask
262	*
263	* @gisa: gisa to operate on
264	*
265	* Clear the IPM atomically with the next alert address and the IAM
266	* of the GISA unconditionally. All three fields are located in the
267	* first long word of the GISA.
268	*/
269	static inline void gisa_clear_ipm(struct kvm_s390_gisa *gisa)
270	{
271	u64 word, _word;
272
273	do {
274	word = READ_ONCE(gisa->u64.word[0]);
275	_word = word & ~(0xffUL << 24);
276	} while (cmpxchg(&gisa->u64.word[0], word, _word) != word);
277	}
278
279	/**
280	* gisa_get_ipm_or_restore_iam - return IPM or restore GISA IAM
281	*
282	* @gi: gisa interrupt struct to work on
283	*
284	* Atomically restores the interruption alert mask if none of the
285	* relevant ISCs are pending and return the IPM.
286	*
287	* Returns: the relevant pending ISCs
288	*/
289	static inline u8 gisa_get_ipm_or_restore_iam(struct kvm_s390_gisa_interrupt *gi)
290	{
291	u8 pending_mask, alert_mask;
292	u64 word, _word;
293
294	do {
295	word = READ_ONCE(gi->origin->u64.word[0]);
296	alert_mask = READ_ONCE(gi->alert.mask);
297	pending_mask = (u8)(word >> 24) & alert_mask;
298	if (pending_mask)
299	return pending_mask;
300	_word = (word & ~0xffUL) | alert_mask;
301	} while (cmpxchg(&gi->origin->u64.word[0], word, _word) != word);
302
303	return 0;
304	}
305
306	static inline void gisa_set_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc)
307	{
308	set_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa);
309	}
310
311	static inline u8 gisa_get_ipm(struct kvm_s390_gisa *gisa)
312	{
313	return READ_ONCE(gisa->ipm);
314	}
315
316	static inline int gisa_tac_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc)
317	{
318	return test_and_clear_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa);
319	}
320
321	static inline unsigned long pending_irqs_no_gisa(struct kvm_vcpu *vcpu)
322	{
323	unsigned long pending = vcpu->kvm->arch.float_int.pending_irqs |
324	vcpu->arch.local_int.pending_irqs;
325
326	pending &= ~vcpu->kvm->arch.float_int.masked_irqs;
327	return pending;
328	}
329
330	static inline unsigned long pending_irqs(struct kvm_vcpu *vcpu)
331	{
332	struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int;
333	unsigned long pending_mask;
334
335	pending_mask = pending_irqs_no_gisa(vcpu);
336	if (gi->origin)
337	pending_mask |= gisa_get_ipm(gi->origin) << IRQ_PEND_IO_ISC_7;
338	return pending_mask;
339	}
340
341	static inline int isc_to_irq_type(unsigned long isc)
342	{
343	return IRQ_PEND_IO_ISC_0 - isc;
344	}
345
346	static inline int irq_type_to_isc(unsigned long irq_type)
347	{
348	return IRQ_PEND_IO_ISC_0 - irq_type;
349	}
350
351	static unsigned long disable_iscs(struct kvm_vcpu *vcpu,
352	unsigned long active_mask)
353	{
354	int i;
355
356	for (i = 0; i <= MAX_ISC; i++)
357	if (!(vcpu->arch.sie_block->gcr[6] & isc_to_isc_bits(i)))
358	active_mask &= ~(1UL << (isc_to_irq_type(i)));
359
360	return active_mask;
361	}
362
363	static unsigned long deliverable_irqs(struct kvm_vcpu *vcpu)
364	{
365	unsigned long active_mask;
366
367	active_mask = pending_irqs(vcpu);
368	if (!active_mask)
369	return 0;
370
371	if (psw_extint_disabled(vcpu))
372	active_mask &= ~IRQ_PEND_EXT_MASK;
373	if (psw_ioint_disabled(vcpu))
374	active_mask &= ~IRQ_PEND_IO_MASK;
375	else
376	active_mask = disable_iscs(vcpu, active_mask);
377	if (!(vcpu->arch.sie_block->gcr[0] & CR0_EXTERNAL_CALL_SUBMASK))
378	__clear_bit(IRQ_PEND_EXT_EXTERNAL, &active_mask);
379	if (!(vcpu->arch.sie_block->gcr[0] & CR0_EMERGENCY_SIGNAL_SUBMASK))
380	__clear_bit(IRQ_PEND_EXT_EMERGENCY, &active_mask);
381	if (!(vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SUBMASK))
382	__clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &active_mask);
383	if (!(vcpu->arch.sie_block->gcr[0] & CR0_CPU_TIMER_SUBMASK))
384	__clear_bit(IRQ_PEND_EXT_CPU_TIMER, &active_mask);
385	if (!(vcpu->arch.sie_block->gcr[0] & CR0_SERVICE_SIGNAL_SUBMASK)) {
386	__clear_bit(IRQ_PEND_EXT_SERVICE, &active_mask);
387	__clear_bit(IRQ_PEND_EXT_SERVICE_EV, &active_mask);
388	}
389	if (psw_mchk_disabled(vcpu))
390	active_mask &= ~IRQ_PEND_MCHK_MASK;
391	/* PV guest cpus can have a single interruption injected at a time. */
392	if (kvm_s390_pv_cpu_get_handle(vcpu) &&
393	vcpu->arch.sie_block->iictl != IICTL_CODE_NONE)
394	active_mask &= ~(IRQ_PEND_EXT_II_MASK |
395	IRQ_PEND_IO_MASK |
396	IRQ_PEND_MCHK_MASK);
397	/*
398	* Check both floating and local interrupt's cr14 because
399	* bit IRQ_PEND_MCHK_REP could be set in both cases.
400	*/
401	if (!(vcpu->arch.sie_block->gcr[14] &
402	(vcpu->kvm->arch.float_int.mchk.cr14 |
403	vcpu->arch.local_int.irq.mchk.cr14)))
404	__clear_bit(IRQ_PEND_MCHK_REP, &active_mask);
405
406	/*
407	* STOP irqs will never be actively delivered. They are triggered via
408	* intercept requests and cleared when the stop intercept is performed.
409	*/
410	__clear_bit(IRQ_PEND_SIGP_STOP, &active_mask);
411
412	return active_mask;
413	}
414
415	static void __set_cpu_idle(struct kvm_vcpu *vcpu)
416	{
417	kvm_s390_set_cpuflags(vcpu, CPUSTAT_WAIT);
418	set_bit(nr: vcpu->vcpu_idx, addr: vcpu->kvm->arch.idle_mask);
419	}
420
421	static void __unset_cpu_idle(struct kvm_vcpu *vcpu)
422	{
423	kvm_s390_clear_cpuflags(vcpu, CPUSTAT_WAIT);
424	clear_bit(nr: vcpu->vcpu_idx, addr: vcpu->kvm->arch.idle_mask);
425	}
426
427	static void __reset_intercept_indicators(struct kvm_vcpu *vcpu)
428	{
429	kvm_s390_clear_cpuflags(vcpu, CPUSTAT_IO_INT | CPUSTAT_EXT_INT |
430	CPUSTAT_STOP_INT);
431	vcpu->arch.sie_block->lctl = 0x0000;
432	vcpu->arch.sie_block->ictl &= ~(ICTL_LPSW | ICTL_STCTL | ICTL_PINT);
433
434	if (guestdbg_enabled(vcpu)) {
435	vcpu->arch.sie_block->lctl |= (LCTL_CR0 | LCTL_CR9 |
436	LCTL_CR10 | LCTL_CR11);
437	vcpu->arch.sie_block->ictl |= (ICTL_STCTL | ICTL_PINT);
438	}
439	}
440
441	static void set_intercept_indicators_io(struct kvm_vcpu *vcpu)
442	{
443	if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_IO_MASK))
444	return;
445	if (psw_ioint_disabled(vcpu))
446	kvm_s390_set_cpuflags(vcpu, CPUSTAT_IO_INT);
447	else
448	vcpu->arch.sie_block->lctl |= LCTL_CR6;
449	}
450
451	static void set_intercept_indicators_ext(struct kvm_vcpu *vcpu)
452	{
453	if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_EXT_MASK))
454	return;
455	if (psw_extint_disabled(vcpu))
456	kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
457	else
458	vcpu->arch.sie_block->lctl |= LCTL_CR0;
459	}
460
461	static void set_intercept_indicators_mchk(struct kvm_vcpu *vcpu)
462	{
463	if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_MCHK_MASK))
464	return;
465	if (psw_mchk_disabled(vcpu))
466	vcpu->arch.sie_block->ictl |= ICTL_LPSW;
467	else
468	vcpu->arch.sie_block->lctl |= LCTL_CR14;
469	}
470
471	static void set_intercept_indicators_stop(struct kvm_vcpu *vcpu)
472	{
473	if (kvm_s390_is_stop_irq_pending(vcpu))
474	kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
475	}
476
477	/* Set interception request for non-deliverable interrupts */
478	static void set_intercept_indicators(struct kvm_vcpu *vcpu)
479	{
480	set_intercept_indicators_io(vcpu);
481	set_intercept_indicators_ext(vcpu);
482	set_intercept_indicators_mchk(vcpu);
483	set_intercept_indicators_stop(vcpu);
484	}
485
486	static int __must_check __deliver_cpu_timer(struct kvm_vcpu *vcpu)
487	{
488	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
489	int rc = 0;
490
491	vcpu->stat.deliver_cputm++;
492	trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_CPU_TIMER,
493	0, 0);
494	if (kvm_s390_pv_cpu_is_protected(vcpu)) {
495	vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
496	vcpu->arch.sie_block->eic = EXT_IRQ_CPU_TIMER;
497	} else {
498	rc = put_guest_lc(vcpu, EXT_IRQ_CPU_TIMER,
499	(u16 *)__LC_EXT_INT_CODE);
500	rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR);
501	rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
502	&vcpu->arch.sie_block->gpsw, sizeof(psw_t));
503	rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
504	&vcpu->arch.sie_block->gpsw, sizeof(psw_t));
505	}
506	clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
507	return rc ? -EFAULT : 0;
508	}
509
510	static int __must_check __deliver_ckc(struct kvm_vcpu *vcpu)
511	{
512	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
513	int rc = 0;
514
515	vcpu->stat.deliver_ckc++;
516	trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_CLOCK_COMP,
517	0, 0);
518	if (kvm_s390_pv_cpu_is_protected(vcpu)) {
519	vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
520	vcpu->arch.sie_block->eic = EXT_IRQ_CLK_COMP;
521	} else {
522	rc = put_guest_lc(vcpu, EXT_IRQ_CLK_COMP,
523	(u16 __user *)__LC_EXT_INT_CODE);
524	rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR);
525	rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
526	&vcpu->arch.sie_block->gpsw, sizeof(psw_t));
527	rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
528	&vcpu->arch.sie_block->gpsw, sizeof(psw_t));
529	}
530	clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
531	return rc ? -EFAULT : 0;
532	}
533
534	static int __must_check __deliver_pfault_init(struct kvm_vcpu *vcpu)
535	{
536	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
537	struct kvm_s390_ext_info ext;
538	int rc;
539
540	spin_lock(lock: &li->lock);
541	ext = li->irq.ext;
542	clear_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs);
543	li->irq.ext.ext_params2 = 0;
544	spin_unlock(lock: &li->lock);
545
546	VCPU_EVENT(vcpu, 4, "deliver: pfault init token 0x%llx",
547	ext.ext_params2);
548	trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
549	KVM_S390_INT_PFAULT_INIT,
550	0, ext.ext_params2);
551
552	rc = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, (u16 *) __LC_EXT_INT_CODE);
553	rc |= put_guest_lc(vcpu, PFAULT_INIT, (u16 *) __LC_EXT_CPU_ADDR);
554	rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
555	&vcpu->arch.sie_block->gpsw, sizeof(psw_t));
556	rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
557	&vcpu->arch.sie_block->gpsw, sizeof(psw_t));
558	rc |= put_guest_lc(vcpu, ext.ext_params2, (u64 *) __LC_EXT_PARAMS2);
559	return rc ? -EFAULT : 0;
560	}
561
562	static int __write_machine_check(struct kvm_vcpu *vcpu,
563	struct kvm_s390_mchk_info *mchk)
564	{
565	unsigned long ext_sa_addr;
566	unsigned long lc;
567	freg_t fprs[NUM_FPRS];
568	union mci mci;
569	int rc;
570
571	/*
572	* All other possible payload for a machine check (e.g. the register
573	* contents in the save area) will be handled by the ultravisor, as
574	* the hypervisor does not not have the needed information for
575	* protected guests.
576	*/
577	if (kvm_s390_pv_cpu_is_protected(vcpu)) {
578	vcpu->arch.sie_block->iictl = IICTL_CODE_MCHK;
579	vcpu->arch.sie_block->mcic = mchk->mcic;
580	vcpu->arch.sie_block->faddr = mchk->failing_storage_address;
581	vcpu->arch.sie_block->edc = mchk->ext_damage_code;
582	return 0;
583	}
584
585	mci.val = mchk->mcic;
586	/* take care of lazy register loading */
587	kvm_s390_fpu_store(run: vcpu->run);
588	save_access_regs(vcpu->run->s.regs.acrs);
589	if (MACHINE_HAS_GS && vcpu->arch.gs_enabled)
590	save_gs_cb(current->thread.gs_cb);
591
592	/* Extended save area */
593	rc = read_guest_lc(vcpu, __LC_MCESAD, &ext_sa_addr,
594	sizeof(unsigned long));
595	/* Only bits 0 through 63-LC are used for address formation */
596	lc = ext_sa_addr & MCESA_LC_MASK;
597	if (test_kvm_facility(kvm: vcpu->kvm, nr: 133)) {
598	switch (lc) {
599	case 0:
600	case 10:
601	ext_sa_addr &= ~0x3ffUL;
602	break;
603	case 11:
604	ext_sa_addr &= ~0x7ffUL;
605	break;
606	case 12:
607	ext_sa_addr &= ~0xfffUL;
608	break;
609	default:
610	ext_sa_addr = 0;
611	break;
612	}
613	} else {
614	ext_sa_addr &= ~0x3ffUL;
615	}
616
617	if (!rc && mci.vr && ext_sa_addr && test_kvm_facility(kvm: vcpu->kvm, nr: 129)) {
618	if (write_guest_abs(vcpu, gpa: ext_sa_addr, data: vcpu->run->s.regs.vrs,
619	len: 512))
620	mci.vr = 0;
621	} else {
622	mci.vr = 0;
623	}
624	if (!rc && mci.gs && ext_sa_addr && test_kvm_facility(kvm: vcpu->kvm, nr: 133)
625	&& (lc == 11 || lc == 12)) {
626	if (write_guest_abs(vcpu, gpa: ext_sa_addr + 1024,
627	data: &vcpu->run->s.regs.gscb, len: 32))
628	mci.gs = 0;
629	} else {
630	mci.gs = 0;
631	}
632
633	/* General interruption information */
634	rc |= put_guest_lc(vcpu, 1, (u8 __user *) __LC_AR_MODE_ID);
635	rc |= write_guest_lc(vcpu, __LC_MCK_OLD_PSW,
636	&vcpu->arch.sie_block->gpsw, sizeof(psw_t));
637	rc |= read_guest_lc(vcpu, __LC_MCK_NEW_PSW,
638	&vcpu->arch.sie_block->gpsw, sizeof(psw_t));
639	rc |= put_guest_lc(vcpu, mci.val, (u64 __user *) __LC_MCCK_CODE);
640
641	/* Register-save areas */
642	if (cpu_has_vx()) {
643	convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
644	rc |= write_guest_lc(vcpu, __LC_FPREGS_SAVE_AREA, fprs, 128);
645	} else {
646	rc |= write_guest_lc(vcpu, __LC_FPREGS_SAVE_AREA,
647	vcpu->run->s.regs.fprs, 128);
648	}
649	rc |= write_guest_lc(vcpu, __LC_GPREGS_SAVE_AREA,
650	vcpu->run->s.regs.gprs, 128);
651	rc |= put_guest_lc(vcpu, vcpu->run->s.regs.fpc,
652	(u32 __user *) __LC_FP_CREG_SAVE_AREA);
653	rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->todpr,
654	(u32 __user *) __LC_TOD_PROGREG_SAVE_AREA);
655	rc |= put_guest_lc(vcpu, kvm_s390_get_cpu_timer(vcpu),
656	(u64 __user *) __LC_CPU_TIMER_SAVE_AREA);
657	rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->ckc >> 8,
658	(u64 __user *) __LC_CLOCK_COMP_SAVE_AREA);
659	rc |= write_guest_lc(vcpu, __LC_AREGS_SAVE_AREA,
660	&vcpu->run->s.regs.acrs, 64);
661	rc |= write_guest_lc(vcpu, __LC_CREGS_SAVE_AREA,
662	&vcpu->arch.sie_block->gcr, 128);
663
664	/* Extended interruption information */
665	rc |= put_guest_lc(vcpu, mchk->ext_damage_code,
666	(u32 __user *) __LC_EXT_DAMAGE_CODE);
667	rc |= put_guest_lc(vcpu, mchk->failing_storage_address,
668	(u64 __user *) __LC_MCCK_FAIL_STOR_ADDR);
669	rc |= write_guest_lc(vcpu, __LC_PSW_SAVE_AREA, &mchk->fixed_logout,
670	sizeof(mchk->fixed_logout));
671	return rc ? -EFAULT : 0;
672	}
673
674	static int __must_check __deliver_machine_check(struct kvm_vcpu *vcpu)
675	{
676	struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
677	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
678	struct kvm_s390_mchk_info mchk = {};
679	int deliver = 0;
680	int rc = 0;
681
682	spin_lock(lock: &fi->lock);
683	spin_lock(lock: &li->lock);
684	if (test_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs) ||
685	test_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs)) {
686	/*
687	* If there was an exigent machine check pending, then any
688	* repressible machine checks that might have been pending
689	* are indicated along with it, so always clear bits for
690	* repressible and exigent interrupts
691	*/
692	mchk = li->irq.mchk;
693	clear_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs);
694	clear_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs);
695	memset(&li->irq.mchk, 0, sizeof(mchk));
696	deliver = 1;
697	}
698	/*
699	* We indicate floating repressible conditions along with
700	* other pending conditions. Channel Report Pending and Channel
701	* Subsystem damage are the only two and are indicated by
702	* bits in mcic and masked in cr14.
703	*/
704	if (test_and_clear_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) {
705	mchk.mcic |= fi->mchk.mcic;
706	mchk.cr14 |= fi->mchk.cr14;
707	memset(&fi->mchk, 0, sizeof(mchk));
708	deliver = 1;
709	}
710	spin_unlock(lock: &li->lock);
711	spin_unlock(lock: &fi->lock);
712
713	if (deliver) {
714	VCPU_EVENT(vcpu, 3, "deliver: machine check mcic 0x%llx",
715	mchk.mcic);
716	trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
717	KVM_S390_MCHK,
718	mchk.cr14, mchk.mcic);
719	vcpu->stat.deliver_machine_check++;
720	rc = __write_machine_check(vcpu, mchk: &mchk);
721	}
722	return rc;
723	}
724
725	static int __must_check __deliver_restart(struct kvm_vcpu *vcpu)
726	{
727	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
728	int rc = 0;
729
730	VCPU_EVENT(vcpu, 3, "%s", "deliver: cpu restart");
731	vcpu->stat.deliver_restart_signal++;
732	trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_RESTART, 0, 0);
733
734	if (kvm_s390_pv_cpu_is_protected(vcpu)) {
735	vcpu->arch.sie_block->iictl = IICTL_CODE_RESTART;
736	} else {
737	rc = write_guest_lc(vcpu,
738	offsetof(struct lowcore, restart_old_psw),
739	&vcpu->arch.sie_block->gpsw, sizeof(psw_t));
740	rc |= read_guest_lc(vcpu, offsetof(struct lowcore, restart_psw),
741	&vcpu->arch.sie_block->gpsw, sizeof(psw_t));
742	}
743	clear_bit(IRQ_PEND_RESTART, &li->pending_irqs);
744	return rc ? -EFAULT : 0;
745	}
746
747	static int __must_check __deliver_set_prefix(struct kvm_vcpu *vcpu)
748	{
749	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
750	struct kvm_s390_prefix_info prefix;
751
752	spin_lock(lock: &li->lock);
753	prefix = li->irq.prefix;
754	li->irq.prefix.address = 0;
755	clear_bit(IRQ_PEND_SET_PREFIX, &li->pending_irqs);
756	spin_unlock(lock: &li->lock);
757
758	vcpu->stat.deliver_prefix_signal++;
759	trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
760	KVM_S390_SIGP_SET_PREFIX,
761	prefix.address, 0);
762
763	kvm_s390_set_prefix(vcpu, prefix: prefix.address);
764	return 0;
765	}
766
767	static int __must_check __deliver_emergency_signal(struct kvm_vcpu *vcpu)
768	{
769	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
770	int rc;
771	int cpu_addr;
772
773	spin_lock(lock: &li->lock);
774	cpu_addr = find_first_bit(addr: li->sigp_emerg_pending, KVM_MAX_VCPUS);
775	clear_bit(nr: cpu_addr, addr: li->sigp_emerg_pending);
776	if (bitmap_empty(li->sigp_emerg_pending, KVM_MAX_VCPUS))
777	clear_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs);
778	spin_unlock(lock: &li->lock);
779
780	VCPU_EVENT(vcpu, 4, "%s", "deliver: sigp emerg");
781	vcpu->stat.deliver_emergency_signal++;
782	trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_EMERGENCY,
783	cpu_addr, 0);
784	if (kvm_s390_pv_cpu_is_protected(vcpu)) {
785	vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
786	vcpu->arch.sie_block->eic = EXT_IRQ_EMERGENCY_SIG;
787	vcpu->arch.sie_block->extcpuaddr = cpu_addr;
788	return 0;
789	}
790
791	rc = put_guest_lc(vcpu, EXT_IRQ_EMERGENCY_SIG,
792	(u16 *)__LC_EXT_INT_CODE);
793	rc |= put_guest_lc(vcpu, cpu_addr, (u16 *)__LC_EXT_CPU_ADDR);
794	rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
795	&vcpu->arch.sie_block->gpsw, sizeof(psw_t));
796	rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
797	&vcpu->arch.sie_block->gpsw, sizeof(psw_t));
798	return rc ? -EFAULT : 0;
799	}
800
801	static int __must_check __deliver_external_call(struct kvm_vcpu *vcpu)
802	{
803	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
804	struct kvm_s390_extcall_info extcall;
805	int rc;
806
807	spin_lock(lock: &li->lock);
808	extcall = li->irq.extcall;
809	li->irq.extcall.code = 0;
810	clear_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs);
811	spin_unlock(lock: &li->lock);
812
813	VCPU_EVENT(vcpu, 4, "%s", "deliver: sigp ext call");
814	vcpu->stat.deliver_external_call++;
815	trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
816	KVM_S390_INT_EXTERNAL_CALL,
817	extcall.code, 0);
818	if (kvm_s390_pv_cpu_is_protected(vcpu)) {
819	vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
820	vcpu->arch.sie_block->eic = EXT_IRQ_EXTERNAL_CALL;
821	vcpu->arch.sie_block->extcpuaddr = extcall.code;
822	return 0;
823	}
824
825	rc = put_guest_lc(vcpu, EXT_IRQ_EXTERNAL_CALL,
826	(u16 *)__LC_EXT_INT_CODE);
827	rc |= put_guest_lc(vcpu, extcall.code, (u16 *)__LC_EXT_CPU_ADDR);
828	rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
829	&vcpu->arch.sie_block->gpsw, sizeof(psw_t));
830	rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, &vcpu->arch.sie_block->gpsw,
831	sizeof(psw_t));
832	return rc ? -EFAULT : 0;
833	}
834
835	static int __deliver_prog_pv(struct kvm_vcpu *vcpu, u16 code)
836	{
837	switch (code) {
838	case PGM_SPECIFICATION:
839	vcpu->arch.sie_block->iictl = IICTL_CODE_SPECIFICATION;
840	break;
841	case PGM_OPERAND:
842	vcpu->arch.sie_block->iictl = IICTL_CODE_OPERAND;
843	break;
844	default:
845	return -EINVAL;
846	}
847	return 0;
848	}
849
850	static int __must_check __deliver_prog(struct kvm_vcpu *vcpu)
851	{
852	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
853	struct kvm_s390_pgm_info pgm_info;
854	int rc = 0, nullifying = false;
855	u16 ilen;
856
857	spin_lock(lock: &li->lock);
858	pgm_info = li->irq.pgm;
859	clear_bit(IRQ_PEND_PROG, &li->pending_irqs);
860	memset(&li->irq.pgm, 0, sizeof(pgm_info));
861	spin_unlock(lock: &li->lock);
862
863	ilen = pgm_info.flags & KVM_S390_PGM_FLAGS_ILC_MASK;
864	VCPU_EVENT(vcpu, 3, "deliver: program irq code 0x%x, ilen:%d",
865	pgm_info.code, ilen);
866	vcpu->stat.deliver_program++;
867	trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_PROGRAM_INT,
868	pgm_info.code, 0);
869
870	/* PER is handled by the ultravisor */
871	if (kvm_s390_pv_cpu_is_protected(vcpu))
872	return __deliver_prog_pv(vcpu, pgm_info.code & ~PGM_PER);
873
874	switch (pgm_info.code & ~PGM_PER) {
875	case PGM_AFX_TRANSLATION:
876	case PGM_ASX_TRANSLATION:
877	case PGM_EX_TRANSLATION:
878	case PGM_LFX_TRANSLATION:
879	case PGM_LSTE_SEQUENCE:
880	case PGM_LSX_TRANSLATION:
881	case PGM_LX_TRANSLATION:
882	case PGM_PRIMARY_AUTHORITY:
883	case PGM_SECONDARY_AUTHORITY:
884	nullifying = true;
885	fallthrough;
886	case PGM_SPACE_SWITCH:
887	rc = put_guest_lc(vcpu, pgm_info.trans_exc_code,
888	(u64 *)__LC_TRANS_EXC_CODE);
889	break;
890	case PGM_ALEN_TRANSLATION:
891	case PGM_ALE_SEQUENCE:
892	case PGM_ASTE_INSTANCE:
893	case PGM_ASTE_SEQUENCE:
894	case PGM_ASTE_VALIDITY:
895	case PGM_EXTENDED_AUTHORITY:
896	rc = put_guest_lc(vcpu, pgm_info.exc_access_id,
897	(u8 *)__LC_EXC_ACCESS_ID);
898	nullifying = true;
899	break;
900	case PGM_ASCE_TYPE:
901	case PGM_PAGE_TRANSLATION:
902	case PGM_REGION_FIRST_TRANS:
903	case PGM_REGION_SECOND_TRANS:
904	case PGM_REGION_THIRD_TRANS:
905	case PGM_SEGMENT_TRANSLATION:
906	rc = put_guest_lc(vcpu, pgm_info.trans_exc_code,
907	(u64 *)__LC_TRANS_EXC_CODE);
908	rc |= put_guest_lc(vcpu, pgm_info.exc_access_id,
909	(u8 *)__LC_EXC_ACCESS_ID);
910	rc |= put_guest_lc(vcpu, pgm_info.op_access_id,
911	(u8 *)__LC_OP_ACCESS_ID);
912	nullifying = true;
913	break;
914	case PGM_MONITOR:
915	rc = put_guest_lc(vcpu, pgm_info.mon_class_nr,
916	(u16 *)__LC_MON_CLASS_NR);
917	rc |= put_guest_lc(vcpu, pgm_info.mon_code,
918	(u64 *)__LC_MON_CODE);
919	break;
920	case PGM_VECTOR_PROCESSING:
921	case PGM_DATA:
922	rc = put_guest_lc(vcpu, pgm_info.data_exc_code,
923	(u32 *)__LC_DATA_EXC_CODE);
924	break;
925	case PGM_PROTECTION:
926	rc = put_guest_lc(vcpu, pgm_info.trans_exc_code,
927	(u64 *)__LC_TRANS_EXC_CODE);
928	rc |= put_guest_lc(vcpu, pgm_info.exc_access_id,
929	(u8 *)__LC_EXC_ACCESS_ID);
930	break;
931	case PGM_STACK_FULL:
932	case PGM_STACK_EMPTY:
933	case PGM_STACK_SPECIFICATION:
934	case PGM_STACK_TYPE:
935	case PGM_STACK_OPERATION:
936	case PGM_TRACE_TABEL:
937	case PGM_CRYPTO_OPERATION:
938	nullifying = true;
939	break;
940	}
941
942	if (pgm_info.code & PGM_PER) {
943	rc |= put_guest_lc(vcpu, pgm_info.per_code,
944	(u8 *) __LC_PER_CODE);
945	rc |= put_guest_lc(vcpu, pgm_info.per_atmid,
946	(u8 *)__LC_PER_ATMID);
947	rc |= put_guest_lc(vcpu, pgm_info.per_address,
948	(u64 *) __LC_PER_ADDRESS);
949	rc |= put_guest_lc(vcpu, pgm_info.per_access_id,
950	(u8 *) __LC_PER_ACCESS_ID);
951	}
952
953	if (nullifying && !(pgm_info.flags & KVM_S390_PGM_FLAGS_NO_REWIND))
954	kvm_s390_rewind_psw(vcpu, ilen);
955
956	/* bit 1+2 of the target are the ilc, so we can directly use ilen */
957	rc |= put_guest_lc(vcpu, ilen, (u16 *) __LC_PGM_ILC);
958	rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->gbea,
959	(u64 *) __LC_PGM_LAST_BREAK);
960	rc |= put_guest_lc(vcpu, pgm_info.code,
961	(u16 *)__LC_PGM_INT_CODE);
962	rc |= write_guest_lc(vcpu, __LC_PGM_OLD_PSW,
963	&vcpu->arch.sie_block->gpsw, sizeof(psw_t));
964	rc |= read_guest_lc(vcpu, __LC_PGM_NEW_PSW,
965	&vcpu->arch.sie_block->gpsw, sizeof(psw_t));
966	return rc ? -EFAULT : 0;
967	}
968
969	#define SCCB_MASK 0xFFFFFFF8
970	#define SCCB_EVENT_PENDING 0x3
971
972	static int write_sclp(struct kvm_vcpu *vcpu, u32 parm)
973	{
974	int rc;
975
976	if (kvm_s390_pv_cpu_get_handle(vcpu)) {
977	vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
978	vcpu->arch.sie_block->eic = EXT_IRQ_SERVICE_SIG;
979	vcpu->arch.sie_block->eiparams = parm;
980	return 0;
981	}
982
983	rc = put_guest_lc(vcpu, EXT_IRQ_SERVICE_SIG, (u16 *)__LC_EXT_INT_CODE);
984	rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR);
985	rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
986	&vcpu->arch.sie_block->gpsw, sizeof(psw_t));
987	rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
988	&vcpu->arch.sie_block->gpsw, sizeof(psw_t));
989	rc |= put_guest_lc(vcpu, parm,
990	(u32 *)__LC_EXT_PARAMS);
991
992	return rc ? -EFAULT : 0;
993	}
994
995	static int __must_check __deliver_service(struct kvm_vcpu *vcpu)
996	{
997	struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
998	struct kvm_s390_ext_info ext;
999
1000	spin_lock(lock: &fi->lock);
1001	if (test_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs) ||
1002	!(test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs))) {
1003	spin_unlock(lock: &fi->lock);
1004	return 0;
1005	}
1006	ext = fi->srv_signal;
1007	memset(&fi->srv_signal, 0, sizeof(ext));
1008	clear_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs);
1009	clear_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs);
1010	if (kvm_s390_pv_cpu_is_protected(vcpu))
1011	set_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs);
1012	spin_unlock(lock: &fi->lock);
1013
1014	VCPU_EVENT(vcpu, 4, "deliver: sclp parameter 0x%x",
1015	ext.ext_params);
1016	vcpu->stat.deliver_service_signal++;
1017	trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_SERVICE,
1018	ext.ext_params, 0);
1019
1020	return write_sclp(vcpu, parm: ext.ext_params);
1021	}
1022
1023	static int __must_check __deliver_service_ev(struct kvm_vcpu *vcpu)
1024	{
1025	struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
1026	struct kvm_s390_ext_info ext;
1027
1028	spin_lock(lock: &fi->lock);
1029	if (!(test_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs))) {
1030	spin_unlock(lock: &fi->lock);
1031	return 0;
1032	}
1033	ext = fi->srv_signal;
1034	/* only clear the event bits */
1035	fi->srv_signal.ext_params &= ~SCCB_EVENT_PENDING;
1036	clear_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs);
1037	spin_unlock(lock: &fi->lock);
1038
1039	VCPU_EVENT(vcpu, 4, "%s", "deliver: sclp parameter event");
1040	vcpu->stat.deliver_service_signal++;
1041	trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_SERVICE,
1042	ext.ext_params, 0);
1043
1044	return write_sclp(vcpu, parm: ext.ext_params & SCCB_EVENT_PENDING);
1045	}
1046
1047	static int __must_check __deliver_pfault_done(struct kvm_vcpu *vcpu)
1048	{
1049	struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
1050	struct kvm_s390_interrupt_info *inti;
1051	int rc = 0;
1052
1053	spin_lock(lock: &fi->lock);
1054	inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_PFAULT],
1055	struct kvm_s390_interrupt_info,
1056	list);
1057	if (inti) {
1058	list_del(entry: &inti->list);
1059	fi->counters[FIRQ_CNTR_PFAULT] -= 1;
1060	}
1061	if (list_empty(&fi->lists[FIRQ_LIST_PFAULT]))
1062	clear_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs);
1063	spin_unlock(lock: &fi->lock);
1064
1065	if (inti) {
1066	trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
1067	KVM_S390_INT_PFAULT_DONE, 0,
1068	inti->ext.ext_params2);
1069	VCPU_EVENT(vcpu, 4, "deliver: pfault done token 0x%llx",
1070	inti->ext.ext_params2);
1071
1072	rc = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE,
1073	(u16 *)__LC_EXT_INT_CODE);
1074	rc |= put_guest_lc(vcpu, PFAULT_DONE,
1075	(u16 *)__LC_EXT_CPU_ADDR);
1076	rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
1077	&vcpu->arch.sie_block->gpsw,
1078	sizeof(psw_t));
1079	rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
1080	&vcpu->arch.sie_block->gpsw,
1081	sizeof(psw_t));
1082	rc |= put_guest_lc(vcpu, inti->ext.ext_params2,
1083	(u64 *)__LC_EXT_PARAMS2);
1084	kfree(objp: inti);
1085	}
1086	return rc ? -EFAULT : 0;
1087	}
1088
1089	static int __must_check __deliver_virtio(struct kvm_vcpu *vcpu)
1090	{
1091	struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
1092	struct kvm_s390_interrupt_info *inti;
1093	int rc = 0;
1094
1095	spin_lock(lock: &fi->lock);
1096	inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_VIRTIO],
1097	struct kvm_s390_interrupt_info,
1098	list);
1099	if (inti) {
1100	VCPU_EVENT(vcpu, 4,
1101	"deliver: virtio parm: 0x%x,parm64: 0x%llx",
1102	inti->ext.ext_params, inti->ext.ext_params2);
1103	vcpu->stat.deliver_virtio++;
1104	trace_kvm_s390_deliver_interrupt(id: vcpu->vcpu_id,
1105	type: inti->type,
1106	data0: inti->ext.ext_params,
1107	data1: inti->ext.ext_params2);
1108	list_del(entry: &inti->list);
1109	fi->counters[FIRQ_CNTR_VIRTIO] -= 1;
1110	}
1111	if (list_empty(&fi->lists[FIRQ_LIST_VIRTIO]))
1112	clear_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs);
1113	spin_unlock(lock: &fi->lock);
1114
1115	if (inti) {
1116	rc = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE,
1117	(u16 *)__LC_EXT_INT_CODE);
1118	rc |= put_guest_lc(vcpu, VIRTIO_PARAM,
1119	(u16 *)__LC_EXT_CPU_ADDR);
1120	rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
1121	&vcpu->arch.sie_block->gpsw,
1122	sizeof(psw_t));
1123	rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
1124	&vcpu->arch.sie_block->gpsw,
1125	sizeof(psw_t));
1126	rc |= put_guest_lc(vcpu, inti->ext.ext_params,
1127	(u32 *)__LC_EXT_PARAMS);
1128	rc |= put_guest_lc(vcpu, inti->ext.ext_params2,
1129	(u64 *)__LC_EXT_PARAMS2);
1130	kfree(objp: inti);
1131	}
1132	return rc ? -EFAULT : 0;
1133	}
1134
1135	static int __do_deliver_io(struct kvm_vcpu *vcpu, struct kvm_s390_io_info *io)
1136	{
1137	int rc;
1138
1139	if (kvm_s390_pv_cpu_is_protected(vcpu)) {
1140	vcpu->arch.sie_block->iictl = IICTL_CODE_IO;
1141	vcpu->arch.sie_block->subchannel_id = io->subchannel_id;
1142	vcpu->arch.sie_block->subchannel_nr = io->subchannel_nr;
1143	vcpu->arch.sie_block->io_int_parm = io->io_int_parm;
1144	vcpu->arch.sie_block->io_int_word = io->io_int_word;
1145	return 0;
1146	}
1147
1148	rc = put_guest_lc(vcpu, io->subchannel_id, (u16 *)__LC_SUBCHANNEL_ID);
1149	rc |= put_guest_lc(vcpu, io->subchannel_nr, (u16 *)__LC_SUBCHANNEL_NR);
1150	rc |= put_guest_lc(vcpu, io->io_int_parm, (u32 *)__LC_IO_INT_PARM);
1151	rc |= put_guest_lc(vcpu, io->io_int_word, (u32 *)__LC_IO_INT_WORD);
1152	rc |= write_guest_lc(vcpu, __LC_IO_OLD_PSW,
1153	&vcpu->arch.sie_block->gpsw,
1154	sizeof(psw_t));
1155	rc |= read_guest_lc(vcpu, __LC_IO_NEW_PSW,
1156	&vcpu->arch.sie_block->gpsw,
1157	sizeof(psw_t));
1158	return rc ? -EFAULT : 0;
1159	}
1160
1161	static int __must_check __deliver_io(struct kvm_vcpu *vcpu,
1162	unsigned long irq_type)
1163	{
1164	struct list_head *isc_list;
1165	struct kvm_s390_float_interrupt *fi;
1166	struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int;
1167	struct kvm_s390_interrupt_info *inti = NULL;
1168	struct kvm_s390_io_info io;
1169	u32 isc;
1170	int rc = 0;
1171
1172	fi = &vcpu->kvm->arch.float_int;
1173
1174	spin_lock(lock: &fi->lock);
1175	isc = irq_type_to_isc(irq_type);
1176	isc_list = &fi->lists[isc];
1177	inti = list_first_entry_or_null(isc_list,
1178	struct kvm_s390_interrupt_info,
1179	list);
1180	if (inti) {
1181	if (inti->type & KVM_S390_INT_IO_AI_MASK)
1182	VCPU_EVENT(vcpu, 4, "%s", "deliver: I/O (AI)");
1183	else
1184	VCPU_EVENT(vcpu, 4, "deliver: I/O %x ss %x schid %04x",
1185	inti->io.subchannel_id >> 8,
1186	inti->io.subchannel_id >> 1 & 0x3,
1187	inti->io.subchannel_nr);
1188
1189	vcpu->stat.deliver_io++;
1190	trace_kvm_s390_deliver_interrupt(id: vcpu->vcpu_id,
1191	type: inti->type,
1192	data0: ((__u32)inti->io.subchannel_id << 16) |
1193	inti->io.subchannel_nr,
1194	data1: ((__u64)inti->io.io_int_parm << 32) |
1195	inti->io.io_int_word);
1196	list_del(entry: &inti->list);
1197	fi->counters[FIRQ_CNTR_IO] -= 1;
1198	}
1199	if (list_empty(head: isc_list))
1200	clear_bit(nr: irq_type, addr: &fi->pending_irqs);
1201	spin_unlock(lock: &fi->lock);
1202
1203	if (inti) {
1204	rc = __do_deliver_io(vcpu, io: &(inti->io));
1205	kfree(objp: inti);
1206	goto out;
1207	}
1208
1209	if (gi->origin && gisa_tac_ipm_gisc(gisa: gi->origin, gisc: isc)) {
1210	/*
1211	* in case an adapter interrupt was not delivered
1212	* in SIE context KVM will handle the delivery
1213	*/
1214	VCPU_EVENT(vcpu, 4, "%s isc %u", "deliver: I/O (AI/gisa)", isc);
1215	memset(&io, 0, sizeof(io));
1216	io.io_int_word = isc_to_int_word(isc);
1217	vcpu->stat.deliver_io++;
1218	trace_kvm_s390_deliver_interrupt(id: vcpu->vcpu_id,
1219	type: KVM_S390_INT_IO(1, 0, 0, 0),
1220	data0: ((__u32)io.subchannel_id << 16) |
1221	io.subchannel_nr,
1222	data1: ((__u64)io.io_int_parm << 32) |
1223	io.io_int_word);
1224	rc = __do_deliver_io(vcpu, io: &io);
1225	}
1226	out:
1227	return rc;
1228	}
1229
1230	/* Check whether an external call is pending (deliverable or not) */
1231	int kvm_s390_ext_call_pending(struct kvm_vcpu *vcpu)
1232	{
1233	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1234
1235	if (!sclp.has_sigpif)
1236	return test_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs);
1237
1238	return sca_ext_call_pending(vcpu, NULL);
1239	}
1240
1241	int kvm_s390_vcpu_has_irq(struct kvm_vcpu *vcpu, int exclude_stop)
1242	{
1243	if (deliverable_irqs(vcpu))
1244	return 1;
1245
1246	if (kvm_cpu_has_pending_timer(vcpu))
1247	return 1;
1248
1249	/* external call pending and deliverable */
1250	if (kvm_s390_ext_call_pending(vcpu) &&
1251	!psw_extint_disabled(vcpu) &&
1252	(vcpu->arch.sie_block->gcr[0] & CR0_EXTERNAL_CALL_SUBMASK))
1253	return 1;
1254
1255	if (!exclude_stop && kvm_s390_is_stop_irq_pending(vcpu))
1256	return 1;
1257	return 0;
1258	}
1259
1260	int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
1261	{
1262	return ckc_irq_pending(vcpu) || cpu_timer_irq_pending(vcpu);
1263	}
1264
1265	static u64 __calculate_sltime(struct kvm_vcpu *vcpu)
1266	{
1267	const u64 now = kvm_s390_get_tod_clock_fast(kvm: vcpu->kvm);
1268	const u64 ckc = vcpu->arch.sie_block->ckc;
1269	u64 cputm, sltime = 0;
1270
1271	if (ckc_interrupts_enabled(vcpu)) {
1272	if (vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SIGN) {
1273	if ((s64)now < (s64)ckc)
1274	sltime = tod_to_ns((s64)ckc - (s64)now);
1275	} else if (now < ckc) {
1276	sltime = tod_to_ns(ckc - now);
1277	}
1278	/* already expired */
1279	if (!sltime)
1280	return 0;
1281	if (cpu_timer_interrupts_enabled(vcpu)) {
1282	cputm = kvm_s390_get_cpu_timer(vcpu);
1283	/* already expired? */
1284	if (cputm >> 63)
1285	return 0;
1286	return min_t(u64, sltime, tod_to_ns(cputm));
1287	}
1288	} else if (cpu_timer_interrupts_enabled(vcpu)) {
1289	sltime = kvm_s390_get_cpu_timer(vcpu);
1290	/* already expired? */
1291	if (sltime >> 63)
1292	return 0;
1293	}
1294	return sltime;
1295	}
1296
1297	int kvm_s390_handle_wait(struct kvm_vcpu *vcpu)
1298	{
1299	struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int;
1300	u64 sltime;
1301
1302	vcpu->stat.exit_wait_state++;
1303
1304	/* fast path */
1305	if (kvm_arch_vcpu_runnable(vcpu))
1306	return 0;
1307
1308	if (psw_interrupts_disabled(vcpu)) {
1309	VCPU_EVENT(vcpu, 3, "%s", "disabled wait");
1310	return -EOPNOTSUPP; /* disabled wait */
1311	}
1312
1313	if (gi->origin &&
1314	(gisa_get_ipm_or_restore_iam(gi) &
1315	vcpu->arch.sie_block->gcr[6] >> 24))
1316	return 0;
1317
1318	if (!ckc_interrupts_enabled(vcpu) &&
1319	!cpu_timer_interrupts_enabled(vcpu)) {
1320	VCPU_EVENT(vcpu, 3, "%s", "enabled wait w/o timer");
1321	__set_cpu_idle(vcpu);
1322	goto no_timer;
1323	}
1324
1325	sltime = __calculate_sltime(vcpu);
1326	if (!sltime)
1327	return 0;
1328
1329	__set_cpu_idle(vcpu);
1330	hrtimer_start(timer: &vcpu->arch.ckc_timer, tim: sltime, mode: HRTIMER_MODE_REL);
1331	VCPU_EVENT(vcpu, 4, "enabled wait: %llu ns", sltime);
1332	no_timer:
1333	kvm_vcpu_srcu_read_unlock(vcpu);
1334	kvm_vcpu_halt(vcpu);
1335	vcpu->valid_wakeup = false;
1336	__unset_cpu_idle(vcpu);
1337	kvm_vcpu_srcu_read_lock(vcpu);
1338
1339	hrtimer_cancel(timer: &vcpu->arch.ckc_timer);
1340	return 0;
1341	}
1342
1343	void kvm_s390_vcpu_wakeup(struct kvm_vcpu *vcpu)
1344	{
1345	vcpu->valid_wakeup = true;
1346	kvm_vcpu_wake_up(vcpu);
1347
1348	/*
1349	* The VCPU might not be sleeping but rather executing VSIE. Let's
1350	* kick it, so it leaves the SIE to process the request.
1351	*/
1352	kvm_s390_vsie_kick(vcpu);
1353	}
1354
1355	enum hrtimer_restart kvm_s390_idle_wakeup(struct hrtimer *timer)
1356	{
1357	struct kvm_vcpu *vcpu;
1358	u64 sltime;
1359
1360	vcpu = container_of(timer, struct kvm_vcpu, arch.ckc_timer);
1361	sltime = __calculate_sltime(vcpu);
1362
1363	/*
1364	* If the monotonic clock runs faster than the tod clock we might be
1365	* woken up too early and have to go back to sleep to avoid deadlocks.
1366	*/
1367	if (sltime && hrtimer_forward_now(timer, interval: ns_to_ktime(ns: sltime)))
1368	return HRTIMER_RESTART;
1369	kvm_s390_vcpu_wakeup(vcpu);
1370	return HRTIMER_NORESTART;
1371	}
1372
1373	void kvm_s390_clear_local_irqs(struct kvm_vcpu *vcpu)
1374	{
1375	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1376
1377	spin_lock(lock: &li->lock);
1378	li->pending_irqs = 0;
1379	bitmap_zero(dst: li->sigp_emerg_pending, KVM_MAX_VCPUS);
1380	memset(&li->irq, 0, sizeof(li->irq));
1381	spin_unlock(lock: &li->lock);
1382
1383	sca_clear_ext_call(vcpu);
1384	}
1385
1386	int __must_check kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu)
1387	{
1388	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1389	int rc = 0;
1390	bool delivered = false;
1391	unsigned long irq_type;
1392	unsigned long irqs;
1393
1394	__reset_intercept_indicators(vcpu);
1395
1396	/* pending ckc conditions might have been invalidated */
1397	clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
1398	if (ckc_irq_pending(vcpu))
1399	set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
1400
1401	/* pending cpu timer conditions might have been invalidated */
1402	clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
1403	if (cpu_timer_irq_pending(vcpu))
1404	set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
1405
1406	while ((irqs = deliverable_irqs(vcpu)) && !rc) {
1407	/* bits are in the reverse order of interrupt priority */
1408	irq_type = find_last_bit(&irqs, IRQ_PEND_COUNT);
1409	switch (irq_type) {
1410	case IRQ_PEND_IO_ISC_0:
1411	case IRQ_PEND_IO_ISC_1:
1412	case IRQ_PEND_IO_ISC_2:
1413	case IRQ_PEND_IO_ISC_3:
1414	case IRQ_PEND_IO_ISC_4:
1415	case IRQ_PEND_IO_ISC_5:
1416	case IRQ_PEND_IO_ISC_6:
1417	case IRQ_PEND_IO_ISC_7:
1418	rc = __deliver_io(vcpu, irq_type);
1419	break;
1420	case IRQ_PEND_MCHK_EX:
1421	case IRQ_PEND_MCHK_REP:
1422	rc = __deliver_machine_check(vcpu);
1423	break;
1424	case IRQ_PEND_PROG:
1425	rc = __deliver_prog(vcpu);
1426	break;
1427	case IRQ_PEND_EXT_EMERGENCY:
1428	rc = __deliver_emergency_signal(vcpu);
1429	break;
1430	case IRQ_PEND_EXT_EXTERNAL:
1431	rc = __deliver_external_call(vcpu);
1432	break;
1433	case IRQ_PEND_EXT_CLOCK_COMP:
1434	rc = __deliver_ckc(vcpu);
1435	break;
1436	case IRQ_PEND_EXT_CPU_TIMER:
1437	rc = __deliver_cpu_timer(vcpu);
1438	break;
1439	case IRQ_PEND_RESTART:
1440	rc = __deliver_restart(vcpu);
1441	break;
1442	case IRQ_PEND_SET_PREFIX:
1443	rc = __deliver_set_prefix(vcpu);
1444	break;
1445	case IRQ_PEND_PFAULT_INIT:
1446	rc = __deliver_pfault_init(vcpu);
1447	break;
1448	case IRQ_PEND_EXT_SERVICE:
1449	rc = __deliver_service(vcpu);
1450	break;
1451	case IRQ_PEND_EXT_SERVICE_EV:
1452	rc = __deliver_service_ev(vcpu);
1453	break;
1454	case IRQ_PEND_PFAULT_DONE:
1455	rc = __deliver_pfault_done(vcpu);
1456	break;
1457	case IRQ_PEND_VIRTIO:
1458	rc = __deliver_virtio(vcpu);
1459	break;
1460	default:
1461	WARN_ONCE(1, "Unknown pending irq type %ld", irq_type);
1462	clear_bit(nr: irq_type, addr: &li->pending_irqs);
1463	}
1464	delivered |= !rc;
1465	}
1466
1467	/*
1468	* We delivered at least one interrupt and modified the PC. Force a
1469	* singlestep event now.
1470	*/
1471	if (delivered && guestdbg_sstep_enabled(vcpu)) {
1472	struct kvm_debug_exit_arch *debug_exit = &vcpu->run->debug.arch;
1473
1474	debug_exit->addr = vcpu->arch.sie_block->gpsw.addr;
1475	debug_exit->type = KVM_SINGLESTEP;
1476	vcpu->guest_debug |= KVM_GUESTDBG_EXIT_PENDING;
1477	}
1478
1479	set_intercept_indicators(vcpu);
1480
1481	return rc;
1482	}
1483
1484	static int __inject_prog(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1485	{
1486	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1487
1488	vcpu->stat.inject_program++;
1489	VCPU_EVENT(vcpu, 3, "inject: program irq code 0x%x", irq->u.pgm.code);
1490	trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_PROGRAM_INT,
1491	irq->u.pgm.code, 0);
1492
1493	if (!(irq->u.pgm.flags & KVM_S390_PGM_FLAGS_ILC_VALID)) {
1494	/* auto detection if no valid ILC was given */
1495	irq->u.pgm.flags &= ~KVM_S390_PGM_FLAGS_ILC_MASK;
1496	irq->u.pgm.flags |= kvm_s390_get_ilen(vcpu);
1497	irq->u.pgm.flags |= KVM_S390_PGM_FLAGS_ILC_VALID;
1498	}
1499
1500	if (irq->u.pgm.code == PGM_PER) {
1501	li->irq.pgm.code |= PGM_PER;
1502	li->irq.pgm.flags = irq->u.pgm.flags;
1503	/* only modify PER related information */
1504	li->irq.pgm.per_address = irq->u.pgm.per_address;
1505	li->irq.pgm.per_code = irq->u.pgm.per_code;
1506	li->irq.pgm.per_atmid = irq->u.pgm.per_atmid;
1507	li->irq.pgm.per_access_id = irq->u.pgm.per_access_id;
1508	} else if (!(irq->u.pgm.code & PGM_PER)) {
1509	li->irq.pgm.code = (li->irq.pgm.code & PGM_PER) |
1510	irq->u.pgm.code;
1511	li->irq.pgm.flags = irq->u.pgm.flags;
1512	/* only modify non-PER information */
1513	li->irq.pgm.trans_exc_code = irq->u.pgm.trans_exc_code;
1514	li->irq.pgm.mon_code = irq->u.pgm.mon_code;
1515	li->irq.pgm.data_exc_code = irq->u.pgm.data_exc_code;
1516	li->irq.pgm.mon_class_nr = irq->u.pgm.mon_class_nr;
1517	li->irq.pgm.exc_access_id = irq->u.pgm.exc_access_id;
1518	li->irq.pgm.op_access_id = irq->u.pgm.op_access_id;
1519	} else {
1520	li->irq.pgm = irq->u.pgm;
1521	}
1522	set_bit(IRQ_PEND_PROG, &li->pending_irqs);
1523	return 0;
1524	}
1525
1526	static int __inject_pfault_init(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1527	{
1528	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1529
1530	vcpu->stat.inject_pfault_init++;
1531	VCPU_EVENT(vcpu, 4, "inject: pfault init parameter block at 0x%llx",
1532	irq->u.ext.ext_params2);
1533	trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_PFAULT_INIT,
1534	irq->u.ext.ext_params,
1535	irq->u.ext.ext_params2);
1536
1537	li->irq.ext = irq->u.ext;
1538	set_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs);
1539	kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1540	return 0;
1541	}
1542
1543	static int __inject_extcall(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1544	{
1545	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1546	struct kvm_s390_extcall_info *extcall = &li->irq.extcall;
1547	uint16_t src_id = irq->u.extcall.code;
1548
1549	vcpu->stat.inject_external_call++;
1550	VCPU_EVENT(vcpu, 4, "inject: external call source-cpu:%u",
1551	src_id);
1552	trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_EXTERNAL_CALL,
1553	src_id, 0);
1554
1555	/* sending vcpu invalid */
1556	if (kvm_get_vcpu_by_id(kvm: vcpu->kvm, id: src_id) == NULL)
1557	return -EINVAL;
1558
1559	if (sclp.has_sigpif && !kvm_s390_pv_cpu_get_handle(vcpu))
1560	return sca_inject_ext_call(vcpu, src_id);
1561
1562	if (test_and_set_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs))
1563	return -EBUSY;
1564	*extcall = irq->u.extcall;
1565	kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1566	return 0;
1567	}
1568
1569	static int __inject_set_prefix(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1570	{
1571	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1572	struct kvm_s390_prefix_info *prefix = &li->irq.prefix;
1573
1574	vcpu->stat.inject_set_prefix++;
1575	VCPU_EVENT(vcpu, 3, "inject: set prefix to %x",
1576	irq->u.prefix.address);
1577	trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_SET_PREFIX,
1578	irq->u.prefix.address, 0);
1579
1580	if (!is_vcpu_stopped(vcpu))
1581	return -EBUSY;
1582
1583	*prefix = irq->u.prefix;
1584	set_bit(IRQ_PEND_SET_PREFIX, &li->pending_irqs);
1585	return 0;
1586	}
1587
1588	#define KVM_S390_STOP_SUPP_FLAGS (KVM_S390_STOP_FLAG_STORE_STATUS)
1589	static int __inject_sigp_stop(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1590	{
1591	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1592	struct kvm_s390_stop_info *stop = &li->irq.stop;
1593	int rc = 0;
1594
1595	vcpu->stat.inject_stop_signal++;
1596	trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_STOP, 0, 0);
1597
1598	if (irq->u.stop.flags & ~KVM_S390_STOP_SUPP_FLAGS)
1599	return -EINVAL;
1600
1601	if (is_vcpu_stopped(vcpu)) {
1602	if (irq->u.stop.flags & KVM_S390_STOP_FLAG_STORE_STATUS)
1603	rc = kvm_s390_store_status_unloaded(vcpu,
1604	KVM_S390_STORE_STATUS_NOADDR);
1605	return rc;
1606	}
1607
1608	if (test_and_set_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs))
1609	return -EBUSY;
1610	stop->flags = irq->u.stop.flags;
1611	kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
1612	return 0;
1613	}
1614
1615	static int __inject_sigp_restart(struct kvm_vcpu *vcpu)
1616	{
1617	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1618
1619	vcpu->stat.inject_restart++;
1620	VCPU_EVENT(vcpu, 3, "%s", "inject: restart int");
1621	trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_RESTART, 0, 0);
1622
1623	set_bit(IRQ_PEND_RESTART, &li->pending_irqs);
1624	return 0;
1625	}
1626
1627	static int __inject_sigp_emergency(struct kvm_vcpu *vcpu,
1628	struct kvm_s390_irq *irq)
1629	{
1630	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1631
1632	vcpu->stat.inject_emergency_signal++;
1633	VCPU_EVENT(vcpu, 4, "inject: emergency from cpu %u",
1634	irq->u.emerg.code);
1635	trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_EMERGENCY,
1636	irq->u.emerg.code, 0);
1637
1638	/* sending vcpu invalid */
1639	if (kvm_get_vcpu_by_id(kvm: vcpu->kvm, id: irq->u.emerg.code) == NULL)
1640	return -EINVAL;
1641
1642	set_bit(nr: irq->u.emerg.code, addr: li->sigp_emerg_pending);
1643	set_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs);
1644	kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1645	return 0;
1646	}
1647
1648	static int __inject_mchk(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1649	{
1650	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1651	struct kvm_s390_mchk_info *mchk = &li->irq.mchk;
1652
1653	vcpu->stat.inject_mchk++;
1654	VCPU_EVENT(vcpu, 3, "inject: machine check mcic 0x%llx",
1655	irq->u.mchk.mcic);
1656	trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_MCHK, 0,
1657	irq->u.mchk.mcic);
1658
1659	/*
1660	* Because repressible machine checks can be indicated along with
1661	* exigent machine checks (PoP, Chapter 11, Interruption action)
1662	* we need to combine cr14, mcic and external damage code.
1663	* Failing storage address and the logout area should not be or'ed
1664	* together, we just indicate the last occurrence of the corresponding
1665	* machine check
1666	*/
1667	mchk->cr14 |= irq->u.mchk.cr14;
1668	mchk->mcic |= irq->u.mchk.mcic;
1669	mchk->ext_damage_code |= irq->u.mchk.ext_damage_code;
1670	mchk->failing_storage_address = irq->u.mchk.failing_storage_address;
1671	memcpy(&mchk->fixed_logout, &irq->u.mchk.fixed_logout,
1672	sizeof(mchk->fixed_logout));
1673	if (mchk->mcic & MCHK_EX_MASK)
1674	set_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs);
1675	else if (mchk->mcic & MCHK_REP_MASK)
1676	set_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs);
1677	return 0;
1678	}
1679
1680	static int __inject_ckc(struct kvm_vcpu *vcpu)
1681	{
1682	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1683
1684	vcpu->stat.inject_ckc++;
1685	VCPU_EVENT(vcpu, 3, "%s", "inject: clock comparator external");
1686	trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_CLOCK_COMP,
1687	0, 0);
1688
1689	set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
1690	kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1691	return 0;
1692	}
1693
1694	static int __inject_cpu_timer(struct kvm_vcpu *vcpu)
1695	{
1696	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1697
1698	vcpu->stat.inject_cputm++;
1699	VCPU_EVENT(vcpu, 3, "%s", "inject: cpu timer external");
1700	trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_CPU_TIMER,
1701	0, 0);
1702
1703	set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
1704	kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1705	return 0;
1706	}
1707
1708	static struct kvm_s390_interrupt_info *get_io_int(struct kvm *kvm,
1709	int isc, u32 schid)
1710	{
1711	struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1712	struct list_head *isc_list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc];
1713	struct kvm_s390_interrupt_info *iter;
1714	u16 id = (schid & 0xffff0000U) >> 16;
1715	u16 nr = schid & 0x0000ffffU;
1716
1717	spin_lock(lock: &fi->lock);
1718	list_for_each_entry(iter, isc_list, list) {
1719	if (schid && (id != iter->io.subchannel_id ||
1720	nr != iter->io.subchannel_nr))
1721	continue;
1722	/* found an appropriate entry */
1723	list_del_init(entry: &iter->list);
1724	fi->counters[FIRQ_CNTR_IO] -= 1;
1725	if (list_empty(head: isc_list))
1726	clear_bit(nr: isc_to_irq_type(isc), addr: &fi->pending_irqs);
1727	spin_unlock(lock: &fi->lock);
1728	return iter;
1729	}
1730	spin_unlock(lock: &fi->lock);
1731	return NULL;
1732	}
1733
1734	static struct kvm_s390_interrupt_info *get_top_io_int(struct kvm *kvm,
1735	u64 isc_mask, u32 schid)
1736	{
1737	struct kvm_s390_interrupt_info *inti = NULL;
1738	int isc;
1739
1740	for (isc = 0; isc <= MAX_ISC && !inti; isc++) {
1741	if (isc_mask & isc_to_isc_bits(isc))
1742	inti = get_io_int(kvm, isc, schid);
1743	}
1744	return inti;
1745	}
1746
1747	static int get_top_gisa_isc(struct kvm *kvm, u64 isc_mask, u32 schid)
1748	{
1749	struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
1750	unsigned long active_mask;
1751	int isc;
1752
1753	if (schid)
1754	goto out;
1755	if (!gi->origin)
1756	goto out;
1757
1758	active_mask = (isc_mask & gisa_get_ipm(gisa: gi->origin) << 24) << 32;
1759	while (active_mask) {
1760	isc = __fls(word: active_mask) ^ (BITS_PER_LONG - 1);
1761	if (gisa_tac_ipm_gisc(gisa: gi->origin, gisc: isc))
1762	return isc;
1763	clear_bit_inv(isc, &active_mask);
1764	}
1765	out:
1766	return -EINVAL;
1767	}
1768
1769	/*
1770	* Dequeue and return an I/O interrupt matching any of the interruption
1771	* subclasses as designated by the isc mask in cr6 and the schid (if != 0).
1772	* Take into account the interrupts pending in the interrupt list and in GISA.
1773	*
1774	* Note that for a guest that does not enable I/O interrupts
1775	* but relies on TPI, a flood of classic interrupts may starve
1776	* out adapter interrupts on the same isc. Linux does not do
1777	* that, and it is possible to work around the issue by configuring
1778	* different iscs for classic and adapter interrupts in the guest,
1779	* but we may want to revisit this in the future.
1780	*/
1781	struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm,
1782	u64 isc_mask, u32 schid)
1783	{
1784	struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
1785	struct kvm_s390_interrupt_info *inti, *tmp_inti;
1786	int isc;
1787
1788	inti = get_top_io_int(kvm, isc_mask, schid);
1789
1790	isc = get_top_gisa_isc(kvm, isc_mask, schid);
1791	if (isc < 0)
1792	/* no AI in GISA */
1793	goto out;
1794
1795	if (!inti)
1796	/* AI in GISA but no classical IO int */
1797	goto gisa_out;
1798
1799	/* both types of interrupts present */
1800	if (int_word_to_isc(int_word: inti->io.io_int_word) <= isc) {
1801	/* classical IO int with higher priority */
1802	gisa_set_ipm_gisc(gisa: gi->origin, gisc: isc);
1803	goto out;
1804	}
1805	gisa_out:
1806	tmp_inti = kzalloc(sizeof(*inti), GFP_KERNEL_ACCOUNT);
1807	if (tmp_inti) {
1808	tmp_inti->type = KVM_S390_INT_IO(1, 0, 0, 0);
1809	tmp_inti->io.io_int_word = isc_to_int_word(isc);
1810	if (inti)
1811	kvm_s390_reinject_io_int(kvm, inti);
1812	inti = tmp_inti;
1813	} else
1814	gisa_set_ipm_gisc(gisa: gi->origin, gisc: isc);
1815	out:
1816	return inti;
1817	}
1818
1819	static int __inject_service(struct kvm *kvm,
1820	struct kvm_s390_interrupt_info *inti)
1821	{
1822	struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1823
1824	kvm->stat.inject_service_signal++;
1825	spin_lock(lock: &fi->lock);
1826	fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_EVENT_PENDING;
1827
1828	/* We always allow events, track them separately from the sccb ints */
1829	if (fi->srv_signal.ext_params & SCCB_EVENT_PENDING)
1830	set_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs);
1831
1832	/*
1833	* Early versions of the QEMU s390 bios will inject several
1834	* service interrupts after another without handling a
1835	* condition code indicating busy.
1836	* We will silently ignore those superfluous sccb values.
1837	* A future version of QEMU will take care of serialization
1838	* of servc requests
1839	*/
1840	if (fi->srv_signal.ext_params & SCCB_MASK)
1841	goto out;
1842	fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_MASK;
1843	set_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs);
1844	out:
1845	spin_unlock(lock: &fi->lock);
1846	kfree(objp: inti);
1847	return 0;
1848	}
1849
1850	static int __inject_virtio(struct kvm *kvm,
1851	struct kvm_s390_interrupt_info *inti)
1852	{
1853	struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1854
1855	kvm->stat.inject_virtio++;
1856	spin_lock(lock: &fi->lock);
1857	if (fi->counters[FIRQ_CNTR_VIRTIO] >= KVM_S390_MAX_VIRTIO_IRQS) {
1858	spin_unlock(lock: &fi->lock);
1859	return -EBUSY;
1860	}
1861	fi->counters[FIRQ_CNTR_VIRTIO] += 1;
1862	list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_VIRTIO]);
1863	set_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs);
1864	spin_unlock(lock: &fi->lock);
1865	return 0;
1866	}
1867
1868	static int __inject_pfault_done(struct kvm *kvm,
1869	struct kvm_s390_interrupt_info *inti)
1870	{
1871	struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1872
1873	kvm->stat.inject_pfault_done++;
1874	spin_lock(lock: &fi->lock);
1875	if (fi->counters[FIRQ_CNTR_PFAULT] >=
1876	(ASYNC_PF_PER_VCPU * KVM_MAX_VCPUS)) {
1877	spin_unlock(lock: &fi->lock);
1878	return -EBUSY;
1879	}
1880	fi->counters[FIRQ_CNTR_PFAULT] += 1;
1881	list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_PFAULT]);
1882	set_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs);
1883	spin_unlock(lock: &fi->lock);
1884	return 0;
1885	}
1886
1887	#define CR_PENDING_SUBCLASS 28
1888	static int __inject_float_mchk(struct kvm *kvm,
1889	struct kvm_s390_interrupt_info *inti)
1890	{
1891	struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1892
1893	kvm->stat.inject_float_mchk++;
1894	spin_lock(lock: &fi->lock);
1895	fi->mchk.cr14 |= inti->mchk.cr14 & (1UL << CR_PENDING_SUBCLASS);
1896	fi->mchk.mcic |= inti->mchk.mcic;
1897	set_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs);
1898	spin_unlock(lock: &fi->lock);
1899	kfree(objp: inti);
1900	return 0;
1901	}
1902
1903	static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
1904	{
1905	struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
1906	struct kvm_s390_float_interrupt *fi;
1907	struct list_head *list;
1908	int isc;
1909
1910	kvm->stat.inject_io++;
1911	isc = int_word_to_isc(int_word: inti->io.io_int_word);
1912
1913	/*
1914	* We do not use the lock checking variant as this is just a
1915	* performance optimization and we do not hold the lock here.
1916	* This is ok as the code will pick interrupts from both "lists"
1917	* for delivery.
1918	*/
1919	if (gi->origin && inti->type & KVM_S390_INT_IO_AI_MASK) {
1920	VM_EVENT(kvm, 4, "%s isc %1u", "inject: I/O (AI/gisa)", isc);
1921	gisa_set_ipm_gisc(gisa: gi->origin, gisc: isc);
1922	kfree(objp: inti);
1923	return 0;
1924	}
1925
1926	fi = &kvm->arch.float_int;
1927	spin_lock(lock: &fi->lock);
1928	if (fi->counters[FIRQ_CNTR_IO] >= KVM_S390_MAX_FLOAT_IRQS) {
1929	spin_unlock(lock: &fi->lock);
1930	return -EBUSY;
1931	}
1932	fi->counters[FIRQ_CNTR_IO] += 1;
1933
1934	if (inti->type & KVM_S390_INT_IO_AI_MASK)
1935	VM_EVENT(kvm, 4, "%s", "inject: I/O (AI)");
1936	else
1937	VM_EVENT(kvm, 4, "inject: I/O %x ss %x schid %04x",
1938	inti->io.subchannel_id >> 8,
1939	inti->io.subchannel_id >> 1 & 0x3,
1940	inti->io.subchannel_nr);
1941	list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc];
1942	list_add_tail(new: &inti->list, head: list);
1943	set_bit(nr: isc_to_irq_type(isc), addr: &fi->pending_irqs);
1944	spin_unlock(lock: &fi->lock);
1945	return 0;
1946	}
1947
1948	/*
1949	* Find a destination VCPU for a floating irq and kick it.
1950	*/
1951	static void __floating_irq_kick(struct kvm *kvm, u64 type)
1952	{
1953	struct kvm_vcpu *dst_vcpu;
1954	int sigcpu, online_vcpus, nr_tries = 0;
1955
1956	online_vcpus = atomic_read(v: &kvm->online_vcpus);
1957	if (!online_vcpus)
1958	return;
1959
1960	/* find idle VCPUs first, then round robin */
1961	sigcpu = find_first_bit(addr: kvm->arch.idle_mask, size: online_vcpus);
1962	if (sigcpu == online_vcpus) {
1963	do {
1964	sigcpu = kvm->arch.float_int.next_rr_cpu++;
1965	kvm->arch.float_int.next_rr_cpu %= online_vcpus;
1966	/* avoid endless loops if all vcpus are stopped */
1967	if (nr_tries++ >= online_vcpus)
1968	return;
1969	} while (is_vcpu_stopped(vcpu: kvm_get_vcpu(kvm, i: sigcpu)));
1970	}
1971	dst_vcpu = kvm_get_vcpu(kvm, i: sigcpu);
1972
1973	/* make the VCPU drop out of the SIE, or wake it up if sleeping */
1974	switch (type) {
1975	case KVM_S390_MCHK:
1976	kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_STOP_INT);
1977	break;
1978	case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
1979	if (!(type & KVM_S390_INT_IO_AI_MASK &&
1980	kvm->arch.gisa_int.origin) ||
1981	kvm_s390_pv_cpu_get_handle(dst_vcpu))
1982	kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_IO_INT);
1983	break;
1984	default:
1985	kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_EXT_INT);
1986	break;
1987	}
1988	kvm_s390_vcpu_wakeup(vcpu: dst_vcpu);
1989	}
1990
1991	static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
1992	{
1993	u64 type = READ_ONCE(inti->type);
1994	int rc;
1995
1996	switch (type) {
1997	case KVM_S390_MCHK:
1998	rc = __inject_float_mchk(kvm, inti);
1999	break;
2000	case KVM_S390_INT_VIRTIO:
2001	rc = __inject_virtio(kvm, inti);
2002	break;
2003	case KVM_S390_INT_SERVICE:
2004	rc = __inject_service(kvm, inti);
2005	break;
2006	case KVM_S390_INT_PFAULT_DONE:
2007	rc = __inject_pfault_done(kvm, inti);
2008	break;
2009	case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2010	rc = __inject_io(kvm, inti);
2011	break;
2012	default:
2013	rc = -EINVAL;
2014	}
2015	if (rc)
2016	return rc;
2017
2018	__floating_irq_kick(kvm, type);
2019	return 0;
2020	}
2021
2022	int kvm_s390_inject_vm(struct kvm *kvm,
2023	struct kvm_s390_interrupt *s390int)
2024	{
2025	struct kvm_s390_interrupt_info *inti;
2026	int rc;
2027
2028	inti = kzalloc(sizeof(*inti), GFP_KERNEL_ACCOUNT);
2029	if (!inti)
2030	return -ENOMEM;
2031
2032	inti->type = s390int->type;
2033	switch (inti->type) {
2034	case KVM_S390_INT_VIRTIO:
2035	VM_EVENT(kvm, 5, "inject: virtio parm:%x,parm64:%llx",
2036	s390int->parm, s390int->parm64);
2037	inti->ext.ext_params = s390int->parm;
2038	inti->ext.ext_params2 = s390int->parm64;
2039	break;
2040	case KVM_S390_INT_SERVICE:
2041	VM_EVENT(kvm, 4, "inject: sclp parm:%x", s390int->parm);
2042	inti->ext.ext_params = s390int->parm;
2043	break;
2044	case KVM_S390_INT_PFAULT_DONE:
2045	inti->ext.ext_params2 = s390int->parm64;
2046	break;
2047	case KVM_S390_MCHK:
2048	VM_EVENT(kvm, 3, "inject: machine check mcic 0x%llx",
2049	s390int->parm64);
2050	inti->mchk.cr14 = s390int->parm; /* upper bits are not used */
2051	inti->mchk.mcic = s390int->parm64;
2052	break;
2053	case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2054	inti->io.subchannel_id = s390int->parm >> 16;
2055	inti->io.subchannel_nr = s390int->parm & 0x0000ffffu;
2056	inti->io.io_int_parm = s390int->parm64 >> 32;
2057	inti->io.io_int_word = s390int->parm64 & 0x00000000ffffffffull;
2058	break;
2059	default:
2060	kfree(objp: inti);
2061	return -EINVAL;
2062	}
2063	trace_kvm_s390_inject_vm(type: s390int->type, parm: s390int->parm, parm64: s390int->parm64,
2064	who: 2);
2065
2066	rc = __inject_vm(kvm, inti);
2067	if (rc)
2068	kfree(objp: inti);
2069	return rc;
2070	}
2071
2072	int kvm_s390_reinject_io_int(struct kvm *kvm,
2073	struct kvm_s390_interrupt_info *inti)
2074	{
2075	return __inject_vm(kvm, inti);
2076	}
2077
2078	int s390int_to_s390irq(struct kvm_s390_interrupt *s390int,
2079	struct kvm_s390_irq *irq)
2080	{
2081	irq->type = s390int->type;
2082	switch (irq->type) {
2083	case KVM_S390_PROGRAM_INT:
2084	if (s390int->parm & 0xffff0000)
2085	return -EINVAL;
2086	irq->u.pgm.code = s390int->parm;
2087	break;
2088	case KVM_S390_SIGP_SET_PREFIX:
2089	irq->u.prefix.address = s390int->parm;
2090	break;
2091	case KVM_S390_SIGP_STOP:
2092	irq->u.stop.flags = s390int->parm;
2093	break;
2094	case KVM_S390_INT_EXTERNAL_CALL:
2095	if (s390int->parm & 0xffff0000)
2096	return -EINVAL;
2097	irq->u.extcall.code = s390int->parm;
2098	break;
2099	case KVM_S390_INT_EMERGENCY:
2100	if (s390int->parm & 0xffff0000)
2101	return -EINVAL;
2102	irq->u.emerg.code = s390int->parm;
2103	break;
2104	case KVM_S390_MCHK:
2105	irq->u.mchk.mcic = s390int->parm64;
2106	break;
2107	case KVM_S390_INT_PFAULT_INIT:
2108	irq->u.ext.ext_params = s390int->parm;
2109	irq->u.ext.ext_params2 = s390int->parm64;
2110	break;
2111	case KVM_S390_RESTART:
2112	case KVM_S390_INT_CLOCK_COMP:
2113	case KVM_S390_INT_CPU_TIMER:
2114	break;
2115	default:
2116	return -EINVAL;
2117	}
2118	return 0;
2119	}
2120
2121	int kvm_s390_is_stop_irq_pending(struct kvm_vcpu *vcpu)
2122	{
2123	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2124
2125	return test_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs);
2126	}
2127
2128	int kvm_s390_is_restart_irq_pending(struct kvm_vcpu *vcpu)
2129	{
2130	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2131
2132	return test_bit(IRQ_PEND_RESTART, &li->pending_irqs);
2133	}
2134
2135	void kvm_s390_clear_stop_irq(struct kvm_vcpu *vcpu)
2136	{
2137	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2138
2139	spin_lock(lock: &li->lock);
2140	li->irq.stop.flags = 0;
2141	clear_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs);
2142	spin_unlock(lock: &li->lock);
2143	}
2144
2145	static int do_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
2146	{
2147	int rc;
2148
2149	switch (irq->type) {
2150	case KVM_S390_PROGRAM_INT:
2151	rc = __inject_prog(vcpu, irq);
2152	break;
2153	case KVM_S390_SIGP_SET_PREFIX:
2154	rc = __inject_set_prefix(vcpu, irq);
2155	break;
2156	case KVM_S390_SIGP_STOP:
2157	rc = __inject_sigp_stop(vcpu, irq);
2158	break;
2159	case KVM_S390_RESTART:
2160	rc = __inject_sigp_restart(vcpu);
2161	break;
2162	case KVM_S390_INT_CLOCK_COMP:
2163	rc = __inject_ckc(vcpu);
2164	break;
2165	case KVM_S390_INT_CPU_TIMER:
2166	rc = __inject_cpu_timer(vcpu);
2167	break;
2168	case KVM_S390_INT_EXTERNAL_CALL:
2169	rc = __inject_extcall(vcpu, irq);
2170	break;
2171	case KVM_S390_INT_EMERGENCY:
2172	rc = __inject_sigp_emergency(vcpu, irq);
2173	break;
2174	case KVM_S390_MCHK:
2175	rc = __inject_mchk(vcpu, irq);
2176	break;
2177	case KVM_S390_INT_PFAULT_INIT:
2178	rc = __inject_pfault_init(vcpu, irq);
2179	break;
2180	case KVM_S390_INT_VIRTIO:
2181	case KVM_S390_INT_SERVICE:
2182	case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2183	default:
2184	rc = -EINVAL;
2185	}
2186
2187	return rc;
2188	}
2189
2190	int kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
2191	{
2192	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2193	int rc;
2194
2195	spin_lock(lock: &li->lock);
2196	rc = do_inject_vcpu(vcpu, irq);
2197	spin_unlock(lock: &li->lock);
2198	if (!rc)
2199	kvm_s390_vcpu_wakeup(vcpu);
2200	return rc;
2201	}
2202
2203	static inline void clear_irq_list(struct list_head *_list)
2204	{
2205	struct kvm_s390_interrupt_info *inti, *n;
2206
2207	list_for_each_entry_safe(inti, n, _list, list) {
2208	list_del(entry: &inti->list);
2209	kfree(objp: inti);
2210	}
2211	}
2212
2213	static void inti_to_irq(struct kvm_s390_interrupt_info *inti,
2214	struct kvm_s390_irq *irq)
2215	{
2216	irq->type = inti->type;
2217	switch (inti->type) {
2218	case KVM_S390_INT_PFAULT_INIT:
2219	case KVM_S390_INT_PFAULT_DONE:
2220	case KVM_S390_INT_VIRTIO:
2221	irq->u.ext = inti->ext;
2222	break;
2223	case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2224	irq->u.io = inti->io;
2225	break;
2226	}
2227	}
2228
2229	void kvm_s390_clear_float_irqs(struct kvm *kvm)
2230	{
2231	struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2232	int i;
2233
2234	mutex_lock(&kvm->lock);
2235	if (!kvm_s390_pv_is_protected(kvm))
2236	fi->masked_irqs = 0;
2237	mutex_unlock(lock: &kvm->lock);
2238	spin_lock(lock: &fi->lock);
2239	fi->pending_irqs = 0;
2240	memset(&fi->srv_signal, 0, sizeof(fi->srv_signal));
2241	memset(&fi->mchk, 0, sizeof(fi->mchk));
2242	for (i = 0; i < FIRQ_LIST_COUNT; i++)
2243	clear_irq_list(&fi->lists[i]);
2244	for (i = 0; i < FIRQ_MAX_COUNT; i++)
2245	fi->counters[i] = 0;
2246	spin_unlock(lock: &fi->lock);
2247	kvm_s390_gisa_clear(kvm);
2248	};
2249
2250	static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len)
2251	{
2252	struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
2253	struct kvm_s390_interrupt_info *inti;
2254	struct kvm_s390_float_interrupt *fi;
2255	struct kvm_s390_irq *buf;
2256	struct kvm_s390_irq *irq;
2257	int max_irqs;
2258	int ret = 0;
2259	int n = 0;
2260	int i;
2261
2262	if (len > KVM_S390_FLIC_MAX_BUFFER || len == 0)
2263	return -EINVAL;
2264
2265	/*
2266	* We are already using -ENOMEM to signal
2267	* userspace it may retry with a bigger buffer,
2268	* so we need to use something else for this case
2269	*/
2270	buf = vzalloc(size: len);
2271	if (!buf)
2272	return -ENOBUFS;
2273
2274	max_irqs = len / sizeof(struct kvm_s390_irq);
2275
2276	if (gi->origin && gisa_get_ipm(gisa: gi->origin)) {
2277	for (i = 0; i <= MAX_ISC; i++) {
2278	if (n == max_irqs) {
2279	/* signal userspace to try again */
2280	ret = -ENOMEM;
2281	goto out_nolock;
2282	}
2283	if (gisa_tac_ipm_gisc(gi->origin, i)) {
2284	irq = (struct kvm_s390_irq *) &buf[n];
2285	irq->type = KVM_S390_INT_IO(1, 0, 0, 0);
2286	irq->u.io.io_int_word = isc_to_int_word(i);
2287	n++;
2288	}
2289	}
2290	}
2291	fi = &kvm->arch.float_int;
2292	spin_lock(lock: &fi->lock);
2293	for (i = 0; i < FIRQ_LIST_COUNT; i++) {
2294	list_for_each_entry(inti, &fi->lists[i], list) {
2295	if (n == max_irqs) {
2296	/* signal userspace to try again */
2297	ret = -ENOMEM;
2298	goto out;
2299	}
2300	inti_to_irq(inti, &buf[n]);
2301	n++;
2302	}
2303	}
2304	if (test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs) ||
2305	test_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs)) {
2306	if (n == max_irqs) {
2307	/* signal userspace to try again */
2308	ret = -ENOMEM;
2309	goto out;
2310	}
2311	irq = (struct kvm_s390_irq *) &buf[n];
2312	irq->type = KVM_S390_INT_SERVICE;
2313	irq->u.ext = fi->srv_signal;
2314	n++;
2315	}
2316	if (test_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) {
2317	if (n == max_irqs) {
2318	/* signal userspace to try again */
2319	ret = -ENOMEM;
2320	goto out;
2321	}
2322	irq = (struct kvm_s390_irq *) &buf[n];
2323	irq->type = KVM_S390_MCHK;
2324	irq->u.mchk = fi->mchk;
2325	n++;
2326	}
2327
2328	out:
2329	spin_unlock(lock: &fi->lock);
2330	out_nolock:
2331	if (!ret && n > 0) {
2332	if (copy_to_user(usrbuf, buf, sizeof(struct kvm_s390_irq) * n))
2333	ret = -EFAULT;
2334	}
2335	vfree(addr: buf);
2336
2337	return ret < 0 ? ret : n;
2338	}
2339
2340	static int flic_ais_mode_get_all(struct kvm *kvm, struct kvm_device_attr *attr)
2341	{
2342	struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2343	struct kvm_s390_ais_all ais;
2344
2345	if (attr->attr < sizeof(ais))
2346	return -EINVAL;
2347
2348	if (!test_kvm_facility(kvm, nr: 72))
2349	return -EOPNOTSUPP;
2350
2351	mutex_lock(&fi->ais_lock);
2352	ais.simm = fi->simm;
2353	ais.nimm = fi->nimm;
2354	mutex_unlock(lock: &fi->ais_lock);
2355
2356	if (copy_to_user(to: (void __user *)attr->addr, from: &ais, n: sizeof(ais)))
2357	return -EFAULT;
2358
2359	return 0;
2360	}
2361
2362	static int flic_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2363	{
2364	int r;
2365
2366	switch (attr->group) {
2367	case KVM_DEV_FLIC_GET_ALL_IRQS:
2368	r = get_all_floating_irqs(kvm: dev->kvm, usrbuf: (u8 __user *) attr->addr,
2369	len: attr->attr);
2370	break;
2371	case KVM_DEV_FLIC_AISM_ALL:
2372	r = flic_ais_mode_get_all(kvm: dev->kvm, attr);
2373	break;
2374	default:
2375	r = -EINVAL;
2376	}
2377
2378	return r;
2379	}
2380
2381	static inline int copy_irq_from_user(struct kvm_s390_interrupt_info *inti,
2382	u64 addr)
2383	{
2384	struct kvm_s390_irq __user *uptr = (struct kvm_s390_irq __user *) addr;
2385	void *target = NULL;
2386	void __user *source;
2387	u64 size;
2388
2389	if (get_user(inti->type, (u64 __user *)addr))
2390	return -EFAULT;
2391
2392	switch (inti->type) {
2393	case KVM_S390_INT_PFAULT_INIT:
2394	case KVM_S390_INT_PFAULT_DONE:
2395	case KVM_S390_INT_VIRTIO:
2396	case KVM_S390_INT_SERVICE:
2397	target = (void *) &inti->ext;
2398	source = &uptr->u.ext;
2399	size = sizeof(inti->ext);
2400	break;
2401	case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2402	target = (void *) &inti->io;
2403	source = &uptr->u.io;
2404	size = sizeof(inti->io);
2405	break;
2406	case KVM_S390_MCHK:
2407	target = (void *) &inti->mchk;
2408	source = &uptr->u.mchk;
2409	size = sizeof(inti->mchk);
2410	break;
2411	default:
2412	return -EINVAL;
2413	}
2414
2415	if (copy_from_user(to: target, from: source, n: size))
2416	return -EFAULT;
2417
2418	return 0;
2419	}
2420
2421	static int enqueue_floating_irq(struct kvm_device *dev,
2422	struct kvm_device_attr *attr)
2423	{
2424	struct kvm_s390_interrupt_info *inti = NULL;
2425	int r = 0;
2426	int len = attr->attr;
2427
2428	if (len % sizeof(struct kvm_s390_irq) != 0)
2429	return -EINVAL;
2430	else if (len > KVM_S390_FLIC_MAX_BUFFER)
2431	return -EINVAL;
2432
2433	while (len >= sizeof(struct kvm_s390_irq)) {
2434	inti = kzalloc(sizeof(*inti), GFP_KERNEL_ACCOUNT);
2435	if (!inti)
2436	return -ENOMEM;
2437
2438	r = copy_irq_from_user(inti, addr: attr->addr);
2439	if (r) {
2440	kfree(objp: inti);
2441	return r;
2442	}
2443	r = __inject_vm(kvm: dev->kvm, inti);
2444	if (r) {
2445	kfree(objp: inti);
2446	return r;
2447	}
2448	len -= sizeof(struct kvm_s390_irq);
2449	attr->addr += sizeof(struct kvm_s390_irq);
2450	}
2451
2452	return r;
2453	}
2454
2455	static struct s390_io_adapter *get_io_adapter(struct kvm *kvm, unsigned int id)
2456	{
2457	if (id >= MAX_S390_IO_ADAPTERS)
2458	return NULL;
2459	id = array_index_nospec(id, MAX_S390_IO_ADAPTERS);
2460	return kvm->arch.adapters[id];
2461	}
2462
2463	static int register_io_adapter(struct kvm_device *dev,
2464	struct kvm_device_attr *attr)
2465	{
2466	struct s390_io_adapter *adapter;
2467	struct kvm_s390_io_adapter adapter_info;
2468
2469	if (copy_from_user(to: &adapter_info,
2470	from: (void __user *)attr->addr, n: sizeof(adapter_info)))
2471	return -EFAULT;
2472
2473	if (adapter_info.id >= MAX_S390_IO_ADAPTERS)
2474	return -EINVAL;
2475
2476	adapter_info.id = array_index_nospec(adapter_info.id,
2477	MAX_S390_IO_ADAPTERS);
2478
2479	if (dev->kvm->arch.adapters[adapter_info.id] != NULL)
2480	return -EINVAL;
2481
2482	adapter = kzalloc(sizeof(*adapter), GFP_KERNEL_ACCOUNT);
2483	if (!adapter)
2484	return -ENOMEM;
2485
2486	adapter->id = adapter_info.id;
2487	adapter->isc = adapter_info.isc;
2488	adapter->maskable = adapter_info.maskable;
2489	adapter->masked = false;
2490	adapter->swap = adapter_info.swap;
2491	adapter->suppressible = (adapter_info.flags) &
2492	KVM_S390_ADAPTER_SUPPRESSIBLE;
2493	dev->kvm->arch.adapters[adapter->id] = adapter;
2494
2495	return 0;
2496	}
2497
2498	int kvm_s390_mask_adapter(struct kvm *kvm, unsigned int id, bool masked)
2499	{
2500	int ret;
2501	struct s390_io_adapter *adapter = get_io_adapter(kvm, id);
2502
2503	if (!adapter || !adapter->maskable)
2504	return -EINVAL;
2505	ret = adapter->masked;
2506	adapter->masked = masked;
2507	return ret;
2508	}
2509
2510	void kvm_s390_destroy_adapters(struct kvm *kvm)
2511	{
2512	int i;
2513
2514	for (i = 0; i < MAX_S390_IO_ADAPTERS; i++)
2515	kfree(kvm->arch.adapters[i]);
2516	}
2517
2518	static int modify_io_adapter(struct kvm_device *dev,
2519	struct kvm_device_attr *attr)
2520	{
2521	struct kvm_s390_io_adapter_req req;
2522	struct s390_io_adapter *adapter;
2523	int ret;
2524
2525	if (copy_from_user(to: &req, from: (void __user *)attr->addr, n: sizeof(req)))
2526	return -EFAULT;
2527
2528	adapter = get_io_adapter(kvm: dev->kvm, id: req.id);
2529	if (!adapter)
2530	return -EINVAL;
2531	switch (req.type) {
2532	case KVM_S390_IO_ADAPTER_MASK:
2533	ret = kvm_s390_mask_adapter(kvm: dev->kvm, id: req.id, masked: req.mask);
2534	if (ret > 0)
2535	ret = 0;
2536	break;
2537	/*
2538	* The following operations are no longer needed and therefore no-ops.
2539	* The gpa to hva translation is done when an IRQ route is set up. The
2540	* set_irq code uses get_user_pages_remote() to do the actual write.
2541	*/
2542	case KVM_S390_IO_ADAPTER_MAP:
2543	case KVM_S390_IO_ADAPTER_UNMAP:
2544	ret = 0;
2545	break;
2546	default:
2547	ret = -EINVAL;
2548	}
2549
2550	return ret;
2551	}
2552
2553	static int clear_io_irq(struct kvm *kvm, struct kvm_device_attr *attr)
2554
2555	{
2556	const u64 isc_mask = 0xffUL << 24; /* all iscs set */
2557	u32 schid;
2558
2559	if (attr->flags)
2560	return -EINVAL;
2561	if (attr->attr != sizeof(schid))
2562	return -EINVAL;
2563	if (copy_from_user(to: &schid, from: (void __user *) attr->addr, n: sizeof(schid)))
2564	return -EFAULT;
2565	if (!schid)
2566	return -EINVAL;
2567	kfree(objp: kvm_s390_get_io_int(kvm, isc_mask, schid));
2568	/*
2569	* If userspace is conforming to the architecture, we can have at most
2570	* one pending I/O interrupt per subchannel, so this is effectively a
2571	* clear all.
2572	*/
2573	return 0;
2574	}
2575
2576	static int modify_ais_mode(struct kvm *kvm, struct kvm_device_attr *attr)
2577	{
2578	struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2579	struct kvm_s390_ais_req req;
2580	int ret = 0;
2581
2582	if (!test_kvm_facility(kvm, nr: 72))
2583	return -EOPNOTSUPP;
2584
2585	if (copy_from_user(to: &req, from: (void __user *)attr->addr, n: sizeof(req)))
2586	return -EFAULT;
2587
2588	if (req.isc > MAX_ISC)
2589	return -EINVAL;
2590
2591	trace_kvm_s390_modify_ais_mode(req.isc,
2592	(fi->simm & AIS_MODE_MASK(req.isc)) ?
2593	(fi->nimm & AIS_MODE_MASK(req.isc)) ?
2594	2 : KVM_S390_AIS_MODE_SINGLE :
2595	KVM_S390_AIS_MODE_ALL, req.mode);
2596
2597	mutex_lock(&fi->ais_lock);
2598	switch (req.mode) {
2599	case KVM_S390_AIS_MODE_ALL:
2600	fi->simm &= ~AIS_MODE_MASK(req.isc);
2601	fi->nimm &= ~AIS_MODE_MASK(req.isc);
2602	break;
2603	case KVM_S390_AIS_MODE_SINGLE:
2604	fi->simm |= AIS_MODE_MASK(req.isc);
2605	fi->nimm &= ~AIS_MODE_MASK(req.isc);
2606	break;
2607	default:
2608	ret = -EINVAL;
2609	}
2610	mutex_unlock(lock: &fi->ais_lock);
2611
2612	return ret;
2613	}
2614
2615	static int kvm_s390_inject_airq(struct kvm *kvm,
2616	struct s390_io_adapter *adapter)
2617	{
2618	struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2619	struct kvm_s390_interrupt s390int = {
2620	.type = KVM_S390_INT_IO(1, 0, 0, 0),
2621	.parm = 0,
2622	.parm64 = isc_to_int_word(adapter->isc),
2623	};
2624	int ret = 0;
2625
2626	if (!test_kvm_facility(kvm, nr: 72) || !adapter->suppressible)
2627	return kvm_s390_inject_vm(kvm, s390int: &s390int);
2628
2629	mutex_lock(&fi->ais_lock);
2630	if (fi->nimm & AIS_MODE_MASK(adapter->isc)) {
2631	trace_kvm_s390_airq_suppressed(id: adapter->id, isc: adapter->isc);
2632	goto out;
2633	}
2634
2635	ret = kvm_s390_inject_vm(kvm, s390int: &s390int);
2636	if (!ret && (fi->simm & AIS_MODE_MASK(adapter->isc))) {
2637	fi->nimm |= AIS_MODE_MASK(adapter->isc);
2638	trace_kvm_s390_modify_ais_mode(adapter->isc,
2639	KVM_S390_AIS_MODE_SINGLE, 2);
2640	}
2641	out:
2642	mutex_unlock(lock: &fi->ais_lock);
2643	return ret;
2644	}
2645
2646	static int flic_inject_airq(struct kvm *kvm, struct kvm_device_attr *attr)
2647	{
2648	unsigned int id = attr->attr;
2649	struct s390_io_adapter *adapter = get_io_adapter(kvm, id);
2650
2651	if (!adapter)
2652	return -EINVAL;
2653
2654	return kvm_s390_inject_airq(kvm, adapter);
2655	}
2656
2657	static int flic_ais_mode_set_all(struct kvm *kvm, struct kvm_device_attr *attr)
2658	{
2659	struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2660	struct kvm_s390_ais_all ais;
2661
2662	if (!test_kvm_facility(kvm, nr: 72))
2663	return -EOPNOTSUPP;
2664
2665	if (copy_from_user(to: &ais, from: (void __user *)attr->addr, n: sizeof(ais)))
2666	return -EFAULT;
2667
2668	mutex_lock(&fi->ais_lock);
2669	fi->simm = ais.simm;
2670	fi->nimm = ais.nimm;
2671	mutex_unlock(lock: &fi->ais_lock);
2672
2673	return 0;
2674	}
2675
2676	static int flic_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2677	{
2678	int r = 0;
2679	unsigned long i;
2680	struct kvm_vcpu *vcpu;
2681
2682	switch (attr->group) {
2683	case KVM_DEV_FLIC_ENQUEUE:
2684	r = enqueue_floating_irq(dev, attr);
2685	break;
2686	case KVM_DEV_FLIC_CLEAR_IRQS:
2687	kvm_s390_clear_float_irqs(kvm: dev->kvm);
2688	break;
2689	case KVM_DEV_FLIC_APF_ENABLE:
2690	dev->kvm->arch.gmap->pfault_enabled = 1;
2691	break;
2692	case KVM_DEV_FLIC_APF_DISABLE_WAIT:
2693	dev->kvm->arch.gmap->pfault_enabled = 0;
2694	/*
2695	* Make sure no async faults are in transition when
2696	* clearing the queues. So we don't need to worry
2697	* about late coming workers.
2698	*/
2699	synchronize_srcu(ssp: &dev->kvm->srcu);
2700	kvm_for_each_vcpu(i, vcpu, dev->kvm)
2701	kvm_clear_async_pf_completion_queue(vcpu);
2702	break;
2703	case KVM_DEV_FLIC_ADAPTER_REGISTER:
2704	r = register_io_adapter(dev, attr);
2705	break;
2706	case KVM_DEV_FLIC_ADAPTER_MODIFY:
2707	r = modify_io_adapter(dev, attr);
2708	break;
2709	case KVM_DEV_FLIC_CLEAR_IO_IRQ:
2710	r = clear_io_irq(kvm: dev->kvm, attr);
2711	break;
2712	case KVM_DEV_FLIC_AISM:
2713	r = modify_ais_mode(kvm: dev->kvm, attr);
2714	break;
2715	case KVM_DEV_FLIC_AIRQ_INJECT:
2716	r = flic_inject_airq(kvm: dev->kvm, attr);
2717	break;
2718	case KVM_DEV_FLIC_AISM_ALL:
2719	r = flic_ais_mode_set_all(kvm: dev->kvm, attr);
2720	break;
2721	default:
2722	r = -EINVAL;
2723	}
2724
2725	return r;
2726	}
2727
2728	static int flic_has_attr(struct kvm_device *dev,
2729	struct kvm_device_attr *attr)
2730	{
2731	switch (attr->group) {
2732	case KVM_DEV_FLIC_GET_ALL_IRQS:
2733	case KVM_DEV_FLIC_ENQUEUE:
2734	case KVM_DEV_FLIC_CLEAR_IRQS:
2735	case KVM_DEV_FLIC_APF_ENABLE:
2736	case KVM_DEV_FLIC_APF_DISABLE_WAIT:
2737	case KVM_DEV_FLIC_ADAPTER_REGISTER:
2738	case KVM_DEV_FLIC_ADAPTER_MODIFY:
2739	case KVM_DEV_FLIC_CLEAR_IO_IRQ:
2740	case KVM_DEV_FLIC_AISM:
2741	case KVM_DEV_FLIC_AIRQ_INJECT:
2742	case KVM_DEV_FLIC_AISM_ALL:
2743	return 0;
2744	}
2745	return -ENXIO;
2746	}
2747
2748	static int flic_create(struct kvm_device *dev, u32 type)
2749	{
2750	if (!dev)
2751	return -EINVAL;
2752	if (dev->kvm->arch.flic)
2753	return -EINVAL;
2754	dev->kvm->arch.flic = dev;
2755	return 0;
2756	}
2757
2758	static void flic_destroy(struct kvm_device *dev)
2759	{
2760	dev->kvm->arch.flic = NULL;
2761	kfree(objp: dev);
2762	}
2763
2764	/* s390 floating irq controller (flic) */
2765	struct kvm_device_ops kvm_flic_ops = {
2766	.name = "kvm-flic",
2767	.get_attr = flic_get_attr,
2768	.set_attr = flic_set_attr,
2769	.has_attr = flic_has_attr,
2770	.create = flic_create,
2771	.destroy = flic_destroy,
2772	};
2773
2774	static unsigned long get_ind_bit(__u64 addr, unsigned long bit_nr, bool swap)
2775	{
2776	unsigned long bit;
2777
2778	bit = bit_nr + (addr % PAGE_SIZE) * 8;
2779
2780	return swap ? (bit ^ (BITS_PER_LONG - 1)) : bit;
2781	}
2782
2783	static struct page *get_map_page(struct kvm *kvm, u64 uaddr)
2784	{
2785	struct page *page = NULL;
2786
2787	mmap_read_lock(mm: kvm->mm);
2788	get_user_pages_remote(mm: kvm->mm, start: uaddr, nr_pages: 1, gup_flags: FOLL_WRITE,
2789	pages: &page, NULL);
2790	mmap_read_unlock(mm: kvm->mm);
2791	return page;
2792	}
2793
2794	static int adapter_indicators_set(struct kvm *kvm,
2795	struct s390_io_adapter *adapter,
2796	struct kvm_s390_adapter_int *adapter_int)
2797	{
2798	unsigned long bit;
2799	int summary_set, idx;
2800	struct page *ind_page, *summary_page;
2801	void *map;
2802
2803	ind_page = get_map_page(kvm, uaddr: adapter_int->ind_addr);
2804	if (!ind_page)
2805	return -1;
2806	summary_page = get_map_page(kvm, uaddr: adapter_int->summary_addr);
2807	if (!summary_page) {
2808	put_page(page: ind_page);
2809	return -1;
2810	}
2811
2812	idx = srcu_read_lock(ssp: &kvm->srcu);
2813	map = page_address(ind_page);
2814	bit = get_ind_bit(addr: adapter_int->ind_addr,
2815	bit_nr: adapter_int->ind_offset, swap: adapter->swap);
2816	set_bit(nr: bit, addr: map);
2817	mark_page_dirty(kvm, gfn: adapter_int->ind_addr >> PAGE_SHIFT);
2818	set_page_dirty_lock(ind_page);
2819	map = page_address(summary_page);
2820	bit = get_ind_bit(addr: adapter_int->summary_addr,
2821	bit_nr: adapter_int->summary_offset, swap: adapter->swap);
2822	summary_set = test_and_set_bit(nr: bit, addr: map);
2823	mark_page_dirty(kvm, gfn: adapter_int->summary_addr >> PAGE_SHIFT);
2824	set_page_dirty_lock(summary_page);
2825	srcu_read_unlock(ssp: &kvm->srcu, idx);
2826
2827	put_page(page: ind_page);
2828	put_page(page: summary_page);
2829	return summary_set ? 0 : 1;
2830	}
2831
2832	/*
2833	* < 0 - not injected due to error
2834	* = 0 - coalesced, summary indicator already active
2835	* > 0 - injected interrupt
2836	*/
2837	static int set_adapter_int(struct kvm_kernel_irq_routing_entry *e,
2838	struct kvm *kvm, int irq_source_id, int level,
2839	bool line_status)
2840	{
2841	int ret;
2842	struct s390_io_adapter *adapter;
2843
2844	/* We're only interested in the 0->1 transition. */
2845	if (!level)
2846	return 0;
2847	adapter = get_io_adapter(kvm, id: e->adapter.adapter_id);
2848	if (!adapter)
2849	return -1;
2850	ret = adapter_indicators_set(kvm, adapter, adapter_int: &e->adapter);
2851	if ((ret > 0) && !adapter->masked) {
2852	ret = kvm_s390_inject_airq(kvm, adapter);
2853	if (ret == 0)
2854	ret = 1;
2855	}
2856	return ret;
2857	}
2858
2859	/*
2860	* Inject the machine check to the guest.
2861	*/
2862	void kvm_s390_reinject_machine_check(struct kvm_vcpu *vcpu,
2863	struct mcck_volatile_info *mcck_info)
2864	{
2865	struct kvm_s390_interrupt_info inti;
2866	struct kvm_s390_irq irq;
2867	struct kvm_s390_mchk_info *mchk;
2868	union mci mci;
2869	__u64 cr14 = 0; /* upper bits are not used */
2870	int rc;
2871
2872	mci.val = mcck_info->mcic;
2873	if (mci.sr)
2874	cr14 |= CR14_RECOVERY_SUBMASK;
2875	if (mci.dg)
2876	cr14 |= CR14_DEGRADATION_SUBMASK;
2877	if (mci.w)
2878	cr14 |= CR14_WARNING_SUBMASK;
2879
2880	mchk = mci.ck ? &inti.mchk : &irq.u.mchk;
2881	mchk->cr14 = cr14;
2882	mchk->mcic = mcck_info->mcic;
2883	mchk->ext_damage_code = mcck_info->ext_damage_code;
2884	mchk->failing_storage_address = mcck_info->failing_storage_address;
2885	if (mci.ck) {
2886	/* Inject the floating machine check */
2887	inti.type = KVM_S390_MCHK;
2888	rc = __inject_vm(kvm: vcpu->kvm, inti: &inti);
2889	} else {
2890	/* Inject the machine check to specified vcpu */
2891	irq.type = KVM_S390_MCHK;
2892	rc = kvm_s390_inject_vcpu(vcpu, irq: &irq);
2893	}
2894	WARN_ON_ONCE(rc);
2895	}
2896
2897	int kvm_set_routing_entry(struct kvm *kvm,
2898	struct kvm_kernel_irq_routing_entry *e,
2899	const struct kvm_irq_routing_entry *ue)
2900	{
2901	u64 uaddr;
2902
2903	switch (ue->type) {
2904	/* we store the userspace addresses instead of the guest addresses */
2905	case KVM_IRQ_ROUTING_S390_ADAPTER:
2906	e->set = set_adapter_int;
2907	uaddr = gmap_translate(kvm->arch.gmap, ue->u.adapter.summary_addr);
2908	if (uaddr == -EFAULT)
2909	return -EFAULT;
2910	e->adapter.summary_addr = uaddr;
2911	uaddr = gmap_translate(kvm->arch.gmap, ue->u.adapter.ind_addr);
2912	if (uaddr == -EFAULT)
2913	return -EFAULT;
2914	e->adapter.ind_addr = uaddr;
2915	e->adapter.summary_offset = ue->u.adapter.summary_offset;
2916	e->adapter.ind_offset = ue->u.adapter.ind_offset;
2917	e->adapter.adapter_id = ue->u.adapter.adapter_id;
2918	return 0;
2919	default:
2920	return -EINVAL;
2921	}
2922	}
2923
2924	int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm,
2925	int irq_source_id, int level, bool line_status)
2926	{
2927	return -EINVAL;
2928	}
2929
2930	int kvm_s390_set_irq_state(struct kvm_vcpu *vcpu, void __user *irqstate, int len)
2931	{
2932	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2933	struct kvm_s390_irq *buf;
2934	int r = 0;
2935	int n;
2936
2937	buf = vmalloc(size: len);
2938	if (!buf)
2939	return -ENOMEM;
2940
2941	if (copy_from_user(to: (void *) buf, from: irqstate, n: len)) {
2942	r = -EFAULT;
2943	goto out_free;
2944	}
2945
2946	/*
2947	* Don't allow setting the interrupt state
2948	* when there are already interrupts pending
2949	*/
2950	spin_lock(lock: &li->lock);
2951	if (li->pending_irqs) {
2952	r = -EBUSY;
2953	goto out_unlock;
2954	}
2955
2956	for (n = 0; n < len / sizeof(*buf); n++) {
2957	r = do_inject_vcpu(vcpu, &buf[n]);
2958	if (r)
2959	break;
2960	}
2961
2962	out_unlock:
2963	spin_unlock(lock: &li->lock);
2964	out_free:
2965	vfree(addr: buf);
2966
2967	return r;
2968	}
2969
2970	static void store_local_irq(struct kvm_s390_local_interrupt *li,
2971	struct kvm_s390_irq *irq,
2972	unsigned long irq_type)
2973	{
2974	switch (irq_type) {
2975	case IRQ_PEND_MCHK_EX:
2976	case IRQ_PEND_MCHK_REP:
2977	irq->type = KVM_S390_MCHK;
2978	irq->u.mchk = li->irq.mchk;
2979	break;
2980	case IRQ_PEND_PROG:
2981	irq->type = KVM_S390_PROGRAM_INT;
2982	irq->u.pgm = li->irq.pgm;
2983	break;
2984	case IRQ_PEND_PFAULT_INIT:
2985	irq->type = KVM_S390_INT_PFAULT_INIT;
2986	irq->u.ext = li->irq.ext;
2987	break;
2988	case IRQ_PEND_EXT_EXTERNAL:
2989	irq->type = KVM_S390_INT_EXTERNAL_CALL;
2990	irq->u.extcall = li->irq.extcall;
2991	break;
2992	case IRQ_PEND_EXT_CLOCK_COMP:
2993	irq->type = KVM_S390_INT_CLOCK_COMP;
2994	break;
2995	case IRQ_PEND_EXT_CPU_TIMER:
2996	irq->type = KVM_S390_INT_CPU_TIMER;
2997	break;
2998	case IRQ_PEND_SIGP_STOP:
2999	irq->type = KVM_S390_SIGP_STOP;
3000	irq->u.stop = li->irq.stop;
3001	break;
3002	case IRQ_PEND_RESTART:
3003	irq->type = KVM_S390_RESTART;
3004	break;
3005	case IRQ_PEND_SET_PREFIX:
3006	irq->type = KVM_S390_SIGP_SET_PREFIX;
3007	irq->u.prefix = li->irq.prefix;
3008	break;
3009	}
3010	}
3011
3012	int kvm_s390_get_irq_state(struct kvm_vcpu *vcpu, __u8 __user *buf, int len)
3013	{
3014	int scn;
3015	DECLARE_BITMAP(sigp_emerg_pending, KVM_MAX_VCPUS);
3016	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
3017	unsigned long pending_irqs;
3018	struct kvm_s390_irq irq;
3019	unsigned long irq_type;
3020	int cpuaddr;
3021	int n = 0;
3022
3023	spin_lock(lock: &li->lock);
3024	pending_irqs = li->pending_irqs;
3025	memcpy(&sigp_emerg_pending, &li->sigp_emerg_pending,
3026	sizeof(sigp_emerg_pending));
3027	spin_unlock(lock: &li->lock);
3028
3029	for_each_set_bit(irq_type, &pending_irqs, IRQ_PEND_COUNT) {
3030	memset(&irq, 0, sizeof(irq));
3031	if (irq_type == IRQ_PEND_EXT_EMERGENCY)
3032	continue;
3033	if (n + sizeof(irq) > len)
3034	return -ENOBUFS;
3035	store_local_irq(&vcpu->arch.local_int, &irq, irq_type);
3036	if (copy_to_user(&buf[n], &irq, sizeof(irq)))
3037	return -EFAULT;
3038	n += sizeof(irq);
3039	}
3040
3041	if (test_bit(IRQ_PEND_EXT_EMERGENCY, &pending_irqs)) {
3042	for_each_set_bit(cpuaddr, sigp_emerg_pending, KVM_MAX_VCPUS) {
3043	memset(&irq, 0, sizeof(irq));
3044	if (n + sizeof(irq) > len)
3045	return -ENOBUFS;
3046	irq.type = KVM_S390_INT_EMERGENCY;
3047	irq.u.emerg.code = cpuaddr;
3048	if (copy_to_user(to: &buf[n], from: &irq, n: sizeof(irq)))
3049	return -EFAULT;
3050	n += sizeof(irq);
3051	}
3052	}
3053
3054	if (sca_ext_call_pending(vcpu, src_id: &scn)) {
3055	if (n + sizeof(irq) > len)
3056	return -ENOBUFS;
3057	memset(&irq, 0, sizeof(irq));
3058	irq.type = KVM_S390_INT_EXTERNAL_CALL;
3059	irq.u.extcall.code = scn;
3060	if (copy_to_user(to: &buf[n], from: &irq, n: sizeof(irq)))
3061	return -EFAULT;
3062	n += sizeof(irq);
3063	}
3064
3065	return n;
3066	}
3067
3068	static void __airqs_kick_single_vcpu(struct kvm *kvm, u8 deliverable_mask)
3069	{
3070	int vcpu_idx, online_vcpus = atomic_read(v: &kvm->online_vcpus);
3071	struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3072	struct kvm_vcpu *vcpu;
3073	u8 vcpu_isc_mask;
3074
3075	for_each_set_bit(vcpu_idx, kvm->arch.idle_mask, online_vcpus) {
3076	vcpu = kvm_get_vcpu(kvm, i: vcpu_idx);
3077	if (psw_ioint_disabled(vcpu))
3078	continue;
3079	vcpu_isc_mask = (u8)(vcpu->arch.sie_block->gcr[6] >> 24);
3080	if (deliverable_mask & vcpu_isc_mask) {
3081	/* lately kicked but not yet running */
3082	if (test_and_set_bit(nr: vcpu_idx, addr: gi->kicked_mask))
3083	return;
3084	kvm_s390_vcpu_wakeup(vcpu);
3085	return;
3086	}
3087	}
3088	}
3089
3090	static enum hrtimer_restart gisa_vcpu_kicker(struct hrtimer *timer)
3091	{
3092	struct kvm_s390_gisa_interrupt *gi =
3093	container_of(timer, struct kvm_s390_gisa_interrupt, timer);
3094	struct kvm *kvm =
3095	container_of(gi->origin, struct sie_page2, gisa)->kvm;
3096	u8 pending_mask;
3097
3098	pending_mask = gisa_get_ipm_or_restore_iam(gi);
3099	if (pending_mask) {
3100	__airqs_kick_single_vcpu(kvm, deliverable_mask: pending_mask);
3101	hrtimer_forward_now(timer, interval: ns_to_ktime(ns: gi->expires));
3102	return HRTIMER_RESTART;
3103	}
3104
3105	return HRTIMER_NORESTART;
3106	}
3107
3108	#define NULL_GISA_ADDR 0x00000000UL
3109	#define NONE_GISA_ADDR 0x00000001UL
3110	#define GISA_ADDR_MASK 0xfffff000UL
3111
3112	static void process_gib_alert_list(void)
3113	{
3114	struct kvm_s390_gisa_interrupt *gi;
3115	u32 final, gisa_phys, origin = 0UL;
3116	struct kvm_s390_gisa *gisa;
3117	struct kvm *kvm;
3118
3119	do {
3120	/*
3121	* If the NONE_GISA_ADDR is still stored in the alert list
3122	* origin, we will leave the outer loop. No further GISA has
3123	* been added to the alert list by millicode while processing
3124	* the current alert list.
3125	*/
3126	final = (origin & NONE_GISA_ADDR);
3127	/*
3128	* Cut off the alert list and store the NONE_GISA_ADDR in the
3129	* alert list origin to avoid further GAL interruptions.
3130	* A new alert list can be build up by millicode in parallel
3131	* for guests not in the yet cut-off alert list. When in the
3132	* final loop, store the NULL_GISA_ADDR instead. This will re-
3133	* enable GAL interruptions on the host again.
3134	*/
3135	origin = xchg(&gib->alert_list_origin,
3136	(!final) ? NONE_GISA_ADDR : NULL_GISA_ADDR);
3137	/*
3138	* Loop through the just cut-off alert list and start the
3139	* gisa timers to kick idle vcpus to consume the pending
3140	* interruptions asap.
3141	*/
3142	while (origin & GISA_ADDR_MASK) {
3143	gisa_phys = origin;
3144	gisa = phys_to_virt(address: gisa_phys);
3145	origin = gisa->next_alert;
3146	gisa->next_alert = gisa_phys;
3147	kvm = container_of(gisa, struct sie_page2, gisa)->kvm;
3148	gi = &kvm->arch.gisa_int;
3149	if (hrtimer_active(timer: &gi->timer))
3150	hrtimer_cancel(timer: &gi->timer);
3151	hrtimer_start(timer: &gi->timer, tim: 0, mode: HRTIMER_MODE_REL);
3152	}
3153	} while (!final);
3154
3155	}
3156
3157	void kvm_s390_gisa_clear(struct kvm *kvm)
3158	{
3159	struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3160
3161	if (!gi->origin)
3162	return;
3163	gisa_clear_ipm(gisa: gi->origin);
3164	VM_EVENT(kvm, 3, "gisa 0x%pK cleared", gi->origin);
3165	}
3166
3167	void kvm_s390_gisa_init(struct kvm *kvm)
3168	{
3169	struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3170
3171	if (!css_general_characteristics.aiv)
3172	return;
3173	gi->origin = &kvm->arch.sie_page2->gisa;
3174	gi->alert.mask = 0;
3175	spin_lock_init(&gi->alert.ref_lock);
3176	gi->expires = 50 * 1000; /* 50 usec */
3177	hrtimer_init(timer: &gi->timer, CLOCK_MONOTONIC, mode: HRTIMER_MODE_REL);
3178	gi->timer.function = gisa_vcpu_kicker;
3179	memset(gi->origin, 0, sizeof(struct kvm_s390_gisa));
3180	gi->origin->next_alert = (u32)virt_to_phys(address: gi->origin);
3181	VM_EVENT(kvm, 3, "gisa 0x%pK initialized", gi->origin);
3182	}
3183
3184	void kvm_s390_gisa_enable(struct kvm *kvm)
3185	{
3186	struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3187	struct kvm_vcpu *vcpu;
3188	unsigned long i;
3189	u32 gisa_desc;
3190
3191	if (gi->origin)
3192	return;
3193	kvm_s390_gisa_init(kvm);
3194	gisa_desc = kvm_s390_get_gisa_desc(kvm);
3195	if (!gisa_desc)
3196	return;
3197	kvm_for_each_vcpu(i, vcpu, kvm) {
3198	mutex_lock(&vcpu->mutex);
3199	vcpu->arch.sie_block->gd = gisa_desc;
3200	vcpu->arch.sie_block->eca |= ECA_AIV;
3201	VCPU_EVENT(vcpu, 3, "AIV gisa format-%u enabled for cpu %03u",
3202	vcpu->arch.sie_block->gd & 0x3, vcpu->vcpu_id);
3203	mutex_unlock(lock: &vcpu->mutex);
3204	}
3205	}
3206
3207	void kvm_s390_gisa_destroy(struct kvm *kvm)
3208	{
3209	struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3210	struct kvm_s390_gisa *gisa = gi->origin;
3211
3212	if (!gi->origin)
3213	return;
3214	WARN(gi->alert.mask != 0x00,
3215	"unexpected non zero alert.mask 0x%02x",
3216	gi->alert.mask);
3217	gi->alert.mask = 0x00;
3218	if (gisa_set_iam(gisa: gi->origin, iam: gi->alert.mask))
3219	process_gib_alert_list();
3220	hrtimer_cancel(timer: &gi->timer);
3221	gi->origin = NULL;
3222	VM_EVENT(kvm, 3, "gisa 0x%pK destroyed", gisa);
3223	}
3224
3225	void kvm_s390_gisa_disable(struct kvm *kvm)
3226	{
3227	struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3228	struct kvm_vcpu *vcpu;
3229	unsigned long i;
3230
3231	if (!gi->origin)
3232	return;
3233	kvm_for_each_vcpu(i, vcpu, kvm) {
3234	mutex_lock(&vcpu->mutex);
3235	vcpu->arch.sie_block->eca &= ~ECA_AIV;
3236	vcpu->arch.sie_block->gd = 0U;
3237	mutex_unlock(lock: &vcpu->mutex);
3238	VCPU_EVENT(vcpu, 3, "AIV disabled for cpu %03u", vcpu->vcpu_id);
3239	}
3240	kvm_s390_gisa_destroy(kvm);
3241	}
3242
3243	/**
3244	* kvm_s390_gisc_register - register a guest ISC
3245	*
3246	* @kvm: the kernel vm to work with
3247	* @gisc: the guest interruption sub class to register
3248	*
3249	* The function extends the vm specific alert mask to use.
3250	* The effective IAM mask in the GISA is updated as well
3251	* in case the GISA is not part of the GIB alert list.
3252	* It will be updated latest when the IAM gets restored
3253	* by gisa_get_ipm_or_restore_iam().
3254	*
3255	* Returns: the nonspecific ISC (NISC) the gib alert mechanism
3256	* has registered with the channel subsystem.
3257	* -ENODEV in case the vm uses no GISA
3258	* -ERANGE in case the guest ISC is invalid
3259	*/
3260	int kvm_s390_gisc_register(struct kvm *kvm, u32 gisc)
3261	{
3262	struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3263
3264	if (!gi->origin)
3265	return -ENODEV;
3266	if (gisc > MAX_ISC)
3267	return -ERANGE;
3268
3269	spin_lock(lock: &gi->alert.ref_lock);
3270	gi->alert.ref_count[gisc]++;
3271	if (gi->alert.ref_count[gisc] == 1) {
3272	gi->alert.mask |= 0x80 >> gisc;
3273	gisa_set_iam(gisa: gi->origin, iam: gi->alert.mask);
3274	}
3275	spin_unlock(lock: &gi->alert.ref_lock);
3276
3277	return gib->nisc;
3278	}
3279	EXPORT_SYMBOL_GPL(kvm_s390_gisc_register);
3280
3281	/**
3282	* kvm_s390_gisc_unregister - unregister a guest ISC
3283	*
3284	* @kvm: the kernel vm to work with
3285	* @gisc: the guest interruption sub class to register
3286	*
3287	* The function reduces the vm specific alert mask to use.
3288	* The effective IAM mask in the GISA is updated as well
3289	* in case the GISA is not part of the GIB alert list.
3290	* It will be updated latest when the IAM gets restored
3291	* by gisa_get_ipm_or_restore_iam().
3292	*
3293	* Returns: the nonspecific ISC (NISC) the gib alert mechanism
3294	* has registered with the channel subsystem.
3295	* -ENODEV in case the vm uses no GISA
3296	* -ERANGE in case the guest ISC is invalid
3297	* -EINVAL in case the guest ISC is not registered
3298	*/
3299	int kvm_s390_gisc_unregister(struct kvm *kvm, u32 gisc)
3300	{
3301	struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3302	int rc = 0;
3303
3304	if (!gi->origin)
3305	return -ENODEV;
3306	if (gisc > MAX_ISC)
3307	return -ERANGE;
3308
3309	spin_lock(lock: &gi->alert.ref_lock);
3310	if (gi->alert.ref_count[gisc] == 0) {
3311	rc = -EINVAL;
3312	goto out;
3313	}
3314	gi->alert.ref_count[gisc]--;
3315	if (gi->alert.ref_count[gisc] == 0) {
3316	gi->alert.mask &= ~(0x80 >> gisc);
3317	gisa_set_iam(gisa: gi->origin, iam: gi->alert.mask);
3318	}
3319	out:
3320	spin_unlock(lock: &gi->alert.ref_lock);
3321
3322	return rc;
3323	}
3324	EXPORT_SYMBOL_GPL(kvm_s390_gisc_unregister);
3325
3326	static void aen_host_forward(unsigned long si)
3327	{
3328	struct kvm_s390_gisa_interrupt *gi;
3329	struct zpci_gaite *gaite;
3330	struct kvm *kvm;
3331
3332	gaite = (struct zpci_gaite *)aift->gait +
3333	(si * sizeof(struct zpci_gaite));
3334	if (gaite->count == 0)
3335	return;
3336	if (gaite->aisb != 0)
3337	set_bit_inv(gaite->aisbo, phys_to_virt(address: gaite->aisb));
3338
3339	kvm = kvm_s390_pci_si_to_kvm(aift, si);
3340	if (!kvm)
3341	return;
3342	gi = &kvm->arch.gisa_int;
3343
3344	if (!(gi->origin->g1.simm & AIS_MODE_MASK(gaite->gisc)) ||
3345	!(gi->origin->g1.nimm & AIS_MODE_MASK(gaite->gisc))) {
3346	gisa_set_ipm_gisc(gisa: gi->origin, gisc: gaite->gisc);
3347	if (hrtimer_active(timer: &gi->timer))
3348	hrtimer_cancel(timer: &gi->timer);
3349	hrtimer_start(timer: &gi->timer, tim: 0, mode: HRTIMER_MODE_REL);
3350	kvm->stat.aen_forward++;
3351	}
3352	}
3353
3354	static void aen_process_gait(u8 isc)
3355	{
3356	bool found = false, first = true;
3357	union zpci_sic_iib iib = {{0}};
3358	unsigned long si, flags;
3359
3360	spin_lock_irqsave(&aift->gait_lock, flags);
3361
3362	if (!aift->gait) {
3363	spin_unlock_irqrestore(lock: &aift->gait_lock, flags);
3364	return;
3365	}
3366
3367	for (si = 0;;) {
3368	/* Scan adapter summary indicator bit vector */
3369	si = airq_iv_scan(aift->sbv, si, airq_iv_end(aift->sbv));
3370	if (si == -1UL) {
3371	if (first || found) {
3372	/* Re-enable interrupts. */
3373	zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, isc,
3374	&iib);
3375	first = found = false;
3376	} else {
3377	/* Interrupts on and all bits processed */
3378	break;
3379	}
3380	found = false;
3381	si = 0;
3382	/* Scan again after re-enabling interrupts */
3383	continue;
3384	}
3385	found = true;
3386	aen_host_forward(si);
3387	}
3388
3389	spin_unlock_irqrestore(lock: &aift->gait_lock, flags);
3390	}
3391
3392	static void gib_alert_irq_handler(struct airq_struct *airq,
3393	struct tpi_info *tpi_info)
3394	{
3395	struct tpi_adapter_info *info = (struct tpi_adapter_info *)tpi_info;
3396
3397	inc_irq_stat(IRQIO_GAL);
3398
3399	if ((info->forward || info->error) &&
3400	IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM)) {
3401	aen_process_gait(isc: info->isc);
3402	if (info->aism != 0)
3403	process_gib_alert_list();
3404	} else {
3405	process_gib_alert_list();
3406	}
3407	}
3408
3409	static struct airq_struct gib_alert_irq = {
3410	.handler = gib_alert_irq_handler,
3411	};
3412
3413	void kvm_s390_gib_destroy(void)
3414	{
3415	if (!gib)
3416	return;
3417	if (kvm_s390_pci_interp_allowed() && aift) {
3418	mutex_lock(&aift->aift_lock);
3419	kvm_s390_pci_aen_exit();
3420	mutex_unlock(lock: &aift->aift_lock);
3421	}
3422	chsc_sgib(0);
3423	unregister_adapter_interrupt(&gib_alert_irq);
3424	free_page((unsigned long)gib);
3425	gib = NULL;
3426	}
3427
3428	int __init kvm_s390_gib_init(u8 nisc)
3429	{
3430	u32 gib_origin;
3431	int rc = 0;
3432
3433	if (!css_general_characteristics.aiv) {
3434	KVM_EVENT(3, "%s", "gib not initialized, no AIV facility");
3435	goto out;
3436	}
3437
3438	gib = (struct kvm_s390_gib *)get_zeroed_page(GFP_KERNEL_ACCOUNT | GFP_DMA);
3439	if (!gib) {
3440	rc = -ENOMEM;
3441	goto out;
3442	}
3443
3444	gib_alert_irq.isc = nisc;
3445	if (register_adapter_interrupt(&gib_alert_irq)) {
3446	pr_err("Registering the GIB alert interruption handler failed\n");
3447	rc = -EIO;
3448	goto out_free_gib;
3449	}
3450	/* adapter interrupts used for AP (applicable here) don't use the LSI */
3451	*gib_alert_irq.lsi_ptr = 0xff;
3452
3453	gib->nisc = nisc;
3454	gib_origin = virt_to_phys(address: gib);
3455	if (chsc_sgib(gib_origin)) {
3456	pr_err("Associating the GIB with the AIV facility failed\n");
3457	free_page((unsigned long)gib);
3458	gib = NULL;
3459	rc = -EIO;
3460	goto out_unreg_gal;
3461	}
3462
3463	if (kvm_s390_pci_interp_allowed()) {
3464	if (kvm_s390_pci_aen_init(nisc)) {
3465	pr_err("Initializing AEN for PCI failed\n");
3466	rc = -EIO;
3467	goto out_unreg_gal;
3468	}
3469	}
3470
3471	KVM_EVENT(3, "gib 0x%pK (nisc=%d) initialized", gib, gib->nisc);
3472	goto out;
3473
3474	out_unreg_gal:
3475	unregister_adapter_interrupt(&gib_alert_irq);
3476	out_free_gib:
3477	free_page((unsigned long)gib);
3478	gib = NULL;
3479	out:
3480	return rc;
3481	}
3482