1	/* SPDX-License-Identifier: GPL-2.0-only */
2	/*
3	*
4	* Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
5	*
6	* Derived from book3s_rmhandlers.S and other files, which are:
7	*
8	* Copyright SUSE Linux Products GmbH 2009
9	*
10	* Authors: Alexander Graf <agraf@suse.de>
11	*/
12
13	#include <linux/export.h>
14	#include <linux/linkage.h>
15	#include <linux/objtool.h>
16	#include <asm/ppc_asm.h>
17	#include <asm/code-patching-asm.h>
18	#include <asm/kvm_asm.h>
19	#include <asm/reg.h>
20	#include <asm/mmu.h>
21	#include <asm/page.h>
22	#include <asm/ptrace.h>
23	#include <asm/hvcall.h>
24	#include <asm/asm-offsets.h>
25	#include <asm/exception-64s.h>
26	#include <asm/kvm_book3s_asm.h>
27	#include <asm/book3s/64/mmu-hash.h>
28	#include <asm/tm.h>
29	#include <asm/opal.h>
30	#include <asm/thread_info.h>
31	#include <asm/asm-compat.h>
32	#include <asm/feature-fixups.h>
33	#include <asm/cpuidle.h>
34
35	/* Values in HSTATE_NAPPING(r13) */
36	#define NAPPING_CEDE 1
37	#define NAPPING_NOVCPU 2
38	#define NAPPING_UNSPLIT 3
39
40	/* Stack frame offsets for kvmppc_hv_entry */
41	#define SFS 160
42	#define STACK_SLOT_TRAP (SFS-4)
43	#define STACK_SLOT_TID (SFS-16)
44	#define STACK_SLOT_PSSCR (SFS-24)
45	#define STACK_SLOT_PID (SFS-32)
46	#define STACK_SLOT_IAMR (SFS-40)
47	#define STACK_SLOT_CIABR (SFS-48)
48	#define STACK_SLOT_DAWR0 (SFS-56)
49	#define STACK_SLOT_DAWRX0 (SFS-64)
50	#define STACK_SLOT_HFSCR (SFS-72)
51	#define STACK_SLOT_AMR (SFS-80)
52	#define STACK_SLOT_UAMOR (SFS-88)
53	#define STACK_SLOT_FSCR (SFS-96)
54
55	/*
56	* Use the last LPID (all implemented LPID bits = 1) for partition switching.
57	* This is reserved in the LPID allocator. POWER7 only implements 0x3ff, but
58	* we write 0xfff into the LPID SPR anyway, which seems to work and just
59	* ignores the top bits.
60	*/
61	#define LPID_RSVD 0xfff
62
63	/*
64	* Call kvmppc_hv_entry in real mode.
65	* Must be called with interrupts hard-disabled.
66	*
67	* Input Registers:
68	*
69	* LR = return address to continue at after eventually re-enabling MMU
70	*/
71	_GLOBAL_TOC(kvmppc_hv_entry_trampoline)
72	mflr r0
73	std r0, PPC_LR_STKOFF(r1)
74	stdu r1, -112(r1)
75	mfmsr r10
76	std r10, HSTATE_HOST_MSR(r13)
77	LOAD_REG_ADDR(r5, kvmppc_call_hv_entry)
78	li r0,MSR_RI
79	andc r0,r10,r0
80	li r6,MSR_IR | MSR_DR
81	andc r6,r10,r6
82	mtmsrd r0,1 /* clear RI in MSR */
83	mtsrr0 r5
84	mtsrr1 r6
85	RFI_TO_KERNEL
86
87	kvmppc_call_hv_entry:
88	ld r4, HSTATE_KVM_VCPU(r13)
89	bl kvmppc_hv_entry
90
91	/* Back from guest - restore host state and return to caller */
92
93	BEGIN_FTR_SECTION
94	/* Restore host DABR and DABRX */
95	ld r5,HSTATE_DABR(r13)
96	li r6,7
97	mtspr SPRN_DABR,r5
98	mtspr SPRN_DABRX,r6
99	END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
100
101	/* Restore SPRG3 */
102	ld r3,PACA_SPRG_VDSO(r13)
103	mtspr SPRN_SPRG_VDSO_WRITE,r3
104
105	/* Reload the host's PMU registers */
106	bl kvmhv_load_host_pmu
107
108	/*
109	* Reload DEC. HDEC interrupts were disabled when
110	* we reloaded the host's LPCR value.
111	*/
112	ld r3, HSTATE_DECEXP(r13)
113	mftb r4
114	subf r4, r4, r3
115	mtspr SPRN_DEC, r4
116
117	/* hwthread_req may have got set by cede or no vcpu, so clear it */
118	li r0, 0
119	stb r0, HSTATE_HWTHREAD_REQ(r13)
120
121	/*
122	* For external interrupts we need to call the Linux
123	* handler to process the interrupt. We do that by jumping
124	* to absolute address 0x500 for external interrupts.
125	* The [h]rfid at the end of the handler will return to
126	* the book3s_hv_interrupts.S code. For other interrupts
127	* we do the rfid to get back to the book3s_hv_interrupts.S
128	* code here.
129	*/
130	ld r8, 112+PPC_LR_STKOFF(r1)
131	addi r1, r1, 112
132	ld r7, HSTATE_HOST_MSR(r13)
133
134	/* Return the trap number on this thread as the return value */
135	mr r3, r12
136
137	/* RFI into the highmem handler */
138	mfmsr r6
139	li r0, MSR_RI
140	andc r6, r6, r0
141	mtmsrd r6, 1 /* Clear RI in MSR */
142	mtsrr0 r8
143	mtsrr1 r7
144	RFI_TO_KERNEL
145
146	kvmppc_primary_no_guest:
147	/* We handle this much like a ceded vcpu */
148	/* put the HDEC into the DEC, since HDEC interrupts don't wake us */
149	/* HDEC may be larger than DEC for arch >= v3.00, but since the */
150	/* HDEC value came from DEC in the first place, it will fit */
151	mfspr r3, SPRN_HDEC
152	mtspr SPRN_DEC, r3
153	/*
154	* Make sure the primary has finished the MMU switch.
155	* We should never get here on a secondary thread, but
156	* check it for robustness' sake.
157	*/
158	ld r5, HSTATE_KVM_VCORE(r13)
159	65: lbz r0, VCORE_IN_GUEST(r5)
160	cmpwi r0, 0
161	beq 65b
162	/* Set LPCR. */
163	ld r8,VCORE_LPCR(r5)
164	mtspr SPRN_LPCR,r8
165	isync
166	/* set our bit in napping_threads */
167	ld r5, HSTATE_KVM_VCORE(r13)
168	lbz r7, HSTATE_PTID(r13)
169	li r0, 1
170	sld r0, r0, r7
171	addi r6, r5, VCORE_NAPPING_THREADS
172	1: lwarx r3, 0, r6
173	or r3, r3, r0
174	stwcx. r3, 0, r6
175	bne 1b
176	/* order napping_threads update vs testing entry_exit_map */
177	isync
178	li r12, 0
179	lwz r7, VCORE_ENTRY_EXIT(r5)
180	cmpwi r7, 0x100
181	bge kvm_novcpu_exit /* another thread already exiting */
182	li r3, NAPPING_NOVCPU
183	stb r3, HSTATE_NAPPING(r13)
184
185	li r3, 0 /* Don't wake on privileged (OS) doorbell */
186	b kvm_do_nap
187
188	/*
189	* kvm_novcpu_wakeup
190	* Entered from kvm_start_guest if kvm_hstate.napping is set
191	* to NAPPING_NOVCPU
192	* r2 = kernel TOC
193	* r13 = paca
194	*/
195	kvm_novcpu_wakeup:
196	ld r1, HSTATE_HOST_R1(r13)
197	ld r5, HSTATE_KVM_VCORE(r13)
198	li r0, 0
199	stb r0, HSTATE_NAPPING(r13)
200
201	/* check the wake reason */
202	bl kvmppc_check_wake_reason
203
204	/*
205	* Restore volatile registers since we could have called
206	* a C routine in kvmppc_check_wake_reason.
207	* r5 = VCORE
208	*/
209	ld r5, HSTATE_KVM_VCORE(r13)
210
211	/* see if any other thread is already exiting */
212	lwz r0, VCORE_ENTRY_EXIT(r5)
213	cmpwi r0, 0x100
214	bge kvm_novcpu_exit
215
216	/* clear our bit in napping_threads */
217	lbz r7, HSTATE_PTID(r13)
218	li r0, 1
219	sld r0, r0, r7
220	addi r6, r5, VCORE_NAPPING_THREADS
221	4: lwarx r7, 0, r6
222	andc r7, r7, r0
223	stwcx. r7, 0, r6
224	bne 4b
225
226	/* See if the wake reason means we need to exit */
227	cmpdi r3, 0
228	bge kvm_novcpu_exit
229
230	/* See if our timeslice has expired (HDEC is negative) */
231	mfspr r0, SPRN_HDEC
232	extsw r0, r0
233	li r12, BOOK3S_INTERRUPT_HV_DECREMENTER
234	cmpdi r0, 0
235	blt kvm_novcpu_exit
236
237	/* Got an IPI but other vcpus aren't yet exiting, must be a latecomer */
238	ld r4, HSTATE_KVM_VCPU(r13)
239	cmpdi r4, 0
240	beq kvmppc_primary_no_guest
241
242	#ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING
243	addi r3, r4, VCPU_TB_RMENTRY
244	bl kvmhv_start_timing
245	#endif
246	b kvmppc_got_guest
247
248	kvm_novcpu_exit:
249	#ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING
250	ld r4, HSTATE_KVM_VCPU(r13)
251	cmpdi r4, 0
252	beq 13f
253	addi r3, r4, VCPU_TB_RMEXIT
254	bl kvmhv_accumulate_time
255	#endif
256	13: mr r3, r12
257	stw r12, STACK_SLOT_TRAP(r1)
258	bl kvmhv_commence_exit
259	nop
260	b kvmhv_switch_to_host
261
262	/*
263	* We come in here when wakened from Linux offline idle code.
264	* Relocation is off
265	* r3 contains the SRR1 wakeup value, SRR1 is trashed.
266	*/
267	_GLOBAL(idle_kvm_start_guest)
268	mfcr r5
269	mflr r0
270	std r5, 8(r1) // Save CR in caller's frame
271	std r0, 16(r1) // Save LR in caller's frame
272	// Create frame on emergency stack
273	ld r4, PACAEMERGSP(r13)
274	stdu r1, -SWITCH_FRAME_SIZE(r4)
275	// Switch to new frame on emergency stack
276	mr r1, r4
277	std r3, 32(r1) // Save SRR1 wakeup value
278	SAVE_NVGPRS(r1)
279
280	/*
281	* Could avoid this and pass it through in r3. For now,
282	* code expects it to be in SRR1.
283	*/
284	mtspr SPRN_SRR1,r3
285
286	li r0,0
287	stb r0,PACA_FTRACE_ENABLED(r13)
288
289	li r0,KVM_HWTHREAD_IN_KVM
290	stb r0,HSTATE_HWTHREAD_STATE(r13)
291
292	/* kvm cede / napping does not come through here */
293	lbz r0,HSTATE_NAPPING(r13)
294	twnei r0,0
295
296	b 1f
297
298	kvm_unsplit_wakeup:
299	li r0, 0
300	stb r0, HSTATE_NAPPING(r13)
301
302	1:
303
304	/*
305	* We weren't napping due to cede, so this must be a secondary
306	* thread being woken up to run a guest, or being woken up due
307	* to a stray IPI. (Or due to some machine check or hypervisor
308	* maintenance interrupt while the core is in KVM.)
309	*/
310
311	/* Check the wake reason in SRR1 to see why we got here */
312	bl kvmppc_check_wake_reason
313	/*
314	* kvmppc_check_wake_reason could invoke a C routine, but we
315	* have no volatile registers to restore when we return.
316	*/
317
318	cmpdi r3, 0
319	bge kvm_no_guest
320
321	/* get vcore pointer, NULL if we have nothing to run */
322	ld r5,HSTATE_KVM_VCORE(r13)
323	cmpdi r5,0
324	/* if we have no vcore to run, go back to sleep */
325	beq kvm_no_guest
326
327	kvm_secondary_got_guest:
328
329	// About to go to guest, clear saved SRR1
330	li r0, 0
331	std r0, 32(r1)
332
333	/* Set HSTATE_DSCR(r13) to something sensible */
334	ld r6, PACA_DSCR_DEFAULT(r13)
335	std r6, HSTATE_DSCR(r13)
336
337	/* On thread 0 of a subcore, set HDEC to max */
338	lbz r4, HSTATE_PTID(r13)
339	cmpwi r4, 0
340	bne 63f
341	lis r6,0x7fff /* MAX_INT@h */
342	mtspr SPRN_HDEC, r6
343	/* and set per-LPAR registers, if doing dynamic micro-threading */
344	ld r6, HSTATE_SPLIT_MODE(r13)
345	cmpdi r6, 0
346	beq 63f
347	ld r0, KVM_SPLIT_RPR(r6)
348	mtspr SPRN_RPR, r0
349	ld r0, KVM_SPLIT_PMMAR(r6)
350	mtspr SPRN_PMMAR, r0
351	ld r0, KVM_SPLIT_LDBAR(r6)
352	mtspr SPRN_LDBAR, r0
353	isync
354	63:
355	/* Order load of vcpu after load of vcore */
356	lwsync
357	ld r4, HSTATE_KVM_VCPU(r13)
358	bl kvmppc_hv_entry
359
360	/* Back from the guest, go back to nap */
361	/* Clear our vcpu and vcore pointers so we don't come back in early */
362	li r0, 0
363	std r0, HSTATE_KVM_VCPU(r13)
364	/*
365	* Once we clear HSTATE_KVM_VCORE(r13), the code in
366	* kvmppc_run_core() is going to assume that all our vcpu
367	* state is visible in memory. This lwsync makes sure
368	* that that is true.
369	*/
370	lwsync
371	std r0, HSTATE_KVM_VCORE(r13)
372
373	/*
374	* All secondaries exiting guest will fall through this path.
375	* Before proceeding, just check for HMI interrupt and
376	* invoke opal hmi handler. By now we are sure that the
377	* primary thread on this core/subcore has already made partition
378	* switch/TB resync and we are good to call opal hmi handler.
379	*/
380	cmpwi r12, BOOK3S_INTERRUPT_HMI
381	bne kvm_no_guest
382
383	li r3,0 /* NULL argument */
384	bl CFUNC(hmi_exception_realmode)
385	/*
386	* At this point we have finished executing in the guest.
387	* We need to wait for hwthread_req to become zero, since
388	* we may not turn on the MMU while hwthread_req is non-zero.
389	* While waiting we also need to check if we get given a vcpu to run.
390	*/
391	kvm_no_guest:
392	lbz r3, HSTATE_HWTHREAD_REQ(r13)
393	cmpwi r3, 0
394	bne 53f
395	HMT_MEDIUM
396	li r0, KVM_HWTHREAD_IN_KERNEL
397	stb r0, HSTATE_HWTHREAD_STATE(r13)
398	/* need to recheck hwthread_req after a barrier, to avoid race */
399	sync
400	lbz r3, HSTATE_HWTHREAD_REQ(r13)
401	cmpwi r3, 0
402	bne 54f
403
404	/*
405	* Jump to idle_return_gpr_loss, which returns to the
406	* idle_kvm_start_guest caller.
407	*/
408	li r3, LPCR_PECE0
409	mfspr r4, SPRN_LPCR
410	rlwimi r4, r3, 0, LPCR_PECE0 | LPCR_PECE1
411	mtspr SPRN_LPCR, r4
412	// Return SRR1 wakeup value, or 0 if we went into the guest
413	ld r3, 32(r1)
414	REST_NVGPRS(r1)
415	ld r1, 0(r1) // Switch back to caller stack
416	ld r0, 16(r1) // Reload LR
417	ld r5, 8(r1) // Reload CR
418	mtlr r0
419	mtcr r5
420	blr
421
422	53:
423	HMT_LOW
424	ld r5, HSTATE_KVM_VCORE(r13)
425	cmpdi r5, 0
426	bne 60f
427	ld r3, HSTATE_SPLIT_MODE(r13)
428	cmpdi r3, 0
429	beq kvm_no_guest
430	lbz r0, KVM_SPLIT_DO_NAP(r3)
431	cmpwi r0, 0
432	beq kvm_no_guest
433	HMT_MEDIUM
434	b kvm_unsplit_nap
435	60: HMT_MEDIUM
436	b kvm_secondary_got_guest
437
438	54: li r0, KVM_HWTHREAD_IN_KVM
439	stb r0, HSTATE_HWTHREAD_STATE(r13)
440	b kvm_no_guest
441
442	/*
443	* Here the primary thread is trying to return the core to
444	* whole-core mode, so we need to nap.
445	*/
446	kvm_unsplit_nap:
447	/*
448	* When secondaries are napping in kvm_unsplit_nap() with
449	* hwthread_req = 1, HMI goes ignored even though subcores are
450	* already exited the guest. Hence HMI keeps waking up secondaries
451	* from nap in a loop and secondaries always go back to nap since
452	* no vcore is assigned to them. This makes impossible for primary
453	* thread to get hold of secondary threads resulting into a soft
454	* lockup in KVM path.
455	*
456	* Let us check if HMI is pending and handle it before we go to nap.
457	*/
458	cmpwi r12, BOOK3S_INTERRUPT_HMI
459	bne 55f
460	li r3, 0 /* NULL argument */
461	bl CFUNC(hmi_exception_realmode)
462	55:
463	/*
464	* Ensure that secondary doesn't nap when it has
465	* its vcore pointer set.
466	*/
467	sync /* matches smp_mb() before setting split_info.do_nap */
468	ld r0, HSTATE_KVM_VCORE(r13)
469	cmpdi r0, 0
470	bne kvm_no_guest
471	/* clear any pending message */
472	BEGIN_FTR_SECTION
473	lis r6, (PPC_DBELL_SERVER << (63-36))@h
474	PPC_MSGCLR(6)
475	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
476	/* Set kvm_split_mode.napped[tid] = 1 */
477	ld r3, HSTATE_SPLIT_MODE(r13)
478	li r0, 1
479	lhz r4, PACAPACAINDEX(r13)
480	clrldi r4, r4, 61 /* micro-threading => P8 => 8 threads/core */
481	addi r4, r4, KVM_SPLIT_NAPPED
482	stbx r0, r3, r4
483	/* Check the do_nap flag again after setting napped[] */
484	sync
485	lbz r0, KVM_SPLIT_DO_NAP(r3)
486	cmpwi r0, 0
487	beq 57f
488	li r3, NAPPING_UNSPLIT
489	stb r3, HSTATE_NAPPING(r13)
490	li r3, (LPCR_PECEDH | LPCR_PECE0) >> 4
491	mfspr r5, SPRN_LPCR
492	rlwimi r5, r3, 4, (LPCR_PECEDP | LPCR_PECEDH | LPCR_PECE0 | LPCR_PECE1)
493	b kvm_nap_sequence
494
495	57: li r0, 0
496	stbx r0, r3, r4
497	b kvm_no_guest
498
499	/******************************************************************************
500	* *
501	* Entry code *
502	* *
503	*****************************************************************************/
504
505	SYM_CODE_START_LOCAL(kvmppc_hv_entry)
506
507	/* Required state:
508	*
509	* R4 = vcpu pointer (or NULL)
510	* MSR = ~IR|DR
511	* R13 = PACA
512	* R1 = host R1
513	* R2 = TOC
514	* all other volatile GPRS = free
515	* Does not preserve non-volatile GPRs or CR fields
516	*/
517	mflr r0
518	std r0, PPC_LR_STKOFF(r1)
519	stdu r1, -SFS(r1)
520
521	/* Save R1 in the PACA */
522	std r1, HSTATE_HOST_R1(r13)
523
524	li r6, KVM_GUEST_MODE_HOST_HV
525	stb r6, HSTATE_IN_GUEST(r13)
526
527	#ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING
528	/* Store initial timestamp */
529	cmpdi r4, 0
530	beq 1f
531	addi r3, r4, VCPU_TB_RMENTRY
532	bl kvmhv_start_timing
533	1:
534	#endif
535
536	ld r5, HSTATE_KVM_VCORE(r13)
537	ld r9, VCORE_KVM(r5) /* pointer to struct kvm */
538
539	/*
540	* POWER7/POWER8 host -> guest partition switch code.
541	* We don't have to lock against concurrent tlbies,
542	* but we do have to coordinate across hardware threads.
543	*/
544	/* Set bit in entry map iff exit map is zero. */
545	li r7, 1
546	lbz r6, HSTATE_PTID(r13)
547	sld r7, r7, r6
548	addi r8, r5, VCORE_ENTRY_EXIT
549	21: lwarx r3, 0, r8
550	cmpwi r3, 0x100 /* any threads starting to exit? */
551	bge secondary_too_late /* if so we're too late to the party */
552	or r3, r3, r7
553	stwcx. r3, 0, r8
554	bne 21b
555
556	/* Primary thread switches to guest partition. */
557	cmpwi r6,0
558	bne 10f
559
560	lwz r7,KVM_LPID(r9)
561	ld r6,KVM_SDR1(r9)
562	li r0,LPID_RSVD /* switch to reserved LPID */
563	mtspr SPRN_LPID,r0
564	ptesync
565	mtspr SPRN_SDR1,r6 /* switch to partition page table */
566	mtspr SPRN_LPID,r7
567	isync
568
569	/* See if we need to flush the TLB. */
570	mr r3, r9 /* kvm pointer */
571	lhz r4, PACAPACAINDEX(r13) /* physical cpu number */
572	li r5, 0 /* nested vcpu pointer */
573	bl kvmppc_check_need_tlb_flush
574	nop
575	ld r5, HSTATE_KVM_VCORE(r13)
576
577	/* Add timebase offset onto timebase */
578	22: ld r8,VCORE_TB_OFFSET(r5)
579	cmpdi r8,0
580	beq 37f
581	std r8, VCORE_TB_OFFSET_APPL(r5)
582	mftb r6 /* current host timebase */
583	add r8,r8,r6
584	mtspr SPRN_TBU40,r8 /* update upper 40 bits */
585	mftb r7 /* check if lower 24 bits overflowed */
586	clrldi r6,r6,40
587	clrldi r7,r7,40
588	cmpld r7,r6
589	bge 37f
590	addis r8,r8,0x100 /* if so, increment upper 40 bits */
591	mtspr SPRN_TBU40,r8
592
593	/* Load guest PCR value to select appropriate compat mode */
594	37: ld r7, VCORE_PCR(r5)
595	LOAD_REG_IMMEDIATE(r6, PCR_MASK)
596	cmpld r7, r6
597	beq 38f
598	or r7, r7, r6
599	mtspr SPRN_PCR, r7
600	38:
601
602	BEGIN_FTR_SECTION
603	/* DPDES and VTB are shared between threads */
604	ld r8, VCORE_DPDES(r5)
605	ld r7, VCORE_VTB(r5)
606	mtspr SPRN_DPDES, r8
607	mtspr SPRN_VTB, r7
608	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
609
610	/* Mark the subcore state as inside guest */
611	bl kvmppc_subcore_enter_guest
612	nop
613	ld r5, HSTATE_KVM_VCORE(r13)
614	ld r4, HSTATE_KVM_VCPU(r13)
615	li r0,1
616	stb r0,VCORE_IN_GUEST(r5) /* signal secondaries to continue */
617
618	/* Do we have a guest vcpu to run? */
619	10: cmpdi r4, 0
620	beq kvmppc_primary_no_guest
621	kvmppc_got_guest:
622	/* Increment yield count if they have a VPA */
623	ld r3, VCPU_VPA(r4)
624	cmpdi r3, 0
625	beq 25f
626	li r6, LPPACA_YIELDCOUNT
627	LWZX_BE r5, r3, r6
628	addi r5, r5, 1
629	STWX_BE r5, r3, r6
630	li r6, 1
631	stb r6, VCPU_VPA_DIRTY(r4)
632	25:
633
634	/* Save purr/spurr */
635	mfspr r5,SPRN_PURR
636	mfspr r6,SPRN_SPURR
637	std r5,HSTATE_PURR(r13)
638	std r6,HSTATE_SPURR(r13)
639	ld r7,VCPU_PURR(r4)
640	ld r8,VCPU_SPURR(r4)
641	mtspr SPRN_PURR,r7
642	mtspr SPRN_SPURR,r8
643
644	/* Save host values of some registers */
645	BEGIN_FTR_SECTION
646	mfspr r5, SPRN_CIABR
647	mfspr r6, SPRN_DAWR0
648	mfspr r7, SPRN_DAWRX0
649	mfspr r8, SPRN_IAMR
650	std r5, STACK_SLOT_CIABR(r1)
651	std r6, STACK_SLOT_DAWR0(r1)
652	std r7, STACK_SLOT_DAWRX0(r1)
653	std r8, STACK_SLOT_IAMR(r1)
654	mfspr r5, SPRN_FSCR
655	std r5, STACK_SLOT_FSCR(r1)
656	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
657
658	mfspr r5, SPRN_AMR
659	std r5, STACK_SLOT_AMR(r1)
660	mfspr r6, SPRN_UAMOR
661	std r6, STACK_SLOT_UAMOR(r1)
662
663	BEGIN_FTR_SECTION
664	/* Set partition DABR */
665	/* Do this before re-enabling PMU to avoid P7 DABR corruption bug */
666	lwz r5,VCPU_DABRX(r4)
667	ld r6,VCPU_DABR(r4)
668	mtspr SPRN_DABRX,r5
669	mtspr SPRN_DABR,r6
670	isync
671	END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
672
673	#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
674	BEGIN_FTR_SECTION
675	b 91f
676	END_FTR_SECTION_IFCLR(CPU_FTR_TM)
677	/*
678	* NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR)
679	*/
680	mr r3, r4
681	ld r4, VCPU_MSR(r3)
682	li r5, 0 /* don't preserve non-vol regs */
683	bl kvmppc_restore_tm_hv
684	nop
685	ld r4, HSTATE_KVM_VCPU(r13)
686	91:
687	#endif
688
689	/* Load guest PMU registers; r4 = vcpu pointer here */
690	mr r3, r4
691	bl kvmhv_load_guest_pmu
692
693	/* Load up FP, VMX and VSX registers */
694	ld r4, HSTATE_KVM_VCPU(r13)
695	bl kvmppc_load_fp
696
697	ld r14, VCPU_GPR(R14)(r4)
698	ld r15, VCPU_GPR(R15)(r4)
699	ld r16, VCPU_GPR(R16)(r4)
700	ld r17, VCPU_GPR(R17)(r4)
701	ld r18, VCPU_GPR(R18)(r4)
702	ld r19, VCPU_GPR(R19)(r4)
703	ld r20, VCPU_GPR(R20)(r4)
704	ld r21, VCPU_GPR(R21)(r4)
705	ld r22, VCPU_GPR(R22)(r4)
706	ld r23, VCPU_GPR(R23)(r4)
707	ld r24, VCPU_GPR(R24)(r4)
708	ld r25, VCPU_GPR(R25)(r4)
709	ld r26, VCPU_GPR(R26)(r4)
710	ld r27, VCPU_GPR(R27)(r4)
711	ld r28, VCPU_GPR(R28)(r4)
712	ld r29, VCPU_GPR(R29)(r4)
713	ld r30, VCPU_GPR(R30)(r4)
714	ld r31, VCPU_GPR(R31)(r4)
715
716	/* Switch DSCR to guest value */
717	ld r5, VCPU_DSCR(r4)
718	mtspr SPRN_DSCR, r5
719
720	BEGIN_FTR_SECTION
721	/* Skip next section on POWER7 */
722	b 8f
723	END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
724	/* Load up POWER8-specific registers */
725	ld r5, VCPU_IAMR(r4)
726	lwz r6, VCPU_PSPB(r4)
727	ld r7, VCPU_FSCR(r4)
728	mtspr SPRN_IAMR, r5
729	mtspr SPRN_PSPB, r6
730	mtspr SPRN_FSCR, r7
731	/*
732	* Handle broken DAWR case by not writing it. This means we
733	* can still store the DAWR register for migration.
734	*/
735	LOAD_REG_ADDR(r5, dawr_force_enable)
736	lbz r5, 0(r5)
737	cmpdi r5, 0
738	beq 1f
739	ld r5, VCPU_DAWR0(r4)
740	ld r6, VCPU_DAWRX0(r4)
741	mtspr SPRN_DAWR0, r5
742	mtspr SPRN_DAWRX0, r6
743	1:
744	ld r7, VCPU_CIABR(r4)
745	ld r8, VCPU_TAR(r4)
746	mtspr SPRN_CIABR, r7
747	mtspr SPRN_TAR, r8
748	ld r5, VCPU_IC(r4)
749	ld r8, VCPU_EBBHR(r4)
750	mtspr SPRN_IC, r5
751	mtspr SPRN_EBBHR, r8
752	ld r5, VCPU_EBBRR(r4)
753	ld r6, VCPU_BESCR(r4)
754	lwz r7, VCPU_GUEST_PID(r4)
755	ld r8, VCPU_WORT(r4)
756	mtspr SPRN_EBBRR, r5
757	mtspr SPRN_BESCR, r6
758	mtspr SPRN_PID, r7
759	mtspr SPRN_WORT, r8
760	/* POWER8-only registers */
761	ld r5, VCPU_TCSCR(r4)
762	ld r6, VCPU_ACOP(r4)
763	ld r7, VCPU_CSIGR(r4)
764	ld r8, VCPU_TACR(r4)
765	mtspr SPRN_TCSCR, r5
766	mtspr SPRN_ACOP, r6
767	mtspr SPRN_CSIGR, r7
768	mtspr SPRN_TACR, r8
769	nop
770	8:
771
772	ld r5, VCPU_SPRG0(r4)
773	ld r6, VCPU_SPRG1(r4)
774	ld r7, VCPU_SPRG2(r4)
775	ld r8, VCPU_SPRG3(r4)
776	mtspr SPRN_SPRG0, r5
777	mtspr SPRN_SPRG1, r6
778	mtspr SPRN_SPRG2, r7
779	mtspr SPRN_SPRG3, r8
780
781	/* Load up DAR and DSISR */
782	ld r5, VCPU_DAR(r4)
783	lwz r6, VCPU_DSISR(r4)
784	mtspr SPRN_DAR, r5
785	mtspr SPRN_DSISR, r6
786
787	/* Restore AMR and UAMOR, set AMOR to all 1s */
788	ld r5,VCPU_AMR(r4)
789	ld r6,VCPU_UAMOR(r4)
790	mtspr SPRN_AMR,r5
791	mtspr SPRN_UAMOR,r6
792
793	/* Restore state of CTRL run bit; the host currently has it set to 1 */
794	lwz r5,VCPU_CTRL(r4)
795	andi. r5,r5,1
796	bne 4f
797	li r6,0
798	mtspr SPRN_CTRLT,r6
799	4:
800	/* Secondary threads wait for primary to have done partition switch */
801	ld r5, HSTATE_KVM_VCORE(r13)
802	lbz r6, HSTATE_PTID(r13)
803	cmpwi r6, 0
804	beq 21f
805	lbz r0, VCORE_IN_GUEST(r5)
806	cmpwi r0, 0
807	bne 21f
808	HMT_LOW
809	20: lwz r3, VCORE_ENTRY_EXIT(r5)
810	cmpwi r3, 0x100
811	bge no_switch_exit
812	lbz r0, VCORE_IN_GUEST(r5)
813	cmpwi r0, 0
814	beq 20b
815	HMT_MEDIUM
816	21:
817	/* Set LPCR. */
818	ld r8,VCORE_LPCR(r5)
819	mtspr SPRN_LPCR,r8
820	isync
821
822	/*
823	* Set the decrementer to the guest decrementer.
824	*/
825	ld r8,VCPU_DEC_EXPIRES(r4)
826	mftb r7
827	subf r3,r7,r8
828	mtspr SPRN_DEC,r3
829
830	/* Check if HDEC expires soon */
831	mfspr r3, SPRN_HDEC
832	extsw r3, r3
833	cmpdi r3, 512 /* 1 microsecond */
834	blt hdec_soon
835
836	/* Clear out and reload the SLB */
837	li r6, 0
838	slbmte r6, r6
839	PPC_SLBIA(6)
840	ptesync
841
842	/* Load up guest SLB entries (N.B. slb_max will be 0 for radix) */
843	lwz r5,VCPU_SLB_MAX(r4)
844	cmpwi r5,0
845	beq 9f
846	mtctr r5
847	addi r6,r4,VCPU_SLB
848	1: ld r8,VCPU_SLB_E(r6)
849	ld r9,VCPU_SLB_V(r6)
850	slbmte r9,r8
851	addi r6,r6,VCPU_SLB_SIZE
852	bdnz 1b
853	9:
854
855	deliver_guest_interrupt: /* r4 = vcpu, r13 = paca */
856	/* Check if we can deliver an external or decrementer interrupt now */
857	ld r0, VCPU_PENDING_EXC(r4)
858	cmpdi r0, 0
859	beq 71f
860	mr r3, r4
861	bl CFUNC(kvmppc_guest_entry_inject_int)
862	ld r4, HSTATE_KVM_VCPU(r13)
863	71:
864	ld r6, VCPU_SRR0(r4)
865	ld r7, VCPU_SRR1(r4)
866	mtspr SPRN_SRR0, r6
867	mtspr SPRN_SRR1, r7
868
869	ld r10, VCPU_PC(r4)
870	ld r11, VCPU_MSR(r4)
871	/* r11 = vcpu->arch.msr & ~MSR_HV */
872	rldicl r11, r11, 63 - MSR_HV_LG, 1
873	rotldi r11, r11, 1 + MSR_HV_LG
874	ori r11, r11, MSR_ME
875
876	ld r6, VCPU_CTR(r4)
877	ld r7, VCPU_XER(r4)
878	mtctr r6
879	mtxer r7
880
881	/*
882	* Required state:
883	* R4 = vcpu
884	* R10: value for HSRR0
885	* R11: value for HSRR1
886	* R13 = PACA
887	*/
888	fast_guest_return:
889	li r0,0
890	stb r0,VCPU_CEDED(r4) /* cancel cede */
891	mtspr SPRN_HSRR0,r10
892	mtspr SPRN_HSRR1,r11
893
894	/* Activate guest mode, so faults get handled by KVM */
895	li r9, KVM_GUEST_MODE_GUEST_HV
896	stb r9, HSTATE_IN_GUEST(r13)
897
898	#ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING
899	/* Accumulate timing */
900	addi r3, r4, VCPU_TB_GUEST
901	bl kvmhv_accumulate_time
902	#endif
903
904	/* Enter guest */
905
906	BEGIN_FTR_SECTION
907	ld r5, VCPU_CFAR(r4)
908	mtspr SPRN_CFAR, r5
909	END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
910	BEGIN_FTR_SECTION
911	ld r0, VCPU_PPR(r4)
912	END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
913
914	ld r5, VCPU_LR(r4)
915	mtlr r5
916
917	ld r1, VCPU_GPR(R1)(r4)
918	ld r5, VCPU_GPR(R5)(r4)
919	ld r8, VCPU_GPR(R8)(r4)
920	ld r9, VCPU_GPR(R9)(r4)
921	ld r10, VCPU_GPR(R10)(r4)
922	ld r11, VCPU_GPR(R11)(r4)
923	ld r12, VCPU_GPR(R12)(r4)
924	ld r13, VCPU_GPR(R13)(r4)
925
926	BEGIN_FTR_SECTION
927	mtspr SPRN_PPR, r0
928	END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
929
930	ld r6, VCPU_GPR(R6)(r4)
931	ld r7, VCPU_GPR(R7)(r4)
932
933	ld r0, VCPU_CR(r4)
934	mtcr r0
935
936	ld r0, VCPU_GPR(R0)(r4)
937	ld r2, VCPU_GPR(R2)(r4)
938	ld r3, VCPU_GPR(R3)(r4)
939	ld r4, VCPU_GPR(R4)(r4)
940	HRFI_TO_GUEST
941	b .
942	SYM_CODE_END(kvmppc_hv_entry)
943
944	secondary_too_late:
945	li r12, 0
946	stw r12, STACK_SLOT_TRAP(r1)
947	cmpdi r4, 0
948	beq 11f
949	stw r12, VCPU_TRAP(r4)
950	#ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING
951	addi r3, r4, VCPU_TB_RMEXIT
952	bl kvmhv_accumulate_time
953	#endif
954	11: b kvmhv_switch_to_host
955
956	no_switch_exit:
957	HMT_MEDIUM
958	li r12, 0
959	b 12f
960	hdec_soon:
961	li r12, BOOK3S_INTERRUPT_HV_DECREMENTER
962	12: stw r12, VCPU_TRAP(r4)
963	mr r9, r4
964	#ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING
965	addi r3, r4, VCPU_TB_RMEXIT
966	bl kvmhv_accumulate_time
967	#endif
968	b guest_bypass
969
970	/******************************************************************************
971	* *
972	* Exit code *
973	* *
974	*****************************************************************************/
975
976	/*
977	* We come here from the first-level interrupt handlers.
978	*/
979	.globl kvmppc_interrupt_hv
980	kvmppc_interrupt_hv:
981	/*
982	* Register contents:
983	* R9 = HSTATE_IN_GUEST
984	* R12 = (guest CR << 32) | interrupt vector
985	* R13 = PACA
986	* guest R12 saved in shadow VCPU SCRATCH0
987	* guest R13 saved in SPRN_SCRATCH0
988	* guest R9 saved in HSTATE_SCRATCH2
989	*/
990	/* We're now back in the host but in guest MMU context */
991	cmpwi r9,KVM_GUEST_MODE_HOST_HV
992	beq kvmppc_bad_host_intr
993	li r9, KVM_GUEST_MODE_HOST_HV
994	stb r9, HSTATE_IN_GUEST(r13)
995
996	ld r9, HSTATE_KVM_VCPU(r13)
997
998	/* Save registers */
999
1000	std r0, VCPU_GPR(R0)(r9)
1001	std r1, VCPU_GPR(R1)(r9)
1002	std r2, VCPU_GPR(R2)(r9)
1003	std r3, VCPU_GPR(R3)(r9)
1004	std r4, VCPU_GPR(R4)(r9)
1005	std r5, VCPU_GPR(R5)(r9)
1006	std r6, VCPU_GPR(R6)(r9)
1007	std r7, VCPU_GPR(R7)(r9)
1008	std r8, VCPU_GPR(R8)(r9)
1009	ld r0, HSTATE_SCRATCH2(r13)
1010	std r0, VCPU_GPR(R9)(r9)
1011	std r10, VCPU_GPR(R10)(r9)
1012	std r11, VCPU_GPR(R11)(r9)
1013	ld r3, HSTATE_SCRATCH0(r13)
1014	std r3, VCPU_GPR(R12)(r9)
1015	/* CR is in the high half of r12 */
1016	srdi r4, r12, 32
1017	std r4, VCPU_CR(r9)
1018	BEGIN_FTR_SECTION
1019	ld r3, HSTATE_CFAR(r13)
1020	std r3, VCPU_CFAR(r9)
1021	END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
1022	BEGIN_FTR_SECTION
1023	ld r4, HSTATE_PPR(r13)
1024	std r4, VCPU_PPR(r9)
1025	END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
1026
1027	/* Restore R1/R2 so we can handle faults */
1028	ld r1, HSTATE_HOST_R1(r13)
1029	LOAD_PACA_TOC()
1030
1031	mfspr r10, SPRN_SRR0
1032	mfspr r11, SPRN_SRR1
1033	std r10, VCPU_SRR0(r9)
1034	std r11, VCPU_SRR1(r9)
1035	/* trap is in the low half of r12, clear CR from the high half */
1036	clrldi r12, r12, 32
1037	andi. r0, r12, 2 /* need to read HSRR0/1? */
1038	beq 1f
1039	mfspr r10, SPRN_HSRR0
1040	mfspr r11, SPRN_HSRR1
1041	clrrdi r12, r12, 2
1042	1: std r10, VCPU_PC(r9)
1043	std r11, VCPU_MSR(r9)
1044
1045	GET_SCRATCH0(r3)
1046	mflr r4
1047	std r3, VCPU_GPR(R13)(r9)
1048	std r4, VCPU_LR(r9)
1049
1050	stw r12,VCPU_TRAP(r9)
1051
1052	/*
1053	* Now that we have saved away SRR0/1 and HSRR0/1,
1054	* interrupts are recoverable in principle, so set MSR_RI.
1055	* This becomes important for relocation-on interrupts from
1056	* the guest, which we can get in radix mode on POWER9.
1057	*/
1058	li r0, MSR_RI
1059	mtmsrd r0, 1
1060
1061	#ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING
1062	addi r3, r9, VCPU_TB_RMINTR
1063	mr r4, r9
1064	bl kvmhv_accumulate_time
1065	ld r5, VCPU_GPR(R5)(r9)
1066	ld r6, VCPU_GPR(R6)(r9)
1067	ld r7, VCPU_GPR(R7)(r9)
1068	ld r8, VCPU_GPR(R8)(r9)
1069	#endif
1070
1071	/* Save HEIR (HV emulation assist reg) in emul_inst
1072	if this is an HEI (HV emulation interrupt, e40) */
1073	li r3,KVM_INST_FETCH_FAILED
1074	std r3,VCPU_LAST_INST(r9)
1075	cmpwi r12,BOOK3S_INTERRUPT_H_EMUL_ASSIST
1076	bne 11f
1077	mfspr r3,SPRN_HEIR
1078	11: std r3,VCPU_HEIR(r9)
1079
1080	/* these are volatile across C function calls */
1081	mfctr r3
1082	mfxer r4
1083	std r3, VCPU_CTR(r9)
1084	std r4, VCPU_XER(r9)
1085
1086	/* Save more register state */
1087	mfdar r3
1088	mfdsisr r4
1089	std r3, VCPU_DAR(r9)
1090	stw r4, VCPU_DSISR(r9)
1091
1092	/* If this is a page table miss then see if it's theirs or ours */
1093	cmpwi r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
1094	beq kvmppc_hdsi
1095	std r3, VCPU_FAULT_DAR(r9)
1096	stw r4, VCPU_FAULT_DSISR(r9)
1097	cmpwi r12, BOOK3S_INTERRUPT_H_INST_STORAGE
1098	beq kvmppc_hisi
1099
1100	/* See if this is a leftover HDEC interrupt */
1101	cmpwi r12,BOOK3S_INTERRUPT_HV_DECREMENTER
1102	bne 2f
1103	mfspr r3,SPRN_HDEC
1104	extsw r3, r3
1105	cmpdi r3,0
1106	mr r4,r9
1107	bge fast_guest_return
1108	2:
1109	/* See if this is an hcall we can handle in real mode */
1110	cmpwi r12,BOOK3S_INTERRUPT_SYSCALL
1111	beq hcall_try_real_mode
1112
1113	/* Hypervisor doorbell - exit only if host IPI flag set */
1114	cmpwi r12, BOOK3S_INTERRUPT_H_DOORBELL
1115	bne 3f
1116	lbz r0, HSTATE_HOST_IPI(r13)
1117	cmpwi r0, 0
1118	beq maybe_reenter_guest
1119	b guest_exit_cont
1120	3:
1121	/* If it's a hypervisor facility unavailable interrupt, save HFSCR */
1122	cmpwi r12, BOOK3S_INTERRUPT_H_FAC_UNAVAIL
1123	bne 14f
1124	mfspr r3, SPRN_HFSCR
1125	std r3, VCPU_HFSCR(r9)
1126	b guest_exit_cont
1127	14:
1128	/* External interrupt ? */
1129	cmpwi r12, BOOK3S_INTERRUPT_EXTERNAL
1130	beq kvmppc_guest_external
1131	/* See if it is a machine check */
1132	cmpwi r12, BOOK3S_INTERRUPT_MACHINE_CHECK
1133	beq machine_check_realmode
1134	/* Or a hypervisor maintenance interrupt */
1135	cmpwi r12, BOOK3S_INTERRUPT_HMI
1136	beq hmi_realmode
1137
1138	guest_exit_cont: /* r9 = vcpu, r12 = trap, r13 = paca */
1139
1140	#ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING
1141	addi r3, r9, VCPU_TB_RMEXIT
1142	mr r4, r9
1143	bl kvmhv_accumulate_time
1144	#endif
1145
1146	/*
1147	* Possibly flush the link stack here, before we do a blr in
1148	* kvmhv_switch_to_host.
1149	*/
1150	1: nop
1151	patch_site 1b patch__call_kvm_flush_link_stack
1152
1153	/* For hash guest, read the guest SLB and save it away */
1154	li r5, 0
1155	lwz r0,VCPU_SLB_NR(r9) /* number of entries in SLB */
1156	mtctr r0
1157	li r6,0
1158	addi r7,r9,VCPU_SLB
1159	1: slbmfee r8,r6
1160	andis. r0,r8,SLB_ESID_V@h
1161	beq 2f
1162	add r8,r8,r6 /* put index in */
1163	slbmfev r3,r6
1164	std r8,VCPU_SLB_E(r7)
1165	std r3,VCPU_SLB_V(r7)
1166	addi r7,r7,VCPU_SLB_SIZE
1167	addi r5,r5,1
1168	2: addi r6,r6,1
1169	bdnz 1b
1170	/* Finally clear out the SLB */
1171	li r0,0
1172	slbmte r0,r0
1173	PPC_SLBIA(6)
1174	ptesync
1175	stw r5,VCPU_SLB_MAX(r9)
1176
1177	/* load host SLB entries */
1178	ld r8,PACA_SLBSHADOWPTR(r13)
1179
1180	.rept SLB_NUM_BOLTED
1181	li r3, SLBSHADOW_SAVEAREA
1182	LDX_BE r5, r8, r3
1183	addi r3, r3, 8
1184	LDX_BE r6, r8, r3
1185	andis. r7,r5,SLB_ESID_V@h
1186	beq 1f
1187	slbmte r6,r5
1188	1: addi r8,r8,16
1189	.endr
1190
1191	guest_bypass:
1192	stw r12, STACK_SLOT_TRAP(r1)
1193
1194	/* Save DEC */
1195	/* Do this before kvmhv_commence_exit so we know TB is guest TB */
1196	ld r3, HSTATE_KVM_VCORE(r13)
1197	mfspr r5,SPRN_DEC
1198	mftb r6
1199	extsw r5,r5
1200	16: add r5,r5,r6
1201	std r5,VCPU_DEC_EXPIRES(r9)
1202
1203	/* Increment exit count, poke other threads to exit */
1204	mr r3, r12
1205	bl kvmhv_commence_exit
1206	nop
1207	ld r9, HSTATE_KVM_VCPU(r13)
1208
1209	/* Stop others sending VCPU interrupts to this physical CPU */
1210	li r0, -1
1211	stw r0, VCPU_CPU(r9)
1212	stw r0, VCPU_THREAD_CPU(r9)
1213
1214	/* Save guest CTRL register, set runlatch to 1 if it was clear */
1215	mfspr r6,SPRN_CTRLF
1216	stw r6,VCPU_CTRL(r9)
1217	andi. r0,r6,1
1218	bne 4f
1219	li r6,1
1220	mtspr SPRN_CTRLT,r6
1221	4:
1222	/*
1223	* Save the guest PURR/SPURR
1224	*/
1225	mfspr r5,SPRN_PURR
1226	mfspr r6,SPRN_SPURR
1227	ld r7,VCPU_PURR(r9)
1228	ld r8,VCPU_SPURR(r9)
1229	std r5,VCPU_PURR(r9)
1230	std r6,VCPU_SPURR(r9)
1231	subf r5,r7,r5
1232	subf r6,r8,r6
1233
1234	/*
1235	* Restore host PURR/SPURR and add guest times
1236	* so that the time in the guest gets accounted.
1237	*/
1238	ld r3,HSTATE_PURR(r13)
1239	ld r4,HSTATE_SPURR(r13)
1240	add r3,r3,r5
1241	add r4,r4,r6
1242	mtspr SPRN_PURR,r3
1243	mtspr SPRN_SPURR,r4
1244
1245	BEGIN_FTR_SECTION
1246	b 8f
1247	END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
1248	/* Save POWER8-specific registers */
1249	mfspr r5, SPRN_IAMR
1250	mfspr r6, SPRN_PSPB
1251	mfspr r7, SPRN_FSCR
1252	std r5, VCPU_IAMR(r9)
1253	stw r6, VCPU_PSPB(r9)
1254	std r7, VCPU_FSCR(r9)
1255	mfspr r5, SPRN_IC
1256	mfspr r7, SPRN_TAR
1257	std r5, VCPU_IC(r9)
1258	std r7, VCPU_TAR(r9)
1259	mfspr r8, SPRN_EBBHR
1260	std r8, VCPU_EBBHR(r9)
1261	mfspr r5, SPRN_EBBRR
1262	mfspr r6, SPRN_BESCR
1263	mfspr r7, SPRN_PID
1264	mfspr r8, SPRN_WORT
1265	std r5, VCPU_EBBRR(r9)
1266	std r6, VCPU_BESCR(r9)
1267	stw r7, VCPU_GUEST_PID(r9)
1268	std r8, VCPU_WORT(r9)
1269	mfspr r5, SPRN_TCSCR
1270	mfspr r6, SPRN_ACOP
1271	mfspr r7, SPRN_CSIGR
1272	mfspr r8, SPRN_TACR
1273	std r5, VCPU_TCSCR(r9)
1274	std r6, VCPU_ACOP(r9)
1275	std r7, VCPU_CSIGR(r9)
1276	std r8, VCPU_TACR(r9)
1277	BEGIN_FTR_SECTION
1278	ld r5, STACK_SLOT_FSCR(r1)
1279	mtspr SPRN_FSCR, r5
1280	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1281	/*
1282	* Restore various registers to 0, where non-zero values
1283	* set by the guest could disrupt the host.
1284	*/
1285	li r0, 0
1286	mtspr SPRN_PSPB, r0
1287	mtspr SPRN_WORT, r0
1288	mtspr SPRN_TCSCR, r0
1289	/* Set MMCRS to 1<<31 to freeze and disable the SPMC counters */
1290	li r0, 1
1291	sldi r0, r0, 31
1292	mtspr SPRN_MMCRS, r0
1293
1294	/* Save and restore AMR, IAMR and UAMOR before turning on the MMU */
1295	ld r8, STACK_SLOT_IAMR(r1)
1296	mtspr SPRN_IAMR, r8
1297
1298	8: /* Power7 jumps back in here */
1299	mfspr r5,SPRN_AMR
1300	mfspr r6,SPRN_UAMOR
1301	std r5,VCPU_AMR(r9)
1302	std r6,VCPU_UAMOR(r9)
1303	ld r5,STACK_SLOT_AMR(r1)
1304	ld r6,STACK_SLOT_UAMOR(r1)
1305	mtspr SPRN_AMR, r5
1306	mtspr SPRN_UAMOR, r6
1307
1308	/* Switch DSCR back to host value */
1309	mfspr r8, SPRN_DSCR
1310	ld r7, HSTATE_DSCR(r13)
1311	std r8, VCPU_DSCR(r9)
1312	mtspr SPRN_DSCR, r7
1313
1314	/* Save non-volatile GPRs */
1315	std r14, VCPU_GPR(R14)(r9)
1316	std r15, VCPU_GPR(R15)(r9)
1317	std r16, VCPU_GPR(R16)(r9)
1318	std r17, VCPU_GPR(R17)(r9)
1319	std r18, VCPU_GPR(R18)(r9)
1320	std r19, VCPU_GPR(R19)(r9)
1321	std r20, VCPU_GPR(R20)(r9)
1322	std r21, VCPU_GPR(R21)(r9)
1323	std r22, VCPU_GPR(R22)(r9)
1324	std r23, VCPU_GPR(R23)(r9)
1325	std r24, VCPU_GPR(R24)(r9)
1326	std r25, VCPU_GPR(R25)(r9)
1327	std r26, VCPU_GPR(R26)(r9)
1328	std r27, VCPU_GPR(R27)(r9)
1329	std r28, VCPU_GPR(R28)(r9)
1330	std r29, VCPU_GPR(R29)(r9)
1331	std r30, VCPU_GPR(R30)(r9)
1332	std r31, VCPU_GPR(R31)(r9)
1333
1334	/* Save SPRGs */
1335	mfspr r3, SPRN_SPRG0
1336	mfspr r4, SPRN_SPRG1
1337	mfspr r5, SPRN_SPRG2
1338	mfspr r6, SPRN_SPRG3
1339	std r3, VCPU_SPRG0(r9)
1340	std r4, VCPU_SPRG1(r9)
1341	std r5, VCPU_SPRG2(r9)
1342	std r6, VCPU_SPRG3(r9)
1343
1344	/* save FP state */
1345	mr r3, r9
1346	bl kvmppc_save_fp
1347
1348	#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1349	BEGIN_FTR_SECTION
1350	b 91f
1351	END_FTR_SECTION_IFCLR(CPU_FTR_TM)
1352	/*
1353	* NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR)
1354	*/
1355	mr r3, r9
1356	ld r4, VCPU_MSR(r3)
1357	li r5, 0 /* don't preserve non-vol regs */
1358	bl kvmppc_save_tm_hv
1359	nop
1360	ld r9, HSTATE_KVM_VCPU(r13)
1361	91:
1362	#endif
1363
1364	/* Increment yield count if they have a VPA */
1365	ld r8, VCPU_VPA(r9) /* do they have a VPA? */
1366	cmpdi r8, 0
1367	beq 25f
1368	li r4, LPPACA_YIELDCOUNT
1369	LWZX_BE r3, r8, r4
1370	addi r3, r3, 1
1371	STWX_BE r3, r8, r4
1372	li r3, 1
1373	stb r3, VCPU_VPA_DIRTY(r9)
1374	25:
1375	/* Save PMU registers if requested */
1376	/* r8 and cr0.eq are live here */
1377	mr r3, r9
1378	li r4, 1
1379	beq 21f /* if no VPA, save PMU stuff anyway */
1380	lbz r4, LPPACA_PMCINUSE(r8)
1381	21: bl kvmhv_save_guest_pmu
1382	ld r9, HSTATE_KVM_VCPU(r13)
1383
1384	/* Restore host values of some registers */
1385	BEGIN_FTR_SECTION
1386	ld r5, STACK_SLOT_CIABR(r1)
1387	ld r6, STACK_SLOT_DAWR0(r1)
1388	ld r7, STACK_SLOT_DAWRX0(r1)
1389	mtspr SPRN_CIABR, r5
1390	/*
1391	* If the DAWR doesn't work, it's ok to write these here as
1392	* this value should always be zero
1393	*/
1394	mtspr SPRN_DAWR0, r6
1395	mtspr SPRN_DAWRX0, r7
1396	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1397
1398	/*
1399	* POWER7/POWER8 guest -> host partition switch code.
1400	* We don't have to lock against tlbies but we do
1401	* have to coordinate the hardware threads.
1402	* Here STACK_SLOT_TRAP(r1) contains the trap number.
1403	*/
1404	kvmhv_switch_to_host:
1405	/* Secondary threads wait for primary to do partition switch */
1406	ld r5,HSTATE_KVM_VCORE(r13)
1407	ld r4,VCORE_KVM(r5) /* pointer to struct kvm */
1408	lbz r3,HSTATE_PTID(r13)
1409	cmpwi r3,0
1410	beq 15f
1411	HMT_LOW
1412	13: lbz r3,VCORE_IN_GUEST(r5)
1413	cmpwi r3,0
1414	bne 13b
1415	HMT_MEDIUM
1416	b 16f
1417
1418	/* Primary thread waits for all the secondaries to exit guest */
1419	15: lwz r3,VCORE_ENTRY_EXIT(r5)
1420	rlwinm r0,r3,32-8,0xff
1421	clrldi r3,r3,56
1422	cmpw r3,r0
1423	bne 15b
1424	isync
1425
1426	/* Did we actually switch to the guest at all? */
1427	lbz r6, VCORE_IN_GUEST(r5)
1428	cmpwi r6, 0
1429	beq 19f
1430
1431	/* Primary thread switches back to host partition */
1432	lwz r7,KVM_HOST_LPID(r4)
1433	ld r6,KVM_HOST_SDR1(r4)
1434	li r8,LPID_RSVD /* switch to reserved LPID */
1435	mtspr SPRN_LPID,r8
1436	ptesync
1437	mtspr SPRN_SDR1,r6 /* switch to host page table */
1438	mtspr SPRN_LPID,r7
1439	isync
1440
1441	BEGIN_FTR_SECTION
1442	/* DPDES and VTB are shared between threads */
1443	mfspr r7, SPRN_DPDES
1444	mfspr r8, SPRN_VTB
1445	std r7, VCORE_DPDES(r5)
1446	std r8, VCORE_VTB(r5)
1447	/* clear DPDES so we don't get guest doorbells in the host */
1448	li r8, 0
1449	mtspr SPRN_DPDES, r8
1450	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1451
1452	/* Subtract timebase offset from timebase */
1453	ld r8, VCORE_TB_OFFSET_APPL(r5)
1454	cmpdi r8,0
1455	beq 17f
1456	li r0, 0
1457	std r0, VCORE_TB_OFFSET_APPL(r5)
1458	mftb r6 /* current guest timebase */
1459	subf r8,r8,r6
1460	mtspr SPRN_TBU40,r8 /* update upper 40 bits */
1461	mftb r7 /* check if lower 24 bits overflowed */
1462	clrldi r6,r6,40
1463	clrldi r7,r7,40
1464	cmpld r7,r6
1465	bge 17f
1466	addis r8,r8,0x100 /* if so, increment upper 40 bits */
1467	mtspr SPRN_TBU40,r8
1468
1469	17:
1470	/*
1471	* If this is an HMI, we called kvmppc_realmode_hmi_handler
1472	* above, which may or may not have already called
1473	* kvmppc_subcore_exit_guest. Fortunately, all that
1474	* kvmppc_subcore_exit_guest does is clear a flag, so calling
1475	* it again here is benign even if kvmppc_realmode_hmi_handler
1476	* has already called it.
1477	*/
1478	bl kvmppc_subcore_exit_guest
1479	nop
1480	30: ld r5,HSTATE_KVM_VCORE(r13)
1481	ld r4,VCORE_KVM(r5) /* pointer to struct kvm */
1482
1483	/* Reset PCR */
1484	ld r0, VCORE_PCR(r5)
1485	LOAD_REG_IMMEDIATE(r6, PCR_MASK)
1486	cmpld r0, r6
1487	beq 18f
1488	mtspr SPRN_PCR, r6
1489	18:
1490	/* Signal secondary CPUs to continue */
1491	li r0, 0
1492	stb r0,VCORE_IN_GUEST(r5)
1493	19: lis r8,0x7fff /* MAX_INT@h */
1494	mtspr SPRN_HDEC,r8
1495
1496	16: ld r8,KVM_HOST_LPCR(r4)
1497	mtspr SPRN_LPCR,r8
1498	isync
1499
1500	#ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING
1501	/* Finish timing, if we have a vcpu */
1502	ld r4, HSTATE_KVM_VCPU(r13)
1503	cmpdi r4, 0
1504	li r3, 0
1505	beq 2f
1506	bl kvmhv_accumulate_time
1507	2:
1508	#endif
1509	/* Unset guest mode */
1510	li r0, KVM_GUEST_MODE_NONE
1511	stb r0, HSTATE_IN_GUEST(r13)
1512
1513	lwz r12, STACK_SLOT_TRAP(r1) /* return trap # in r12 */
1514	ld r0, SFS+PPC_LR_STKOFF(r1)
1515	addi r1, r1, SFS
1516	mtlr r0
1517	blr
1518
1519	.balign 32
1520	.global kvm_flush_link_stack
1521	kvm_flush_link_stack:
1522	/* Save LR into r0 */
1523	mflr r0
1524
1525	/* Flush the link stack. On Power8 it's up to 32 entries in size. */
1526	.rept 32
1527	ANNOTATE_INTRA_FUNCTION_CALL
1528	bl .+4
1529	.endr
1530
1531	/* And on Power9 it's up to 64. */
1532	BEGIN_FTR_SECTION
1533	.rept 32
1534	ANNOTATE_INTRA_FUNCTION_CALL
1535	bl .+4
1536	.endr
1537	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
1538
1539	/* Restore LR */
1540	mtlr r0
1541	blr
1542
1543	kvmppc_guest_external:
1544	/* External interrupt, first check for host_ipi. If this is
1545	* set, we know the host wants us out so let's do it now
1546	*/
1547	bl CFUNC(kvmppc_read_intr)
1548
1549	/*
1550	* Restore the active volatile registers after returning from
1551	* a C function.
1552	*/
1553	ld r9, HSTATE_KVM_VCPU(r13)
1554	li r12, BOOK3S_INTERRUPT_EXTERNAL
1555
1556	/*
1557	* kvmppc_read_intr return codes:
1558	*
1559	* Exit to host (r3 > 0)
1560	* 1 An interrupt is pending that needs to be handled by the host
1561	* Exit guest and return to host by branching to guest_exit_cont
1562	*
1563	* 2 Passthrough that needs completion in the host
1564	* Exit guest and return to host by branching to guest_exit_cont
1565	* However, we also set r12 to BOOK3S_INTERRUPT_HV_RM_HARD
1566	* to indicate to the host to complete handling the interrupt
1567	*
1568	* Before returning to guest, we check if any CPU is heading out
1569	* to the host and if so, we head out also. If no CPUs are heading
1570	* check return values <= 0.
1571	*
1572	* Return to guest (r3 <= 0)
1573	* 0 No external interrupt is pending
1574	* -1 A guest wakeup IPI (which has now been cleared)
1575	* In either case, we return to guest to deliver any pending
1576	* guest interrupts.
1577	*
1578	* -2 A PCI passthrough external interrupt was handled
1579	* (interrupt was delivered directly to guest)
1580	* Return to guest to deliver any pending guest interrupts.
1581	*/
1582
1583	cmpdi r3, 1
1584	ble 1f
1585
1586	/* Return code = 2 */
1587	li r12, BOOK3S_INTERRUPT_HV_RM_HARD
1588	stw r12, VCPU_TRAP(r9)
1589	b guest_exit_cont
1590
1591	1: /* Return code <= 1 */
1592	cmpdi r3, 0
1593	bgt guest_exit_cont
1594
1595	/* Return code <= 0 */
1596	maybe_reenter_guest:
1597	ld r5, HSTATE_KVM_VCORE(r13)
1598	lwz r0, VCORE_ENTRY_EXIT(r5)
1599	cmpwi r0, 0x100
1600	mr r4, r9
1601	blt deliver_guest_interrupt
1602	b guest_exit_cont
1603
1604	/*
1605	* Check whether an HDSI is an HPTE not found fault or something else.
1606	* If it is an HPTE not found fault that is due to the guest accessing
1607	* a page that they have mapped but which we have paged out, then
1608	* we continue on with the guest exit path. In all other cases,
1609	* reflect the HDSI to the guest as a DSI.
1610	*/
1611	kvmppc_hdsi:
1612	mfspr r4, SPRN_HDAR
1613	mfspr r6, SPRN_HDSISR
1614	/* HPTE not found fault or protection fault? */
1615	andis. r0, r6, (DSISR_NOHPTE | DSISR_PROTFAULT)@h
1616	beq 1f /* if not, send it to the guest */
1617	andi. r0, r11, MSR_DR /* data relocation enabled? */
1618	beq 3f
1619	clrrdi r0, r4, 28
1620	PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */
1621	li r0, BOOK3S_INTERRUPT_DATA_SEGMENT
1622	bne 7f /* if no SLB entry found */
1623	4: std r4, VCPU_FAULT_DAR(r9)
1624	stw r6, VCPU_FAULT_DSISR(r9)
1625
1626	/* Search the hash table. */
1627	mr r3, r9 /* vcpu pointer */
1628	li r7, 1 /* data fault */
1629	bl CFUNC(kvmppc_hpte_hv_fault)
1630	ld r9, HSTATE_KVM_VCPU(r13)
1631	ld r10, VCPU_PC(r9)
1632	ld r11, VCPU_MSR(r9)
1633	li r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
1634	cmpdi r3, 0 /* retry the instruction */
1635	beq 6f
1636	cmpdi r3, -1 /* handle in kernel mode */
1637	beq guest_exit_cont
1638	cmpdi r3, -2 /* MMIO emulation; need instr word */
1639	beq 2f
1640
1641	/* Synthesize a DSI (or DSegI) for the guest */
1642	ld r4, VCPU_FAULT_DAR(r9)
1643	mr r6, r3
1644	1: li r0, BOOK3S_INTERRUPT_DATA_STORAGE
1645	mtspr SPRN_DSISR, r6
1646	7: mtspr SPRN_DAR, r4
1647	mtspr SPRN_SRR0, r10
1648	mtspr SPRN_SRR1, r11
1649	mr r10, r0
1650	bl kvmppc_msr_interrupt
1651	fast_interrupt_c_return:
1652	6: ld r7, VCPU_CTR(r9)
1653	ld r8, VCPU_XER(r9)
1654	mtctr r7
1655	mtxer r8
1656	mr r4, r9
1657	b fast_guest_return
1658
1659	3: ld r5, VCPU_KVM(r9) /* not relocated, use VRMA */
1660	ld r5, KVM_VRMA_SLB_V(r5)
1661	b 4b
1662
1663	/* If this is for emulated MMIO, load the instruction word */
1664	2: li r8, KVM_INST_FETCH_FAILED /* In case lwz faults */
1665
1666	/* Set guest mode to 'jump over instruction' so if lwz faults
1667	* we'll just continue at the next IP. */
1668	li r0, KVM_GUEST_MODE_SKIP
1669	stb r0, HSTATE_IN_GUEST(r13)
1670
1671	/* Do the access with MSR:DR enabled */
1672	mfmsr r3
1673	ori r4, r3, MSR_DR /* Enable paging for data */
1674	mtmsrd r4
1675	lwz r8, 0(r10)
1676	mtmsrd r3
1677
1678	/* Store the result */
1679	std r8, VCPU_LAST_INST(r9)
1680
1681	/* Unset guest mode. */
1682	li r0, KVM_GUEST_MODE_HOST_HV
1683	stb r0, HSTATE_IN_GUEST(r13)
1684	b guest_exit_cont
1685
1686	/*
1687	* Similarly for an HISI, reflect it to the guest as an ISI unless
1688	* it is an HPTE not found fault for a page that we have paged out.
1689	*/
1690	kvmppc_hisi:
1691	andis. r0, r11, SRR1_ISI_NOPT@h
1692	beq 1f
1693	andi. r0, r11, MSR_IR /* instruction relocation enabled? */
1694	beq 3f
1695	clrrdi r0, r10, 28
1696	PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */
1697	li r0, BOOK3S_INTERRUPT_INST_SEGMENT
1698	bne 7f /* if no SLB entry found */
1699	4:
1700	/* Search the hash table. */
1701	mr r3, r9 /* vcpu pointer */
1702	mr r4, r10
1703	mr r6, r11
1704	li r7, 0 /* instruction fault */
1705	bl CFUNC(kvmppc_hpte_hv_fault)
1706	ld r9, HSTATE_KVM_VCPU(r13)
1707	ld r10, VCPU_PC(r9)
1708	ld r11, VCPU_MSR(r9)
1709	li r12, BOOK3S_INTERRUPT_H_INST_STORAGE
1710	cmpdi r3, 0 /* retry the instruction */
1711	beq fast_interrupt_c_return
1712	cmpdi r3, -1 /* handle in kernel mode */
1713	beq guest_exit_cont
1714
1715	/* Synthesize an ISI (or ISegI) for the guest */
1716	mr r11, r3
1717	1: li r0, BOOK3S_INTERRUPT_INST_STORAGE
1718	7: mtspr SPRN_SRR0, r10
1719	mtspr SPRN_SRR1, r11
1720	mr r10, r0
1721	bl kvmppc_msr_interrupt
1722	b fast_interrupt_c_return
1723
1724	3: ld r6, VCPU_KVM(r9) /* not relocated, use VRMA */
1725	ld r5, KVM_VRMA_SLB_V(r6)
1726	b 4b
1727
1728	/*
1729	* Try to handle an hcall in real mode.
1730	* Returns to the guest if we handle it, or continues on up to
1731	* the kernel if we can't (i.e. if we don't have a handler for
1732	* it, or if the handler returns H_TOO_HARD).
1733	*
1734	* r5 - r8 contain hcall args,
1735	* r9 = vcpu, r10 = pc, r11 = msr, r12 = trap, r13 = paca
1736	*/
1737	hcall_try_real_mode:
1738	ld r3,VCPU_GPR(R3)(r9)
1739	andi. r0,r11,MSR_PR
1740	/* sc 1 from userspace - reflect to guest syscall */
1741	bne sc_1_fast_return
1742	clrrdi r3,r3,2
1743	cmpldi r3,hcall_real_table_end - hcall_real_table
1744	bge guest_exit_cont
1745	/* See if this hcall is enabled for in-kernel handling */
1746	ld r4, VCPU_KVM(r9)
1747	srdi r0, r3, 8 /* r0 = (r3 / 4) >> 6 */
1748	sldi r0, r0, 3 /* index into kvm->arch.enabled_hcalls[] */
1749	add r4, r4, r0
1750	ld r0, KVM_ENABLED_HCALLS(r4)
1751	rlwinm r4, r3, 32-2, 0x3f /* r4 = (r3 / 4) & 0x3f */
1752	srd r0, r0, r4
1753	andi. r0, r0, 1
1754	beq guest_exit_cont
1755	/* Get pointer to handler, if any, and call it */
1756	LOAD_REG_ADDR(r4, hcall_real_table)
1757	lwax r3,r3,r4
1758	cmpwi r3,0
1759	beq guest_exit_cont
1760	add r12,r3,r4
1761	mtctr r12
1762	mr r3,r9 /* get vcpu pointer */
1763	ld r4,VCPU_GPR(R4)(r9)
1764	bctrl
1765	cmpdi r3,H_TOO_HARD
1766	beq hcall_real_fallback
1767	ld r4,HSTATE_KVM_VCPU(r13)
1768	std r3,VCPU_GPR(R3)(r4)
1769	ld r10,VCPU_PC(r4)
1770	ld r11,VCPU_MSR(r4)
1771	b fast_guest_return
1772
1773	sc_1_fast_return:
1774	mtspr SPRN_SRR0,r10
1775	mtspr SPRN_SRR1,r11
1776	li r10, BOOK3S_INTERRUPT_SYSCALL
1777	bl kvmppc_msr_interrupt
1778	mr r4,r9
1779	b fast_guest_return
1780
1781	/* We've attempted a real mode hcall, but it's punted it back
1782	* to userspace. We need to restore some clobbered volatiles
1783	* before resuming the pass-it-to-qemu path */
1784	hcall_real_fallback:
1785	li r12,BOOK3S_INTERRUPT_SYSCALL
1786	ld r9, HSTATE_KVM_VCPU(r13)
1787
1788	b guest_exit_cont
1789
1790	.globl hcall_real_table
1791	hcall_real_table:
1792	.long 0 /* 0 - unused */
1793	.long DOTSYM(kvmppc_h_remove) - hcall_real_table
1794	.long DOTSYM(kvmppc_h_enter) - hcall_real_table
1795	.long DOTSYM(kvmppc_h_read) - hcall_real_table
1796	.long DOTSYM(kvmppc_h_clear_mod) - hcall_real_table
1797	.long DOTSYM(kvmppc_h_clear_ref) - hcall_real_table
1798	.long DOTSYM(kvmppc_h_protect) - hcall_real_table
1799	.long 0 /* 0x1c */
1800	.long 0 /* 0x20 */
1801	.long 0 /* 0x24 - H_SET_SPRG0 */
1802	.long DOTSYM(kvmppc_h_set_dabr) - hcall_real_table
1803	.long DOTSYM(kvmppc_rm_h_page_init) - hcall_real_table
1804	.long 0 /* 0x30 */
1805	.long 0 /* 0x34 */
1806	.long 0 /* 0x38 */
1807	.long 0 /* 0x3c */
1808	.long 0 /* 0x40 */
1809	.long 0 /* 0x44 */
1810	.long 0 /* 0x48 */
1811	.long 0 /* 0x4c */
1812	.long 0 /* 0x50 */
1813	.long 0 /* 0x54 */
1814	.long 0 /* 0x58 */
1815	.long 0 /* 0x5c */
1816	.long 0 /* 0x60 */
1817	#ifdef CONFIG_KVM_XICS
1818	.long DOTSYM(xics_rm_h_eoi) - hcall_real_table
1819	.long DOTSYM(xics_rm_h_cppr) - hcall_real_table
1820	.long DOTSYM(xics_rm_h_ipi) - hcall_real_table
1821	.long 0 /* 0x70 - H_IPOLL */
1822	.long DOTSYM(xics_rm_h_xirr) - hcall_real_table
1823	#else
1824	.long 0 /* 0x64 - H_EOI */
1825	.long 0 /* 0x68 - H_CPPR */
1826	.long 0 /* 0x6c - H_IPI */
1827	.long 0 /* 0x70 - H_IPOLL */
1828	.long 0 /* 0x74 - H_XIRR */
1829	#endif
1830	.long 0 /* 0x78 */
1831	.long 0 /* 0x7c */
1832	.long 0 /* 0x80 */
1833	.long 0 /* 0x84 */
1834	.long 0 /* 0x88 */
1835	.long 0 /* 0x8c */
1836	.long 0 /* 0x90 */
1837	.long 0 /* 0x94 */
1838	.long 0 /* 0x98 */
1839	.long 0 /* 0x9c */
1840	.long 0 /* 0xa0 */
1841	.long 0 /* 0xa4 */
1842	.long 0 /* 0xa8 */
1843	.long 0 /* 0xac */
1844	.long 0 /* 0xb0 */
1845	.long 0 /* 0xb4 */
1846	.long 0 /* 0xb8 */
1847	.long 0 /* 0xbc */
1848	.long 0 /* 0xc0 */
1849	.long 0 /* 0xc4 */
1850	.long 0 /* 0xc8 */
1851	.long 0 /* 0xcc */
1852	.long 0 /* 0xd0 */
1853	.long 0 /* 0xd4 */
1854	.long 0 /* 0xd8 */
1855	.long 0 /* 0xdc */
1856	.long DOTSYM(kvmppc_h_cede) - hcall_real_table
1857	.long DOTSYM(kvmppc_rm_h_confer) - hcall_real_table
1858	.long 0 /* 0xe8 */
1859	.long 0 /* 0xec */
1860	.long 0 /* 0xf0 */
1861	.long 0 /* 0xf4 */
1862	.long 0 /* 0xf8 */
1863	.long 0 /* 0xfc */
1864	.long 0 /* 0x100 */
1865	.long 0 /* 0x104 */
1866	.long 0 /* 0x108 */
1867	.long 0 /* 0x10c */
1868	.long 0 /* 0x110 */
1869	.long 0 /* 0x114 */
1870	.long 0 /* 0x118 */
1871	.long 0 /* 0x11c */
1872	.long 0 /* 0x120 */
1873	.long DOTSYM(kvmppc_h_bulk_remove) - hcall_real_table
1874	.long 0 /* 0x128 */
1875	.long 0 /* 0x12c */
1876	.long 0 /* 0x130 */
1877	.long DOTSYM(kvmppc_h_set_xdabr) - hcall_real_table
1878	.long 0 /* 0x138 */
1879	.long 0 /* 0x13c */
1880	.long 0 /* 0x140 */
1881	.long 0 /* 0x144 */
1882	.long 0 /* 0x148 */
1883	.long 0 /* 0x14c */
1884	.long 0 /* 0x150 */
1885	.long 0 /* 0x154 */
1886	.long 0 /* 0x158 */
1887	.long 0 /* 0x15c */
1888	.long 0 /* 0x160 */
1889	.long 0 /* 0x164 */
1890	.long 0 /* 0x168 */
1891	.long 0 /* 0x16c */
1892	.long 0 /* 0x170 */
1893	.long 0 /* 0x174 */
1894	.long 0 /* 0x178 */
1895	.long 0 /* 0x17c */
1896	.long 0 /* 0x180 */
1897	.long 0 /* 0x184 */
1898	.long 0 /* 0x188 */
1899	.long 0 /* 0x18c */
1900	.long 0 /* 0x190 */
1901	.long 0 /* 0x194 */
1902	.long 0 /* 0x198 */
1903	.long 0 /* 0x19c */
1904	.long 0 /* 0x1a0 */
1905	.long 0 /* 0x1a4 */
1906	.long 0 /* 0x1a8 */
1907	.long 0 /* 0x1ac */
1908	.long 0 /* 0x1b0 */
1909	.long 0 /* 0x1b4 */
1910	.long 0 /* 0x1b8 */
1911	.long 0 /* 0x1bc */
1912	.long 0 /* 0x1c0 */
1913	.long 0 /* 0x1c4 */
1914	.long 0 /* 0x1c8 */
1915	.long 0 /* 0x1cc */
1916	.long 0 /* 0x1d0 */
1917	.long 0 /* 0x1d4 */
1918	.long 0 /* 0x1d8 */
1919	.long 0 /* 0x1dc */
1920	.long 0 /* 0x1e0 */
1921	.long 0 /* 0x1e4 */
1922	.long 0 /* 0x1e8 */
1923	.long 0 /* 0x1ec */
1924	.long 0 /* 0x1f0 */
1925	.long 0 /* 0x1f4 */
1926	.long 0 /* 0x1f8 */
1927	.long 0 /* 0x1fc */
1928	.long 0 /* 0x200 */
1929	.long 0 /* 0x204 */
1930	.long 0 /* 0x208 */
1931	.long 0 /* 0x20c */
1932	.long 0 /* 0x210 */
1933	.long 0 /* 0x214 */
1934	.long 0 /* 0x218 */
1935	.long 0 /* 0x21c */
1936	.long 0 /* 0x220 */
1937	.long 0 /* 0x224 */
1938	.long 0 /* 0x228 */
1939	.long 0 /* 0x22c */
1940	.long 0 /* 0x230 */
1941	.long 0 /* 0x234 */
1942	.long 0 /* 0x238 */
1943	.long 0 /* 0x23c */
1944	.long 0 /* 0x240 */
1945	.long 0 /* 0x244 */
1946	.long 0 /* 0x248 */
1947	.long 0 /* 0x24c */
1948	.long 0 /* 0x250 */
1949	.long 0 /* 0x254 */
1950	.long 0 /* 0x258 */
1951	.long 0 /* 0x25c */
1952	.long 0 /* 0x260 */
1953	.long 0 /* 0x264 */
1954	.long 0 /* 0x268 */
1955	.long 0 /* 0x26c */
1956	.long 0 /* 0x270 */
1957	.long 0 /* 0x274 */
1958	.long 0 /* 0x278 */
1959	.long 0 /* 0x27c */
1960	.long 0 /* 0x280 */
1961	.long 0 /* 0x284 */
1962	.long 0 /* 0x288 */
1963	.long 0 /* 0x28c */
1964	.long 0 /* 0x290 */
1965	.long 0 /* 0x294 */
1966	.long 0 /* 0x298 */
1967	.long 0 /* 0x29c */
1968	.long 0 /* 0x2a0 */
1969	.long 0 /* 0x2a4 */
1970	.long 0 /* 0x2a8 */
1971	.long 0 /* 0x2ac */
1972	.long 0 /* 0x2b0 */
1973	.long 0 /* 0x2b4 */
1974	.long 0 /* 0x2b8 */
1975	.long 0 /* 0x2bc */
1976	.long 0 /* 0x2c0 */
1977	.long 0 /* 0x2c4 */
1978	.long 0 /* 0x2c8 */
1979	.long 0 /* 0x2cc */
1980	.long 0 /* 0x2d0 */
1981	.long 0 /* 0x2d4 */
1982	.long 0 /* 0x2d8 */
1983	.long 0 /* 0x2dc */
1984	.long 0 /* 0x2e0 */
1985	.long 0 /* 0x2e4 */
1986	.long 0 /* 0x2e8 */
1987	.long 0 /* 0x2ec */
1988	.long 0 /* 0x2f0 */
1989	.long 0 /* 0x2f4 */
1990	.long 0 /* 0x2f8 */
1991	#ifdef CONFIG_KVM_XICS
1992	.long DOTSYM(xics_rm_h_xirr_x) - hcall_real_table
1993	#else
1994	.long 0 /* 0x2fc - H_XIRR_X*/
1995	#endif
1996	.long DOTSYM(kvmppc_rm_h_random) - hcall_real_table
1997	.globl hcall_real_table_end
1998	hcall_real_table_end:
1999
2000	_GLOBAL_TOC(kvmppc_h_set_xdabr)
2001	EXPORT_SYMBOL_GPL(kvmppc_h_set_xdabr)
2002	andi. r0, r5, DABRX_USER | DABRX_KERNEL
2003	beq 6f
2004	li r0, DABRX_USER | DABRX_KERNEL | DABRX_BTI
2005	andc. r0, r5, r0
2006	beq 3f
2007	6: li r3, H_PARAMETER
2008	blr
2009
2010	_GLOBAL_TOC(kvmppc_h_set_dabr)
2011	EXPORT_SYMBOL_GPL(kvmppc_h_set_dabr)
2012	li r5, DABRX_USER | DABRX_KERNEL
2013	3:
2014	BEGIN_FTR_SECTION
2015	b 2f
2016	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2017	std r4,VCPU_DABR(r3)
2018	stw r5, VCPU_DABRX(r3)
2019	mtspr SPRN_DABRX, r5
2020	/* Work around P7 bug where DABR can get corrupted on mtspr */
2021	1: mtspr SPRN_DABR,r4
2022	mfspr r5, SPRN_DABR
2023	cmpd r4, r5
2024	bne 1b
2025	isync
2026	li r3,0
2027	blr
2028
2029	2:
2030	LOAD_REG_ADDR(r11, dawr_force_enable)
2031	lbz r11, 0(r11)
2032	cmpdi r11, 0
2033	bne 3f
2034	li r3, H_HARDWARE
2035	blr
2036	3:
2037	/* Emulate H_SET_DABR/X on P8 for the sake of compat mode guests */
2038	rlwimi r5, r4, 5, DAWRX_DR | DAWRX_DW
2039	rlwimi r5, r4, 2, DAWRX_WT
2040	clrrdi r4, r4, 3
2041	std r4, VCPU_DAWR0(r3)
2042	std r5, VCPU_DAWRX0(r3)
2043	/*
2044	* If came in through the real mode hcall handler then it is necessary
2045	* to write the registers since the return path won't. Otherwise it is
2046	* sufficient to store then in the vcpu struct as they will be loaded
2047	* next time the vcpu is run.
2048	*/
2049	mfmsr r6
2050	andi. r6, r6, MSR_DR /* in real mode? */
2051	bne 4f
2052	mtspr SPRN_DAWR0, r4
2053	mtspr SPRN_DAWRX0, r5
2054	4: li r3, 0
2055	blr
2056
2057	_GLOBAL(kvmppc_h_cede) /* r3 = vcpu pointer, r11 = msr, r13 = paca */
2058	ori r11,r11,MSR_EE
2059	std r11,VCPU_MSR(r3)
2060	li r0,1
2061	stb r0,VCPU_CEDED(r3)
2062	sync /* order setting ceded vs. testing prodded */
2063	lbz r5,VCPU_PRODDED(r3)
2064	cmpwi r5,0
2065	bne kvm_cede_prodded
2066	li r12,0 /* set trap to 0 to say hcall is handled */
2067	stw r12,VCPU_TRAP(r3)
2068	li r0,H_SUCCESS
2069	std r0,VCPU_GPR(R3)(r3)
2070
2071	/*
2072	* Set our bit in the bitmask of napping threads unless all the
2073	* other threads are already napping, in which case we send this
2074	* up to the host.
2075	*/
2076	ld r5,HSTATE_KVM_VCORE(r13)
2077	lbz r6,HSTATE_PTID(r13)
2078	lwz r8,VCORE_ENTRY_EXIT(r5)
2079	clrldi r8,r8,56
2080	li r0,1
2081	sld r0,r0,r6
2082	addi r6,r5,VCORE_NAPPING_THREADS
2083	31: lwarx r4,0,r6
2084	or r4,r4,r0
2085	cmpw r4,r8
2086	beq kvm_cede_exit
2087	stwcx. r4,0,r6
2088	bne 31b
2089	/* order napping_threads update vs testing entry_exit_map */
2090	isync
2091	li r0,NAPPING_CEDE
2092	stb r0,HSTATE_NAPPING(r13)
2093	lwz r7,VCORE_ENTRY_EXIT(r5)
2094	cmpwi r7,0x100
2095	bge 33f /* another thread already exiting */
2096
2097	/*
2098	* Although not specifically required by the architecture, POWER7
2099	* preserves the following registers in nap mode, even if an SMT mode
2100	* switch occurs: SLB entries, PURR, SPURR, AMOR, UAMOR, AMR, SPRG0-3,
2101	* DAR, DSISR, DABR, DABRX, DSCR, PMCx, MMCRx, SIAR, SDAR.
2102	*/
2103	/* Save non-volatile GPRs */
2104	std r14, VCPU_GPR(R14)(r3)
2105	std r15, VCPU_GPR(R15)(r3)
2106	std r16, VCPU_GPR(R16)(r3)
2107	std r17, VCPU_GPR(R17)(r3)
2108	std r18, VCPU_GPR(R18)(r3)
2109	std r19, VCPU_GPR(R19)(r3)
2110	std r20, VCPU_GPR(R20)(r3)
2111	std r21, VCPU_GPR(R21)(r3)
2112	std r22, VCPU_GPR(R22)(r3)
2113	std r23, VCPU_GPR(R23)(r3)
2114	std r24, VCPU_GPR(R24)(r3)
2115	std r25, VCPU_GPR(R25)(r3)
2116	std r26, VCPU_GPR(R26)(r3)
2117	std r27, VCPU_GPR(R27)(r3)
2118	std r28, VCPU_GPR(R28)(r3)
2119	std r29, VCPU_GPR(R29)(r3)
2120	std r30, VCPU_GPR(R30)(r3)
2121	std r31, VCPU_GPR(R31)(r3)
2122
2123	/* save FP state */
2124	bl kvmppc_save_fp
2125
2126	#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2127	BEGIN_FTR_SECTION
2128	b 91f
2129	END_FTR_SECTION_IFCLR(CPU_FTR_TM)
2130	/*
2131	* NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR)
2132	*/
2133	ld r3, HSTATE_KVM_VCPU(r13)
2134	ld r4, VCPU_MSR(r3)
2135	li r5, 0 /* don't preserve non-vol regs */
2136	bl kvmppc_save_tm_hv
2137	nop
2138	91:
2139	#endif
2140
2141	/*
2142	* Set DEC to the smaller of DEC and HDEC, so that we wake
2143	* no later than the end of our timeslice (HDEC interrupts
2144	* don't wake us from nap).
2145	*/
2146	mfspr r3, SPRN_DEC
2147	mfspr r4, SPRN_HDEC
2148	mftb r5
2149	extsw r3, r3
2150	extsw r4, r4
2151	cmpd r3, r4
2152	ble 67f
2153	mtspr SPRN_DEC, r4
2154	67:
2155	/* save expiry time of guest decrementer */
2156	add r3, r3, r5
2157	ld r4, HSTATE_KVM_VCPU(r13)
2158	std r3, VCPU_DEC_EXPIRES(r4)
2159
2160	#ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING
2161	ld r4, HSTATE_KVM_VCPU(r13)
2162	addi r3, r4, VCPU_TB_CEDE
2163	bl kvmhv_accumulate_time
2164	#endif
2165
2166	lis r3, LPCR_PECEDP@h /* Do wake on privileged doorbell */
2167
2168	/* Go back to host stack */
2169	ld r1, HSTATE_HOST_R1(r13)
2170
2171	/*
2172	* Take a nap until a decrementer or external or doobell interrupt
2173	* occurs, with PECE1 and PECE0 set in LPCR.
2174	* On POWER8, set PECEDH, and if we are ceding, also set PECEDP.
2175	* Also clear the runlatch bit before napping.
2176	*/
2177	kvm_do_nap:
2178	li r0,0
2179	mtspr SPRN_CTRLT, r0
2180
2181	li r0,1
2182	stb r0,HSTATE_HWTHREAD_REQ(r13)
2183	mfspr r5,SPRN_LPCR
2184	ori r5,r5,LPCR_PECE0 | LPCR_PECE1
2185	BEGIN_FTR_SECTION
2186	ori r5, r5, LPCR_PECEDH
2187	rlwimi r5, r3, 0, LPCR_PECEDP
2188	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2189
2190	kvm_nap_sequence: /* desired LPCR value in r5 */
2191	li r3, PNV_THREAD_NAP
2192	mtspr SPRN_LPCR,r5
2193	isync
2194
2195	bl isa206_idle_insn_mayloss
2196
2197	li r0,1
2198	mtspr SPRN_CTRLT, r0
2199
2200	mtspr SPRN_SRR1, r3
2201
2202	li r0, 0
2203	stb r0, PACA_FTRACE_ENABLED(r13)
2204
2205	li r0, KVM_HWTHREAD_IN_KVM
2206	stb r0, HSTATE_HWTHREAD_STATE(r13)
2207
2208	lbz r0, HSTATE_NAPPING(r13)
2209	cmpwi r0, NAPPING_CEDE
2210	beq kvm_end_cede
2211	cmpwi r0, NAPPING_NOVCPU
2212	beq kvm_novcpu_wakeup
2213	cmpwi r0, NAPPING_UNSPLIT
2214	beq kvm_unsplit_wakeup
2215	twi 31,0,0 /* Nap state must not be zero */
2216
2217	33: mr r4, r3
2218	li r3, 0
2219	li r12, 0
2220	b 34f
2221
2222	kvm_end_cede:
2223	/* Woken by external or decrementer interrupt */
2224
2225	/* get vcpu pointer */
2226	ld r4, HSTATE_KVM_VCPU(r13)
2227
2228	#ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING
2229	addi r3, r4, VCPU_TB_RMINTR
2230	bl kvmhv_accumulate_time
2231	#endif
2232
2233	#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2234	BEGIN_FTR_SECTION
2235	b 91f
2236	END_FTR_SECTION_IFCLR(CPU_FTR_TM)
2237	/*
2238	* NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR)
2239	*/
2240	mr r3, r4
2241	ld r4, VCPU_MSR(r3)
2242	li r5, 0 /* don't preserve non-vol regs */
2243	bl kvmppc_restore_tm_hv
2244	nop
2245	ld r4, HSTATE_KVM_VCPU(r13)
2246	91:
2247	#endif
2248
2249	/* load up FP state */
2250	bl kvmppc_load_fp
2251
2252	/* Restore guest decrementer */
2253	ld r3, VCPU_DEC_EXPIRES(r4)
2254	mftb r7
2255	subf r3, r7, r3
2256	mtspr SPRN_DEC, r3
2257
2258	/* Load NV GPRS */
2259	ld r14, VCPU_GPR(R14)(r4)
2260	ld r15, VCPU_GPR(R15)(r4)
2261	ld r16, VCPU_GPR(R16)(r4)
2262	ld r17, VCPU_GPR(R17)(r4)
2263	ld r18, VCPU_GPR(R18)(r4)
2264	ld r19, VCPU_GPR(R19)(r4)
2265	ld r20, VCPU_GPR(R20)(r4)
2266	ld r21, VCPU_GPR(R21)(r4)
2267	ld r22, VCPU_GPR(R22)(r4)
2268	ld r23, VCPU_GPR(R23)(r4)
2269	ld r24, VCPU_GPR(R24)(r4)
2270	ld r25, VCPU_GPR(R25)(r4)
2271	ld r26, VCPU_GPR(R26)(r4)
2272	ld r27, VCPU_GPR(R27)(r4)
2273	ld r28, VCPU_GPR(R28)(r4)
2274	ld r29, VCPU_GPR(R29)(r4)
2275	ld r30, VCPU_GPR(R30)(r4)
2276	ld r31, VCPU_GPR(R31)(r4)
2277
2278	/* Check the wake reason in SRR1 to see why we got here */
2279	bl kvmppc_check_wake_reason
2280
2281	/*
2282	* Restore volatile registers since we could have called a
2283	* C routine in kvmppc_check_wake_reason
2284	* r4 = VCPU
2285	* r3 tells us whether we need to return to host or not
2286	* WARNING: it gets checked further down:
2287	* should not modify r3 until this check is done.
2288	*/
2289	ld r4, HSTATE_KVM_VCPU(r13)
2290
2291	/* clear our bit in vcore->napping_threads */
2292	34: ld r5,HSTATE_KVM_VCORE(r13)
2293	lbz r7,HSTATE_PTID(r13)
2294	li r0,1
2295	sld r0,r0,r7
2296	addi r6,r5,VCORE_NAPPING_THREADS
2297	32: lwarx r7,0,r6
2298	andc r7,r7,r0
2299	stwcx. r7,0,r6
2300	bne 32b
2301	li r0,0
2302	stb r0,HSTATE_NAPPING(r13)
2303
2304	/* See if the wake reason saved in r3 means we need to exit */
2305	stw r12, VCPU_TRAP(r4)
2306	mr r9, r4
2307	cmpdi r3, 0
2308	bgt guest_exit_cont
2309	b maybe_reenter_guest
2310
2311	/* cede when already previously prodded case */
2312	kvm_cede_prodded:
2313	li r0,0
2314	stb r0,VCPU_PRODDED(r3)
2315	sync /* order testing prodded vs. clearing ceded */
2316	stb r0,VCPU_CEDED(r3)
2317	li r3,H_SUCCESS
2318	blr
2319
2320	/* we've ceded but we want to give control to the host */
2321	kvm_cede_exit:
2322	ld r9, HSTATE_KVM_VCPU(r13)
2323	b guest_exit_cont
2324
2325	/* Try to do machine check recovery in real mode */
2326	machine_check_realmode:
2327	mr r3, r9 /* get vcpu pointer */
2328	bl kvmppc_realmode_machine_check
2329	nop
2330	/* all machine checks go to virtual mode for further handling */
2331	ld r9, HSTATE_KVM_VCPU(r13)
2332	li r12, BOOK3S_INTERRUPT_MACHINE_CHECK
2333	b guest_exit_cont
2334
2335	/*
2336	* Call C code to handle a HMI in real mode.
2337	* Only the primary thread does the call, secondary threads are handled
2338	* by calling hmi_exception_realmode() after kvmppc_hv_entry returns.
2339	* r9 points to the vcpu on entry
2340	*/
2341	hmi_realmode:
2342	lbz r0, HSTATE_PTID(r13)
2343	cmpwi r0, 0
2344	bne guest_exit_cont
2345	bl CFUNC(kvmppc_realmode_hmi_handler)
2346	ld r9, HSTATE_KVM_VCPU(r13)
2347	li r12, BOOK3S_INTERRUPT_HMI
2348	b guest_exit_cont
2349
2350	/*
2351	* Check the reason we woke from nap, and take appropriate action.
2352	* Returns (in r3):
2353	* 0 if nothing needs to be done
2354	* 1 if something happened that needs to be handled by the host
2355	* -1 if there was a guest wakeup (IPI or msgsnd)
2356	* -2 if we handled a PCI passthrough interrupt (returned by
2357	* kvmppc_read_intr only)
2358	*
2359	* Also sets r12 to the interrupt vector for any interrupt that needs
2360	* to be handled now by the host (0x500 for external interrupt), or zero.
2361	* Modifies all volatile registers (since it may call a C function).
2362	* This routine calls kvmppc_read_intr, a C function, if an external
2363	* interrupt is pending.
2364	*/
2365	SYM_FUNC_START_LOCAL(kvmppc_check_wake_reason)
2366	mfspr r6, SPRN_SRR1
2367	BEGIN_FTR_SECTION
2368	rlwinm r6, r6, 45-31, 0xf /* extract wake reason field (P8) */
2369	FTR_SECTION_ELSE
2370	rlwinm r6, r6, 45-31, 0xe /* P7 wake reason field is 3 bits */
2371	ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S)
2372	cmpwi r6, 8 /* was it an external interrupt? */
2373	beq 7f /* if so, see what it was */
2374	li r3, 0
2375	li r12, 0
2376	cmpwi r6, 6 /* was it the decrementer? */
2377	beq 0f
2378	BEGIN_FTR_SECTION
2379	cmpwi r6, 5 /* privileged doorbell? */
2380	beq 0f
2381	cmpwi r6, 3 /* hypervisor doorbell? */
2382	beq 3f
2383	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2384	cmpwi r6, 0xa /* Hypervisor maintenance ? */
2385	beq 4f
2386	li r3, 1 /* anything else, return 1 */
2387	0: blr
2388
2389	/* hypervisor doorbell */
2390	3: li r12, BOOK3S_INTERRUPT_H_DOORBELL
2391
2392	/*
2393	* Clear the doorbell as we will invoke the handler
2394	* explicitly in the guest exit path.
2395	*/
2396	lis r6, (PPC_DBELL_SERVER << (63-36))@h
2397	PPC_MSGCLR(6)
2398	/* see if it's a host IPI */
2399	li r3, 1
2400	lbz r0, HSTATE_HOST_IPI(r13)
2401	cmpwi r0, 0
2402	bnelr
2403	/* if not, return -1 */
2404	li r3, -1
2405	blr
2406
2407	/* Woken up due to Hypervisor maintenance interrupt */
2408	4: li r12, BOOK3S_INTERRUPT_HMI
2409	li r3, 1
2410	blr
2411
2412	/* external interrupt - create a stack frame so we can call C */
2413	7: mflr r0
2414	std r0, PPC_LR_STKOFF(r1)
2415	stdu r1, -PPC_MIN_STKFRM(r1)
2416	bl CFUNC(kvmppc_read_intr)
2417	nop
2418	li r12, BOOK3S_INTERRUPT_EXTERNAL
2419	cmpdi r3, 1
2420	ble 1f
2421
2422	/*
2423	* Return code of 2 means PCI passthrough interrupt, but
2424	* we need to return back to host to complete handling the
2425	* interrupt. Trap reason is expected in r12 by guest
2426	* exit code.
2427	*/
2428	li r12, BOOK3S_INTERRUPT_HV_RM_HARD
2429	1:
2430	ld r0, PPC_MIN_STKFRM+PPC_LR_STKOFF(r1)
2431	addi r1, r1, PPC_MIN_STKFRM
2432	mtlr r0
2433	blr
2434	SYM_FUNC_END(kvmppc_check_wake_reason)
2435
2436	/*
2437	* Save away FP, VMX and VSX registers.
2438	* r3 = vcpu pointer
2439	* N.B. r30 and r31 are volatile across this function,
2440	* thus it is not callable from C.
2441	*/
2442	SYM_FUNC_START_LOCAL(kvmppc_save_fp)
2443	mflr r30
2444	mr r31,r3
2445	mfmsr r5
2446	ori r8,r5,MSR_FP
2447	#ifdef CONFIG_ALTIVEC
2448	BEGIN_FTR_SECTION
2449	oris r8,r8,MSR_VEC@h
2450	END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2451	#endif
2452	#ifdef CONFIG_VSX
2453	BEGIN_FTR_SECTION
2454	oris r8,r8,MSR_VSX@h
2455	END_FTR_SECTION_IFSET(CPU_FTR_VSX)
2456	#endif
2457	mtmsrd r8
2458	addi r3,r3,VCPU_FPRS
2459	bl store_fp_state
2460	#ifdef CONFIG_ALTIVEC
2461	BEGIN_FTR_SECTION
2462	addi r3,r31,VCPU_VRS
2463	bl store_vr_state
2464	END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2465	#endif
2466	mfspr r6,SPRN_VRSAVE
2467	stw r6,VCPU_VRSAVE(r31)
2468	mtlr r30
2469	blr
2470	SYM_FUNC_END(kvmppc_save_fp)
2471
2472	/*
2473	* Load up FP, VMX and VSX registers
2474	* r4 = vcpu pointer
2475	* N.B. r30 and r31 are volatile across this function,
2476	* thus it is not callable from C.
2477	*/
2478	SYM_FUNC_START_LOCAL(kvmppc_load_fp)
2479	mflr r30
2480	mr r31,r4
2481	mfmsr r9
2482	ori r8,r9,MSR_FP
2483	#ifdef CONFIG_ALTIVEC
2484	BEGIN_FTR_SECTION
2485	oris r8,r8,MSR_VEC@h
2486	END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2487	#endif
2488	#ifdef CONFIG_VSX
2489	BEGIN_FTR_SECTION
2490	oris r8,r8,MSR_VSX@h
2491	END_FTR_SECTION_IFSET(CPU_FTR_VSX)
2492	#endif
2493	mtmsrd r8
2494	addi r3,r4,VCPU_FPRS
2495	bl load_fp_state
2496	#ifdef CONFIG_ALTIVEC
2497	BEGIN_FTR_SECTION
2498	addi r3,r31,VCPU_VRS
2499	bl load_vr_state
2500	END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2501	#endif
2502	lwz r7,VCPU_VRSAVE(r31)
2503	mtspr SPRN_VRSAVE,r7
2504	mtlr r30
2505	mr r4,r31
2506	blr
2507	SYM_FUNC_END(kvmppc_load_fp)
2508
2509	#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2510	/*
2511	* Save transactional state and TM-related registers.
2512	* Called with r3 pointing to the vcpu struct and r4 containing
2513	* the guest MSR value.
2514	* r5 is non-zero iff non-volatile register state needs to be maintained.
2515	* If r5 == 0, this can modify all checkpointed registers, but
2516	* restores r1 and r2 before exit.
2517	*/
2518	_GLOBAL_TOC(kvmppc_save_tm_hv)
2519	EXPORT_SYMBOL_GPL(kvmppc_save_tm_hv)
2520	/* See if we need to handle fake suspend mode */
2521	BEGIN_FTR_SECTION
2522	b __kvmppc_save_tm
2523	END_FTR_SECTION_IFCLR(CPU_FTR_P9_TM_HV_ASSIST)
2524
2525	lbz r0, HSTATE_FAKE_SUSPEND(r13) /* Were we fake suspended? */
2526	cmpwi r0, 0
2527	beq __kvmppc_save_tm
2528
2529	/* The following code handles the fake_suspend = 1 case */
2530	mflr r0
2531	std r0, PPC_LR_STKOFF(r1)
2532	stdu r1, -TM_FRAME_SIZE(r1)
2533
2534	/* Turn on TM. */
2535	mfmsr r8
2536	li r0, 1
2537	rldimi r8, r0, MSR_TM_LG, 63-MSR_TM_LG
2538	mtmsrd r8
2539
2540	rldicl. r8, r8, 64 - MSR_TS_S_LG, 62 /* Did we actually hrfid? */
2541	beq 4f
2542	BEGIN_FTR_SECTION
2543	bl pnv_power9_force_smt4_catch
2544	END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_XER_SO_BUG)
2545	nop
2546
2547	/*
2548	* It's possible that treclaim. may modify registers, if we have lost
2549	* track of fake-suspend state in the guest due to it using rfscv.
2550	* Save and restore registers in case this occurs.
2551	*/
2552	mfspr r3, SPRN_DSCR
2553	mfspr r4, SPRN_XER
2554	mfspr r5, SPRN_AMR
2555	/* SPRN_TAR would need to be saved here if the kernel ever used it */
2556	mfcr r12
2557	SAVE_NVGPRS(r1)
2558	SAVE_GPR(2, r1)
2559	SAVE_GPR(3, r1)
2560	SAVE_GPR(4, r1)
2561	SAVE_GPR(5, r1)
2562	stw r12, 8(r1)
2563	std r1, HSTATE_HOST_R1(r13)
2564
2565	/* We have to treclaim here because that's the only way to do S->N */
2566	li r3, TM_CAUSE_KVM_RESCHED
2567	TRECLAIM(R3)
2568
2569	GET_PACA(r13)
2570	ld r1, HSTATE_HOST_R1(r13)
2571	REST_GPR(2, r1)
2572	REST_GPR(3, r1)
2573	REST_GPR(4, r1)
2574	REST_GPR(5, r1)
2575	lwz r12, 8(r1)
2576	REST_NVGPRS(r1)
2577	mtspr SPRN_DSCR, r3
2578	mtspr SPRN_XER, r4
2579	mtspr SPRN_AMR, r5
2580	mtcr r12
2581	HMT_MEDIUM
2582
2583	/*
2584	* We were in fake suspend, so we are not going to save the
2585	* register state as the guest checkpointed state (since
2586	* we already have it), therefore we can now use any volatile GPR.
2587	* In fact treclaim in fake suspend state doesn't modify
2588	* any registers.
2589	*/
2590
2591	BEGIN_FTR_SECTION
2592	bl pnv_power9_force_smt4_release
2593	END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_XER_SO_BUG)
2594	nop
2595
2596	4:
2597	mfspr r3, SPRN_PSSCR
2598	/* PSSCR_FAKE_SUSPEND is a write-only bit, but clear it anyway */
2599	li r0, PSSCR_FAKE_SUSPEND
2600	andc r3, r3, r0
2601	mtspr SPRN_PSSCR, r3
2602
2603	/* Don't save TEXASR, use value from last exit in real suspend state */
2604	ld r9, HSTATE_KVM_VCPU(r13)
2605	mfspr r5, SPRN_TFHAR
2606	mfspr r6, SPRN_TFIAR
2607	std r5, VCPU_TFHAR(r9)
2608	std r6, VCPU_TFIAR(r9)
2609
2610	addi r1, r1, TM_FRAME_SIZE
2611	ld r0, PPC_LR_STKOFF(r1)
2612	mtlr r0
2613	blr
2614
2615	/*
2616	* Restore transactional state and TM-related registers.
2617	* Called with r3 pointing to the vcpu struct
2618	* and r4 containing the guest MSR value.
2619	* r5 is non-zero iff non-volatile register state needs to be maintained.
2620	* This potentially modifies all checkpointed registers.
2621	* It restores r1 and r2 from the PACA.
2622	*/
2623	_GLOBAL_TOC(kvmppc_restore_tm_hv)
2624	EXPORT_SYMBOL_GPL(kvmppc_restore_tm_hv)
2625	/*
2626	* If we are doing TM emulation for the guest on a POWER9 DD2,
2627	* then we don't actually do a trechkpt -- we either set up
2628	* fake-suspend mode, or emulate a TM rollback.
2629	*/
2630	BEGIN_FTR_SECTION
2631	b __kvmppc_restore_tm
2632	END_FTR_SECTION_IFCLR(CPU_FTR_P9_TM_HV_ASSIST)
2633	mflr r0
2634	std r0, PPC_LR_STKOFF(r1)
2635
2636	li r0, 0
2637	stb r0, HSTATE_FAKE_SUSPEND(r13)
2638
2639	/* Turn on TM so we can restore TM SPRs */
2640	mfmsr r5
2641	li r0, 1
2642	rldimi r5, r0, MSR_TM_LG, 63-MSR_TM_LG
2643	mtmsrd r5
2644
2645	/*
2646	* The user may change these outside of a transaction, so they must
2647	* always be context switched.
2648	*/
2649	ld r5, VCPU_TFHAR(r3)
2650	ld r6, VCPU_TFIAR(r3)
2651	ld r7, VCPU_TEXASR(r3)
2652	mtspr SPRN_TFHAR, r5
2653	mtspr SPRN_TFIAR, r6
2654	mtspr SPRN_TEXASR, r7
2655
2656	rldicl. r5, r4, 64 - MSR_TS_S_LG, 62
2657	beqlr /* TM not active in guest */
2658
2659	/* Make sure the failure summary is set */
2660	oris r7, r7, (TEXASR_FS)@h
2661	mtspr SPRN_TEXASR, r7
2662
2663	cmpwi r5, 1 /* check for suspended state */
2664	bgt 10f
2665	stb r5, HSTATE_FAKE_SUSPEND(r13)
2666	b 9f /* and return */
2667	10: stdu r1, -PPC_MIN_STKFRM(r1)
2668	/* guest is in transactional state, so simulate rollback */
2669	bl kvmhv_emulate_tm_rollback
2670	nop
2671	addi r1, r1, PPC_MIN_STKFRM
2672	9: ld r0, PPC_LR_STKOFF(r1)
2673	mtlr r0
2674	blr
2675	#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
2676
2677	/*
2678	* We come here if we get any exception or interrupt while we are
2679	* executing host real mode code while in guest MMU context.
2680	* r12 is (CR << 32) | vector
2681	* r13 points to our PACA
2682	* r12 is saved in HSTATE_SCRATCH0(r13)
2683	* r9 is saved in HSTATE_SCRATCH2(r13)
2684	* r13 is saved in HSPRG1
2685	* cfar is saved in HSTATE_CFAR(r13)
2686	* ppr is saved in HSTATE_PPR(r13)
2687	*/
2688	kvmppc_bad_host_intr:
2689	/*
2690	* Switch to the emergency stack, but start half-way down in
2691	* case we were already on it.
2692	*/
2693	mr r9, r1
2694	std r1, PACAR1(r13)
2695	ld r1, PACAEMERGSP(r13)
2696	subi r1, r1, THREAD_SIZE/2 + INT_FRAME_SIZE
2697	std r9, 0(r1)
2698	std r0, GPR0(r1)
2699	std r9, GPR1(r1)
2700	std r2, GPR2(r1)
2701	SAVE_GPRS(3, 8, r1)
2702	srdi r0, r12, 32
2703	clrldi r12, r12, 32
2704	std r0, _CCR(r1)
2705	std r12, _TRAP(r1)
2706	andi. r0, r12, 2
2707	beq 1f
2708	mfspr r3, SPRN_HSRR0
2709	mfspr r4, SPRN_HSRR1
2710	mfspr r5, SPRN_HDAR
2711	mfspr r6, SPRN_HDSISR
2712	b 2f
2713	1: mfspr r3, SPRN_SRR0
2714	mfspr r4, SPRN_SRR1
2715	mfspr r5, SPRN_DAR
2716	mfspr r6, SPRN_DSISR
2717	2: std r3, _NIP(r1)
2718	std r4, _MSR(r1)
2719	std r5, _DAR(r1)
2720	std r6, _DSISR(r1)
2721	ld r9, HSTATE_SCRATCH2(r13)
2722	ld r12, HSTATE_SCRATCH0(r13)
2723	GET_SCRATCH0(r0)
2724	SAVE_GPRS(9, 12, r1)
2725	std r0, GPR13(r1)
2726	SAVE_NVGPRS(r1)
2727	ld r5, HSTATE_CFAR(r13)
2728	std r5, ORIG_GPR3(r1)
2729	mflr r3
2730	mfctr r4
2731	mfxer r5
2732	lbz r6, PACAIRQSOFTMASK(r13)
2733	std r3, _LINK(r1)
2734	std r4, _CTR(r1)
2735	std r5, _XER(r1)
2736	std r6, SOFTE(r1)
2737	LOAD_PACA_TOC()
2738	LOAD_REG_IMMEDIATE(3, STACK_FRAME_REGS_MARKER)
2739	std r3, STACK_INT_FRAME_MARKER(r1)
2740
2741	/*
2742	* XXX On POWER7 and POWER8, we just spin here since we don't
2743	* know what the other threads are doing (and we don't want to
2744	* coordinate with them) - but at least we now have register state
2745	* in memory that we might be able to look at from another CPU.
2746	*/
2747	b .
2748
2749	/*
2750	* This mimics the MSR transition on IRQ delivery. The new guest MSR is taken
2751	* from VCPU_INTR_MSR and is modified based on the required TM state changes.
2752	* r11 has the guest MSR value (in/out)
2753	* r9 has a vcpu pointer (in)
2754	* r0 is used as a scratch register
2755	*/
2756	SYM_FUNC_START_LOCAL(kvmppc_msr_interrupt)
2757	rldicl r0, r11, 64 - MSR_TS_S_LG, 62
2758	cmpwi r0, 2 /* Check if we are in transactional state.. */
2759	ld r11, VCPU_INTR_MSR(r9)
2760	bne 1f
2761	/* ... if transactional, change to suspended */
2762	li r0, 1
2763	1: rldimi r11, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG
2764	blr
2765	SYM_FUNC_END(kvmppc_msr_interrupt)
2766
2767	/*
2768	* void kvmhv_load_guest_pmu(struct kvm_vcpu *vcpu)
2769	*
2770	* Load up guest PMU state. R3 points to the vcpu struct.
2771	*/
2772	SYM_FUNC_START_LOCAL(kvmhv_load_guest_pmu)
2773	mr r4, r3
2774	mflr r0
2775	li r3, 1
2776	sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */
2777	mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */
2778	isync
2779	BEGIN_FTR_SECTION
2780	ld r3, VCPU_MMCR(r4)
2781	andi. r5, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO
2782	cmpwi r5, MMCR0_PMAO
2783	beql kvmppc_fix_pmao
2784	END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG)
2785	lwz r3, VCPU_PMC(r4) /* always load up guest PMU registers */
2786	lwz r5, VCPU_PMC + 4(r4) /* to prevent information leak */
2787	lwz r6, VCPU_PMC + 8(r4)
2788	lwz r7, VCPU_PMC + 12(r4)
2789	lwz r8, VCPU_PMC + 16(r4)
2790	lwz r9, VCPU_PMC + 20(r4)
2791	mtspr SPRN_PMC1, r3
2792	mtspr SPRN_PMC2, r5
2793	mtspr SPRN_PMC3, r6
2794	mtspr SPRN_PMC4, r7
2795	mtspr SPRN_PMC5, r8
2796	mtspr SPRN_PMC6, r9
2797	ld r3, VCPU_MMCR(r4)
2798	ld r5, VCPU_MMCR + 8(r4)
2799	ld r6, VCPU_MMCRA(r4)
2800	ld r7, VCPU_SIAR(r4)
2801	ld r8, VCPU_SDAR(r4)
2802	mtspr SPRN_MMCR1, r5
2803	mtspr SPRN_MMCRA, r6
2804	mtspr SPRN_SIAR, r7
2805	mtspr SPRN_SDAR, r8
2806	BEGIN_FTR_SECTION
2807	ld r5, VCPU_MMCR + 16(r4)
2808	ld r6, VCPU_SIER(r4)
2809	mtspr SPRN_MMCR2, r5
2810	mtspr SPRN_SIER, r6
2811	lwz r7, VCPU_PMC + 24(r4)
2812	lwz r8, VCPU_PMC + 28(r4)
2813	ld r9, VCPU_MMCRS(r4)
2814	mtspr SPRN_SPMC1, r7
2815	mtspr SPRN_SPMC2, r8
2816	mtspr SPRN_MMCRS, r9
2817	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2818	mtspr SPRN_MMCR0, r3
2819	isync
2820	mtlr r0
2821	blr
2822	SYM_FUNC_END(kvmhv_load_guest_pmu)
2823
2824	/*
2825	* void kvmhv_load_host_pmu(void)
2826	*
2827	* Reload host PMU state saved in the PACA by kvmhv_save_host_pmu.
2828	*/
2829	SYM_FUNC_START_LOCAL(kvmhv_load_host_pmu)
2830	mflr r0
2831	lbz r4, PACA_PMCINUSE(r13) /* is the host using the PMU? */
2832	cmpwi r4, 0
2833	beq 23f /* skip if not */
2834	BEGIN_FTR_SECTION
2835	ld r3, HSTATE_MMCR0(r13)
2836	andi. r4, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO
2837	cmpwi r4, MMCR0_PMAO
2838	beql kvmppc_fix_pmao
2839	END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG)
2840	lwz r3, HSTATE_PMC1(r13)
2841	lwz r4, HSTATE_PMC2(r13)
2842	lwz r5, HSTATE_PMC3(r13)
2843	lwz r6, HSTATE_PMC4(r13)
2844	lwz r8, HSTATE_PMC5(r13)
2845	lwz r9, HSTATE_PMC6(r13)
2846	mtspr SPRN_PMC1, r3
2847	mtspr SPRN_PMC2, r4
2848	mtspr SPRN_PMC3, r5
2849	mtspr SPRN_PMC4, r6
2850	mtspr SPRN_PMC5, r8
2851	mtspr SPRN_PMC6, r9
2852	ld r3, HSTATE_MMCR0(r13)
2853	ld r4, HSTATE_MMCR1(r13)
2854	ld r5, HSTATE_MMCRA(r13)
2855	ld r6, HSTATE_SIAR(r13)
2856	ld r7, HSTATE_SDAR(r13)
2857	mtspr SPRN_MMCR1, r4
2858	mtspr SPRN_MMCRA, r5
2859	mtspr SPRN_SIAR, r6
2860	mtspr SPRN_SDAR, r7
2861	BEGIN_FTR_SECTION
2862	ld r8, HSTATE_MMCR2(r13)
2863	ld r9, HSTATE_SIER(r13)
2864	mtspr SPRN_MMCR2, r8
2865	mtspr SPRN_SIER, r9
2866	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2867	mtspr SPRN_MMCR0, r3
2868	isync
2869	mtlr r0
2870	23: blr
2871	SYM_FUNC_END(kvmhv_load_host_pmu)
2872
2873	/*
2874	* void kvmhv_save_guest_pmu(struct kvm_vcpu *vcpu, bool pmu_in_use)
2875	*
2876	* Save guest PMU state into the vcpu struct.
2877	* r3 = vcpu, r4 = full save flag (PMU in use flag set in VPA)
2878	*/
2879	SYM_FUNC_START_LOCAL(kvmhv_save_guest_pmu)
2880	mr r9, r3
2881	mr r8, r4
2882	BEGIN_FTR_SECTION
2883	/*
2884	* POWER8 seems to have a hardware bug where setting
2885	* MMCR0[PMAE] along with MMCR0[PMC1CE] and/or MMCR0[PMCjCE]
2886	* when some counters are already negative doesn't seem
2887	* to cause a performance monitor alert (and hence interrupt).
2888	* The effect of this is that when saving the PMU state,
2889	* if there is no PMU alert pending when we read MMCR0
2890	* before freezing the counters, but one becomes pending
2891	* before we read the counters, we lose it.
2892	* To work around this, we need a way to freeze the counters
2893	* before reading MMCR0. Normally, freezing the counters
2894	* is done by writing MMCR0 (to set MMCR0[FC]) which
2895	* unavoidably writes MMCR0[PMA0] as well. On POWER8,
2896	* we can also freeze the counters using MMCR2, by writing
2897	* 1s to all the counter freeze condition bits (there are
2898	* 9 bits each for 6 counters).
2899	*/
2900	li r3, -1 /* set all freeze bits */
2901	clrrdi r3, r3, 10
2902	mfspr r10, SPRN_MMCR2
2903	mtspr SPRN_MMCR2, r3
2904	isync
2905	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2906	li r3, 1
2907	sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */
2908	mfspr r4, SPRN_MMCR0 /* save MMCR0 */
2909	mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */
2910	mfspr r6, SPRN_MMCRA
2911	/* Clear MMCRA in order to disable SDAR updates */
2912	li r7, 0
2913	mtspr SPRN_MMCRA, r7
2914	isync
2915	cmpwi r8, 0 /* did they ask for PMU stuff to be saved? */
2916	bne 21f
2917	std r3, VCPU_MMCR(r9) /* if not, set saved MMCR0 to FC */
2918	b 22f
2919	21: mfspr r5, SPRN_MMCR1
2920	mfspr r7, SPRN_SIAR
2921	mfspr r8, SPRN_SDAR
2922	std r4, VCPU_MMCR(r9)
2923	std r5, VCPU_MMCR + 8(r9)
2924	std r6, VCPU_MMCRA(r9)
2925	BEGIN_FTR_SECTION
2926	std r10, VCPU_MMCR + 16(r9)
2927	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2928	std r7, VCPU_SIAR(r9)
2929	std r8, VCPU_SDAR(r9)
2930	mfspr r3, SPRN_PMC1
2931	mfspr r4, SPRN_PMC2
2932	mfspr r5, SPRN_PMC3
2933	mfspr r6, SPRN_PMC4
2934	mfspr r7, SPRN_PMC5
2935	mfspr r8, SPRN_PMC6
2936	stw r3, VCPU_PMC(r9)
2937	stw r4, VCPU_PMC + 4(r9)
2938	stw r5, VCPU_PMC + 8(r9)
2939	stw r6, VCPU_PMC + 12(r9)
2940	stw r7, VCPU_PMC + 16(r9)
2941	stw r8, VCPU_PMC + 20(r9)
2942	BEGIN_FTR_SECTION
2943	mfspr r5, SPRN_SIER
2944	std r5, VCPU_SIER(r9)
2945	mfspr r6, SPRN_SPMC1
2946	mfspr r7, SPRN_SPMC2
2947	mfspr r8, SPRN_MMCRS
2948	stw r6, VCPU_PMC + 24(r9)
2949	stw r7, VCPU_PMC + 28(r9)
2950	std r8, VCPU_MMCRS(r9)
2951	lis r4, 0x8000
2952	mtspr SPRN_MMCRS, r4
2953	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2954	22: blr
2955	SYM_FUNC_END(kvmhv_save_guest_pmu)
2956
2957	/*
2958	* This works around a hardware bug on POWER8E processors, where
2959	* writing a 1 to the MMCR0[PMAO] bit doesn't generate a
2960	* performance monitor interrupt. Instead, when we need to have
2961	* an interrupt pending, we have to arrange for a counter to overflow.
2962	*/
2963	kvmppc_fix_pmao:
2964	li r3, 0
2965	mtspr SPRN_MMCR2, r3
2966	lis r3, (MMCR0_PMXE | MMCR0_FCECE)@h
2967	ori r3, r3, MMCR0_PMCjCE | MMCR0_C56RUN
2968	mtspr SPRN_MMCR0, r3
2969	lis r3, 0x7fff
2970	ori r3, r3, 0xffff
2971	mtspr SPRN_PMC6, r3
2972	isync
2973	blr
2974
2975	#ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING
2976	/*
2977	* Start timing an activity
2978	* r3 = pointer to time accumulation struct, r4 = vcpu
2979	*/
2980	kvmhv_start_timing:
2981	ld r5, HSTATE_KVM_VCORE(r13)
2982	ld r6, VCORE_TB_OFFSET_APPL(r5)
2983	mftb r5
2984	subf r5, r6, r5 /* subtract current timebase offset */
2985	std r3, VCPU_CUR_ACTIVITY(r4)
2986	std r5, VCPU_ACTIVITY_START(r4)
2987	blr
2988
2989	/*
2990	* Accumulate time to one activity and start another.
2991	* r3 = pointer to new time accumulation struct, r4 = vcpu
2992	*/
2993	kvmhv_accumulate_time:
2994	ld r5, HSTATE_KVM_VCORE(r13)
2995	ld r8, VCORE_TB_OFFSET_APPL(r5)
2996	ld r5, VCPU_CUR_ACTIVITY(r4)
2997	ld r6, VCPU_ACTIVITY_START(r4)
2998	std r3, VCPU_CUR_ACTIVITY(r4)
2999	mftb r7
3000	subf r7, r8, r7 /* subtract current timebase offset */
3001	std r7, VCPU_ACTIVITY_START(r4)
3002	cmpdi r5, 0
3003	beqlr
3004	subf r3, r6, r7
3005	ld r8, TAS_SEQCOUNT(r5)
3006	cmpdi r8, 0
3007	addi r8, r8, 1
3008	std r8, TAS_SEQCOUNT(r5)
3009	lwsync
3010	ld r7, TAS_TOTAL(r5)
3011	add r7, r7, r3
3012	std r7, TAS_TOTAL(r5)
3013	ld r6, TAS_MIN(r5)
3014	ld r7, TAS_MAX(r5)
3015	beq 3f
3016	cmpd r3, r6
3017	bge 1f
3018	3: std r3, TAS_MIN(r5)
3019	1: cmpd r3, r7
3020	ble 2f
3021	std r3, TAS_MAX(r5)
3022	2: lwsync
3023	addi r8, r8, 1
3024	std r8, TAS_SEQCOUNT(r5)
3025	blr
3026	#endif
3027