1	/* SPDX-License-Identifier: GPL-2.0-only */
2	/*
3	*
4	* Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
5	*
6	* Derived from book3s_rmhandlers.S and other files, which are:
7	*
8	* Copyright SUSE Linux Products GmbH 2009
9	*
10	* Authors: Alexander Graf <agraf@suse.de>
11	*/
12
13	#include <linux/export.h>
14	#include <linux/linkage.h>
15	#include <linux/objtool.h>
16	#include <asm/ppc_asm.h>
17	#include <asm/code-patching-asm.h>
18	#include <asm/kvm_asm.h>
19	#include <asm/reg.h>
20	#include <asm/mmu.h>
21	#include <asm/page.h>
22	#include <asm/ptrace.h>
23	#include <asm/hvcall.h>
24	#include <asm/asm-offsets.h>
25	#include <asm/exception-64s.h>
26	#include <asm/kvm_book3s_asm.h>
27	#include <asm/book3s/64/mmu-hash.h>
28	#include <asm/tm.h>
29	#include <asm/opal.h>
30	#include <asm/thread_info.h>
31	#include <asm/asm-compat.h>
32	#include <asm/feature-fixups.h>
33	#include <asm/cpuidle.h>
34
35	/* Values in HSTATE_NAPPING(r13) */
36	#define NAPPING_CEDE 1
37	#define NAPPING_NOVCPU 2
38	#define NAPPING_UNSPLIT 3
39
40	/* Stack frame offsets for kvmppc_hv_entry */
41	#define SFS 160
42	#define STACK_SLOT_TRAP (SFS-4)
43	#define STACK_SLOT_TID (SFS-16)
44	#define STACK_SLOT_PSSCR (SFS-24)
45	#define STACK_SLOT_PID (SFS-32)
46	#define STACK_SLOT_IAMR (SFS-40)
47	#define STACK_SLOT_CIABR (SFS-48)
48	#define STACK_SLOT_DAWR0 (SFS-56)
49	#define STACK_SLOT_DAWRX0 (SFS-64)
50	#define STACK_SLOT_HFSCR (SFS-72)
51	#define STACK_SLOT_AMR (SFS-80)
52	#define STACK_SLOT_UAMOR (SFS-88)
53	#define STACK_SLOT_FSCR (SFS-96)
54
55	/*
56	* Use the last LPID (all implemented LPID bits = 1) for partition switching.
57	* This is reserved in the LPID allocator. POWER7 only implements 0x3ff, but
58	* we write 0xfff into the LPID SPR anyway, which seems to work and just
59	* ignores the top bits.
60	*/
61	#define LPID_RSVD 0xfff
62
63	/*
64	* Call kvmppc_hv_entry in real mode.
65	* Must be called with interrupts hard-disabled.
66	*
67	* Input Registers:
68	*
69	* LR = return address to continue at after eventually re-enabling MMU
70	*/
71	_GLOBAL_TOC(kvmppc_hv_entry_trampoline)
72	mflr r0
73	std r0, PPC_LR_STKOFF(r1)
74	stdu r1, -112(r1)
75	mfmsr r10
76	std r10, HSTATE_HOST_MSR(r13)
77	LOAD_REG_ADDR(r5, kvmppc_call_hv_entry)
78	li r0,MSR_RI
79	andc r0,r10,r0
80	li r6,MSR_IR | MSR_DR
81	andc r6,r10,r6
82	mtmsrd r0,1 /* clear RI in MSR */
83	mtsrr0 r5
84	mtsrr1 r6
85	RFI_TO_KERNEL
86
87	kvmppc_call_hv_entry:
88	ld r4, HSTATE_KVM_VCPU(r13)
89	bl kvmppc_hv_entry
90
91	/* Back from guest - restore host state and return to caller */
92
93	BEGIN_FTR_SECTION
94	/* Restore host DABR and DABRX */
95	ld r5,HSTATE_DABR(r13)
96	li r6,7
97	mtspr SPRN_DABR,r5
98	mtspr SPRN_DABRX,r6
99	END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
100
101	/* Restore SPRG3 */
102	ld r3,PACA_SPRG_VDSO(r13)
103	mtspr SPRN_SPRG_VDSO_WRITE,r3
104
105	/* Reload the host's PMU registers */
106	bl kvmhv_load_host_pmu
107
108	/*
109	* Reload DEC. HDEC interrupts were disabled when
110	* we reloaded the host's LPCR value.
111	*/
112	ld r3, HSTATE_DECEXP(r13)
113	mftb r4
114	subf r4, r4, r3
115	mtspr SPRN_DEC, r4
116
117	/* hwthread_req may have got set by cede or no vcpu, so clear it */
118	li r0, 0
119	stb r0, HSTATE_HWTHREAD_REQ(r13)
120
121	/*
122	* For external interrupts we need to call the Linux
123	* handler to process the interrupt. We do that by jumping
124	* to absolute address 0x500 for external interrupts.
125	* The [h]rfid at the end of the handler will return to
126	* the book3s_hv_interrupts.S code. For other interrupts
127	* we do the rfid to get back to the book3s_hv_interrupts.S
128	* code here.
129	*/
130	ld r8, 112+PPC_LR_STKOFF(r1)
131	addi r1, r1, 112
132	ld r7, HSTATE_HOST_MSR(r13)
133
134	/* Return the trap number on this thread as the return value */
135	mr r3, r12
136
137	/* RFI into the highmem handler */
138	mfmsr r6
139	li r0, MSR_RI
140	andc r6, r6, r0
141	mtmsrd r6, 1 /* Clear RI in MSR */
142	mtsrr0 r8
143	mtsrr1 r7
144	RFI_TO_KERNEL
145
146	kvmppc_primary_no_guest:
147	/* We handle this much like a ceded vcpu */
148	/* put the HDEC into the DEC, since HDEC interrupts don't wake us */
149	/* HDEC may be larger than DEC for arch >= v3.00, but since the */
150	/* HDEC value came from DEC in the first place, it will fit */
151	mfspr r3, SPRN_HDEC
152	mtspr SPRN_DEC, r3
153	/*
154	* Make sure the primary has finished the MMU switch.
155	* We should never get here on a secondary thread, but
156	* check it for robustness' sake.
157	*/
158	ld r5, HSTATE_KVM_VCORE(r13)
159	65: lbz r0, VCORE_IN_GUEST(r5)
160	cmpwi r0, 0
161	beq 65b
162	/* Set LPCR. */
163	ld r8,VCORE_LPCR(r5)
164	mtspr SPRN_LPCR,r8
165	isync
166	/* set our bit in napping_threads */
167	ld r5, HSTATE_KVM_VCORE(r13)
168	lbz r7, HSTATE_PTID(r13)
169	li r0, 1
170	sld r0, r0, r7
171	addi r6, r5, VCORE_NAPPING_THREADS
172	1: lwarx r3, 0, r6
173	or r3, r3, r0
174	stwcx. r3, 0, r6
175	bne 1b
176	/* order napping_threads update vs testing entry_exit_map */
177	isync
178	li r12, 0
179	lwz r7, VCORE_ENTRY_EXIT(r5)
180	cmpwi r7, 0x100
181	bge kvm_novcpu_exit /* another thread already exiting */
182	li r3, NAPPING_NOVCPU
183	stb r3, HSTATE_NAPPING(r13)
184
185	li r3, 0 /* Don't wake on privileged (OS) doorbell */
186	b kvm_do_nap
187
188	/*
189	* kvm_novcpu_wakeup
190	* Entered from kvm_start_guest if kvm_hstate.napping is set
191	* to NAPPING_NOVCPU
192	* r2 = kernel TOC
193	* r13 = paca
194	*/
195	kvm_novcpu_wakeup:
196	ld r1, HSTATE_HOST_R1(r13)
197	ld r5, HSTATE_KVM_VCORE(r13)
198	li r0, 0
199	stb r0, HSTATE_NAPPING(r13)
200
201	/* check the wake reason */
202	bl kvmppc_check_wake_reason
203
204	/*
205	* Restore volatile registers since we could have called
206	* a C routine in kvmppc_check_wake_reason.
207	* r5 = VCORE
208	*/
209	ld r5, HSTATE_KVM_VCORE(r13)
210
211	/* see if any other thread is already exiting */
212	lwz r0, VCORE_ENTRY_EXIT(r5)
213	cmpwi r0, 0x100
214	bge kvm_novcpu_exit
215
216	/* clear our bit in napping_threads */
217	lbz r7, HSTATE_PTID(r13)
218	li r0, 1
219	sld r0, r0, r7
220	addi r6, r5, VCORE_NAPPING_THREADS
221	4: lwarx r7, 0, r6
222	andc r7, r7, r0
223	stwcx. r7, 0, r6
224	bne 4b
225
226	/* See if the wake reason means we need to exit */
227	cmpdi r3, 0
228	bge kvm_novcpu_exit
229
230	/* See if our timeslice has expired (HDEC is negative) */
231	mfspr r0, SPRN_HDEC
232	extsw r0, r0
233	li r12, BOOK3S_INTERRUPT_HV_DECREMENTER
234	cmpdi r0, 0
235	blt kvm_novcpu_exit
236
237	/* Got an IPI but other vcpus aren't yet exiting, must be a latecomer */
238	ld r4, HSTATE_KVM_VCPU(r13)
239	cmpdi r4, 0
240	beq kvmppc_primary_no_guest
241
242	#ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING
243	addi r3, r4, VCPU_TB_RMENTRY
244	bl kvmhv_start_timing
245	#endif
246	b kvmppc_got_guest
247
248	kvm_novcpu_exit:
249	#ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING
250	ld r4, HSTATE_KVM_VCPU(r13)
251	cmpdi r4, 0
252	beq 13f
253	addi r3, r4, VCPU_TB_RMEXIT
254	bl kvmhv_accumulate_time
255	#endif
256	13: mr r3, r12
257	stw r12, STACK_SLOT_TRAP(r1)
258	bl kvmhv_commence_exit
259	nop
260	b kvmhv_switch_to_host
261
262	/*
263	* We come in here when wakened from Linux offline idle code.
264	* Relocation is off
265	* r3 contains the SRR1 wakeup value, SRR1 is trashed.
266	*/
267	_GLOBAL(idle_kvm_start_guest)
268	mfcr r5
269	mflr r0
270	std r5, 8(r1) // Save CR in caller's frame
271	std r0, 16(r1) // Save LR in caller's frame
272	// Create frame on emergency stack
273	ld r4, PACAEMERGSP(r13)
274	stdu r1, -SWITCH_FRAME_SIZE(r4)
275	// Switch to new frame on emergency stack
276	mr r1, r4
277	std r3, 32(r1) // Save SRR1 wakeup value
278	SAVE_NVGPRS(r1)
279
280	/*
281	* Could avoid this and pass it through in r3. For now,
282	* code expects it to be in SRR1.
283	*/
284	mtspr SPRN_SRR1,r3
285
286	li r0,0
287	stb r0,PACA_FTRACE_ENABLED(r13)
288
289	li r0,KVM_HWTHREAD_IN_KVM
290	stb r0,HSTATE_HWTHREAD_STATE(r13)
291
292	/* kvm cede / napping does not come through here */
293	lbz r0,HSTATE_NAPPING(r13)
294	twnei r0,0
295
296	b 1f
297
298	kvm_unsplit_wakeup:
299	li r0, 0
300	stb r0, HSTATE_NAPPING(r13)
301
302	1:
303
304	/*
305	* We weren't napping due to cede, so this must be a secondary
306	* thread being woken up to run a guest, or being woken up due
307	* to a stray IPI. (Or due to some machine check or hypervisor
308	* maintenance interrupt while the core is in KVM.)
309	*/
310
311	/* Check the wake reason in SRR1 to see why we got here */
312	bl kvmppc_check_wake_reason
313	/*
314	* kvmppc_check_wake_reason could invoke a C routine, but we
315	* have no volatile registers to restore when we return.
316	*/
317
318	cmpdi r3, 0
319	bge kvm_no_guest
320
321	/* get vcore pointer, NULL if we have nothing to run */
322	ld r5,HSTATE_KVM_VCORE(r13)
323	cmpdi r5,0
324	/* if we have no vcore to run, go back to sleep */
325	beq kvm_no_guest
326
327	kvm_secondary_got_guest:
328
329	// About to go to guest, clear saved SRR1
330	li r0, 0
331	std r0, 32(r1)
332
333	/* Set HSTATE_DSCR(r13) to something sensible */
334	ld r6, PACA_DSCR_DEFAULT(r13)
335	std r6, HSTATE_DSCR(r13)
336
337	/* On thread 0 of a subcore, set HDEC to max */
338	lbz r4, HSTATE_PTID(r13)
339	cmpwi r4, 0
340	bne 63f
341	lis r6,0x7fff /* MAX_INT@h */
342	mtspr SPRN_HDEC, r6
343	/* and set per-LPAR registers, if doing dynamic micro-threading */
344	ld r6, HSTATE_SPLIT_MODE(r13)
345	cmpdi r6, 0
346	beq 63f
347	ld r0, KVM_SPLIT_RPR(r6)
348	mtspr SPRN_RPR, r0
349	ld r0, KVM_SPLIT_PMMAR(r6)
350	mtspr SPRN_PMMAR, r0
351	ld r0, KVM_SPLIT_LDBAR(r6)
352	mtspr SPRN_LDBAR, r0
353	isync
354	63:
355	/* Order load of vcpu after load of vcore */
356	lwsync
357	ld r4, HSTATE_KVM_VCPU(r13)
358	bl kvmppc_hv_entry
359
360	/* Back from the guest, go back to nap */
361	/* Clear our vcpu and vcore pointers so we don't come back in early */
362	li r0, 0
363	std r0, HSTATE_KVM_VCPU(r13)
364	/*
365	* Once we clear HSTATE_KVM_VCORE(r13), the code in
366	* kvmppc_run_core() is going to assume that all our vcpu
367	* state is visible in memory. This lwsync makes sure
368	* that that is true.
369	*/
370	lwsync
371	std r0, HSTATE_KVM_VCORE(r13)
372
373	/*
374	* All secondaries exiting guest will fall through this path.
375	* Before proceeding, just check for HMI interrupt and
376	* invoke opal hmi handler. By now we are sure that the
377	* primary thread on this core/subcore has already made partition
378	* switch/TB resync and we are good to call opal hmi handler.
379	*/
380	cmpwi r12, BOOK3S_INTERRUPT_HMI
381	bne kvm_no_guest
382
383	li r3,0 /* NULL argument */
384	bl CFUNC(hmi_exception_realmode)
385	/*
386	* At this point we have finished executing in the guest.
387	* We need to wait for hwthread_req to become zero, since
388	* we may not turn on the MMU while hwthread_req is non-zero.
389	* While waiting we also need to check if we get given a vcpu to run.
390	*/
391	kvm_no_guest:
392	lbz r3, HSTATE_HWTHREAD_REQ(r13)
393	cmpwi r3, 0
394	bne 53f
395	HMT_MEDIUM
396	li r0, KVM_HWTHREAD_IN_KERNEL
397	stb r0, HSTATE_HWTHREAD_STATE(r13)
398	/* need to recheck hwthread_req after a barrier, to avoid race */
399	sync
400	lbz r3, HSTATE_HWTHREAD_REQ(r13)
401	cmpwi r3, 0
402	bne 54f
403
404	/*
405	* Jump to idle_return_gpr_loss, which returns to the
406	* idle_kvm_start_guest caller.
407	*/
408	li r3, LPCR_PECE0
409	mfspr r4, SPRN_LPCR
410	rlwimi r4, r3, 0, LPCR_PECE0 | LPCR_PECE1
411	mtspr SPRN_LPCR, r4
412	// Return SRR1 wakeup value, or 0 if we went into the guest
413	ld r3, 32(r1)
414	REST_NVGPRS(r1)
415	ld r1, 0(r1) // Switch back to caller stack
416	ld r0, 16(r1) // Reload LR
417	ld r5, 8(r1) // Reload CR
418	mtlr r0
419	mtcr r5
420	blr
421
422	53:
423	HMT_LOW
424	ld r5, HSTATE_KVM_VCORE(r13)
425	cmpdi r5, 0
426	bne 60f
427	ld r3, HSTATE_SPLIT_MODE(r13)
428	cmpdi r3, 0
429	beq kvm_no_guest
430	lbz r0, KVM_SPLIT_DO_NAP(r3)
431	cmpwi r0, 0
432	beq kvm_no_guest
433	HMT_MEDIUM
434	b kvm_unsplit_nap
435	60: HMT_MEDIUM
436	b kvm_secondary_got_guest
437
438	54: li r0, KVM_HWTHREAD_IN_KVM
439	stb r0, HSTATE_HWTHREAD_STATE(r13)
440	b kvm_no_guest
441
442	/*
443	* Here the primary thread is trying to return the core to
444	* whole-core mode, so we need to nap.
445	*/
446	kvm_unsplit_nap:
447	/*
448	* When secondaries are napping in kvm_unsplit_nap() with
449	* hwthread_req = 1, HMI goes ignored even though subcores are
450	* already exited the guest. Hence HMI keeps waking up secondaries
451	* from nap in a loop and secondaries always go back to nap since
452	* no vcore is assigned to them. This makes impossible for primary
453	* thread to get hold of secondary threads resulting into a soft
454	* lockup in KVM path.
455	*
456	* Let us check if HMI is pending and handle it before we go to nap.
457	*/
458	cmpwi r12, BOOK3S_INTERRUPT_HMI
459	bne 55f
460	li r3, 0 /* NULL argument */
461	bl CFUNC(hmi_exception_realmode)
462	55:
463	/*
464	* Ensure that secondary doesn't nap when it has
465	* its vcore pointer set.
466	*/
467	sync /* matches smp_mb() before setting split_info.do_nap */
468	ld r0, HSTATE_KVM_VCORE(r13)
469	cmpdi r0, 0
470	bne kvm_no_guest
471	/* clear any pending message */
472	BEGIN_FTR_SECTION
473	lis r6, (PPC_DBELL_SERVER << (63-36))@h
474	PPC_MSGCLR(6)
475	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
476	/* Set kvm_split_mode.napped[tid] = 1 */
477	ld r3, HSTATE_SPLIT_MODE(r13)
478	li r0, 1
479	lhz r4, PACAPACAINDEX(r13)
480	clrldi r4, r4, 61 /* micro-threading => P8 => 8 threads/core */
481	addi r4, r4, KVM_SPLIT_NAPPED
482	stbx r0, r3, r4
483	/* Check the do_nap flag again after setting napped[] */
484	sync
485	lbz r0, KVM_SPLIT_DO_NAP(r3)
486	cmpwi r0, 0
487	beq 57f
488	li r3, NAPPING_UNSPLIT
489	stb r3, HSTATE_NAPPING(r13)
490	li r3, (LPCR_PECEDH | LPCR_PECE0) >> 4
491	mfspr r5, SPRN_LPCR
492	rlwimi r5, r3, 4, (LPCR_PECEDP | LPCR_PECEDH | LPCR_PECE0 | LPCR_PECE1)
493	b kvm_nap_sequence
494
495	57: li r0, 0
496	stbx r0, r3, r4
497	b kvm_no_guest
498
499	/******************************************************************************
500	* *
501	* Entry code *
502	* *
503	*****************************************************************************/
504
505	SYM_CODE_START_LOCAL(kvmppc_hv_entry)
506
507	/* Required state:
508	*
509	* R4 = vcpu pointer (or NULL)
510	* MSR = ~IR|DR
511	* R13 = PACA
512	* R1 = host R1
513	* R2 = TOC
514	* all other volatile GPRS = free
515	* Does not preserve non-volatile GPRs or CR fields
516	*/
517	mflr r0
518	std r0, PPC_LR_STKOFF(r1)
519	stdu r1, -SFS(r1)
520
521	/* Save R1 in the PACA */
522	std r1, HSTATE_HOST_R1(r13)
523
524	li r6, KVM_GUEST_MODE_HOST_HV
525	stb r6, HSTATE_IN_GUEST(r13)
526
527	#ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING
528	/* Store initial timestamp */
529	cmpdi r4, 0
530	beq 1f
531	addi r3, r4, VCPU_TB_RMENTRY
532	bl kvmhv_start_timing
533	1:
534	#endif
535
536	ld r5, HSTATE_KVM_VCORE(r13)
537	ld r9, VCORE_KVM(r5) /* pointer to struct kvm */
538
539	/*
540	* POWER7/POWER8 host -> guest partition switch code.
541	* We don't have to lock against concurrent tlbies,
542	* but we do have to coordinate across hardware threads.
543	*/
544	/* Set bit in entry map iff exit map is zero. */
545	li r7, 1
546	lbz r6, HSTATE_PTID(r13)
547	sld r7, r7, r6
548	addi r8, r5, VCORE_ENTRY_EXIT
549	21: lwarx r3, 0, r8
550	cmpwi r3, 0x100 /* any threads starting to exit? */
551	bge secondary_too_late /* if so we're too late to the party */
552	or r3, r3, r7
553	stwcx. r3, 0, r8
554	bne 21b
555
556	/* Primary thread switches to guest partition. */
557	cmpwi r6,0
558	bne 10f
559
560	lwz r7,KVM_LPID(r9)
561	ld r6,KVM_SDR1(r9)
562	li r0,LPID_RSVD /* switch to reserved LPID */
563	mtspr SPRN_LPID,r0
564	ptesync
565	mtspr SPRN_SDR1,r6 /* switch to partition page table */
566	mtspr SPRN_LPID,r7
567	isync
568
569	/* See if we need to flush the TLB. */
570	mr r3, r9 /* kvm pointer */
571	lhz r4, PACAPACAINDEX(r13) /* physical cpu number */
572	li r5, 0 /* nested vcpu pointer */
573	bl kvmppc_check_need_tlb_flush
574	nop
575	ld r5, HSTATE_KVM_VCORE(r13)
576
577	/* Add timebase offset onto timebase */
578	22: ld r8,VCORE_TB_OFFSET(r5)
579	cmpdi r8,0
580	beq 37f
581	std r8, VCORE_TB_OFFSET_APPL(r5)
582	mftb r6 /* current host timebase */
583	add r8,r8,r6
584	mtspr SPRN_TBU40,r8 /* update upper 40 bits */
585	mftb r7 /* check if lower 24 bits overflowed */
586	clrldi r6,r6,40
587	clrldi r7,r7,40
588	cmpld r7,r6
589	bge 37f
590	addis r8,r8,0x100 /* if so, increment upper 40 bits */
591	mtspr SPRN_TBU40,r8
592
593	/* Load guest PCR value to select appropriate compat mode */
594	37: ld r7, VCORE_PCR(r5)
595	LOAD_REG_IMMEDIATE(r6, PCR_MASK)
596	cmpld r7, r6
597	beq 38f
598	or r7, r7, r6
599	mtspr SPRN_PCR, r7
600	38:
601
602	BEGIN_FTR_SECTION
603	/* DPDES and VTB are shared between threads */
604	ld r8, VCORE_DPDES(r5)
605	ld r7, VCORE_VTB(r5)
606	mtspr SPRN_DPDES, r8
607	mtspr SPRN_VTB, r7
608	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
609
610	/* Mark the subcore state as inside guest */
611	bl kvmppc_subcore_enter_guest
612	nop
613	ld r5, HSTATE_KVM_VCORE(r13)
614	ld r4, HSTATE_KVM_VCPU(r13)
615	li r0,1
616	stb r0,VCORE_IN_GUEST(r5) /* signal secondaries to continue */
617
618	/* Do we have a guest vcpu to run? */
619	10: cmpdi r4, 0
620	beq kvmppc_primary_no_guest
621	kvmppc_got_guest:
622	/* Increment yield count if they have a VPA */
623	ld r3, VCPU_VPA(r4)
624	cmpdi r3, 0
625	beq 25f
626	li r6, LPPACA_YIELDCOUNT
627	LWZX_BE r5, r3, r6
628	addi r5, r5, 1
629	STWX_BE r5, r3, r6
630	li r6, 1
631	stb r6, VCPU_VPA_DIRTY(r4)
632	25:
633
634	/* Save purr/spurr */
635	mfspr r5,SPRN_PURR
636	mfspr r6,SPRN_SPURR
637	std r5,HSTATE_PURR(r13)
638	std r6,HSTATE_SPURR(r13)
639	ld r7,VCPU_PURR(r4)
640	ld r8,VCPU_SPURR(r4)
641	mtspr SPRN_PURR,r7
642	mtspr SPRN_SPURR,r8
643
644	/* Save host values of some registers */
645	BEGIN_FTR_SECTION
646	mfspr r5, SPRN_CIABR
647	mfspr r6, SPRN_DAWR0
648	mfspr r7, SPRN_DAWRX0
649	mfspr r8, SPRN_IAMR
650	std r5, STACK_SLOT_CIABR(r1)
651	std r6, STACK_SLOT_DAWR0(r1)
652	std r7, STACK_SLOT_DAWRX0(r1)
653	std r8, STACK_SLOT_IAMR(r1)
654	mfspr r5, SPRN_FSCR
655	std r5, STACK_SLOT_FSCR(r1)
656	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
657
658	mfspr r5, SPRN_AMR
659	std r5, STACK_SLOT_AMR(r1)
660	mfspr r6, SPRN_UAMOR
661	std r6, STACK_SLOT_UAMOR(r1)
662
663	BEGIN_FTR_SECTION
664	/* Set partition DABR */
665	/* Do this before re-enabling PMU to avoid P7 DABR corruption bug */
666	lwz r5,VCPU_DABRX(r4)
667	ld r6,VCPU_DABR(r4)
668	mtspr SPRN_DABRX,r5
669	mtspr SPRN_DABR,r6
670	isync
671	END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
672
673	#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
674	BEGIN_FTR_SECTION
675	b 91f
676	END_FTR_SECTION_IFCLR(CPU_FTR_TM)
677	/*
678	* NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR)
679	*/
680	mr r3, r4
681	ld r4, VCPU_MSR(r3)
682	li r5, 0 /* don't preserve non-vol regs */
683	bl kvmppc_restore_tm_hv
684	nop
685	ld r4, HSTATE_KVM_VCPU(r13)
686	91:
687	#endif
688
689	/* Load guest PMU registers; r4 = vcpu pointer here */
690	mr r3, r4
691	bl kvmhv_load_guest_pmu
692
693	/* Load up FP, VMX and VSX registers */
694	ld r4, HSTATE_KVM_VCPU(r13)
695	bl kvmppc_load_fp
696
697	ld r14, VCPU_GPR(R14)(r4)
698	ld r15, VCPU_GPR(R15)(r4)
699	ld r16, VCPU_GPR(R16)(r4)
700	ld r17, VCPU_GPR(R17)(r4)
701	ld r18, VCPU_GPR(R18)(r4)
702	ld r19, VCPU_GPR(R19)(r4)
703	ld r20, VCPU_GPR(R20)(r4)
704	ld r21, VCPU_GPR(R21)(r4)
705	ld r22, VCPU_GPR(R22)(r4)
706	ld r23, VCPU_GPR(R23)(r4)
707	ld r24, VCPU_GPR(R24)(r4)
708	ld r25, VCPU_GPR(R25)(r4)
709	ld r26, VCPU_GPR(R26)(r4)
710	ld r27, VCPU_GPR(R27)(r4)
711	ld r28, VCPU_GPR(R28)(r4)
712	ld r29, VCPU_GPR(R29)(r4)
713	ld r30, VCPU_GPR(R30)(r4)
714	ld r31, VCPU_GPR(R31)(r4)
715
716	/* Switch DSCR to guest value */
717	ld r5, VCPU_DSCR(r4)
718	mtspr SPRN_DSCR, r5
719
720	BEGIN_FTR_SECTION
721	/* Skip next section on POWER7 */
722	b 8f
723	END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
724	/* Load up POWER8-specific registers */
725	ld r5, VCPU_IAMR(r4)
726	lwz r6, VCPU_PSPB(r4)
727	ld r7, VCPU_FSCR(r4)
728	mtspr SPRN_IAMR, r5
729	mtspr SPRN_PSPB, r6
730	mtspr SPRN_FSCR, r7
731	/*
732	* Handle broken DAWR case by not writing it. This means we
733	* can still store the DAWR register for migration.
734	*/
735	LOAD_REG_ADDR(r5, dawr_force_enable)
736	lbz r5, 0(r5)
737	cmpdi r5, 0
738	beq 1f
739	ld r5, VCPU_DAWR0(r4)
740	ld r6, VCPU_DAWRX0(r4)
741	mtspr SPRN_DAWR0, r5
742	mtspr SPRN_DAWRX0, r6
743	1:
744	ld r7, VCPU_CIABR(r4)
745	ld r8, VCPU_TAR(r4)
746	mtspr SPRN_CIABR, r7
747	mtspr SPRN_TAR, r8
748	ld r5, VCPU_IC(r4)
749	ld r8, VCPU_EBBHR(r4)
750	mtspr SPRN_IC, r5
751	mtspr SPRN_EBBHR, r8
752	ld r5, VCPU_EBBRR(r4)
753	ld r6, VCPU_BESCR(r4)
754	lwz r7, VCPU_GUEST_PID(r4)
755	ld r8, VCPU_WORT(r4)
756	mtspr SPRN_EBBRR, r5
757	mtspr SPRN_BESCR, r6
758	mtspr SPRN_PID, r7
759	mtspr SPRN_WORT, r8
760	/* POWER8-only registers */
761	ld r5, VCPU_TCSCR(r4)
762	ld r6, VCPU_ACOP(r4)
763	ld r7, VCPU_CSIGR(r4)
764	ld r8, VCPU_TACR(r4)
765	mtspr SPRN_TCSCR, r5
766	mtspr SPRN_ACOP, r6
767	mtspr SPRN_CSIGR, r7
768	mtspr SPRN_TACR, r8
769	nop
770	8:
771
772	ld r5, VCPU_SPRG0(r4)
773	ld r6, VCPU_SPRG1(r4)
774	ld r7, VCPU_SPRG2(r4)
775	ld r8, VCPU_SPRG3(r4)
776	mtspr SPRN_SPRG0, r5
777	mtspr SPRN_SPRG1, r6
778	mtspr SPRN_SPRG2, r7
779	mtspr SPRN_SPRG3, r8
780
781	/* Load up DAR and DSISR */
782	ld r5, VCPU_DAR(r4)
783	lwz r6, VCPU_DSISR(r4)
784	mtspr SPRN_DAR, r5
785	mtspr SPRN_DSISR, r6
786
787	/* Restore AMR and UAMOR, set AMOR to all 1s */
788	ld r5,VCPU_AMR(r4)
789	ld r6,VCPU_UAMOR(r4)
790	mtspr SPRN_AMR,r5
791	mtspr SPRN_UAMOR,r6
792
793	/* Restore state of CTRL run bit; the host currently has it set to 1 */
794	lwz r5,VCPU_CTRL(r4)
795	andi. r5,r5,1
796	bne 4f
797	li r6,0
798	mtspr SPRN_CTRLT,r6
799	4:
800	/* Secondary threads wait for primary to have done partition switch */
801	ld r5, HSTATE_KVM_VCORE(r13)
802	lbz r6, HSTATE_PTID(r13)
803	cmpwi r6, 0
804	beq 21f
805	lbz r0, VCORE_IN_GUEST(r5)
806	cmpwi r0, 0
807	bne 21f
808	HMT_LOW
809	20: lwz r3, VCORE_ENTRY_EXIT(r5)
810	cmpwi r3, 0x100
811	bge no_switch_exit
812	lbz r0, VCORE_IN_GUEST(r5)
813	cmpwi r0, 0
814	beq 20b
815	HMT_MEDIUM
816	21:
817	/* Set LPCR. */
818	ld r8,VCORE_LPCR(r5)
819	mtspr SPRN_LPCR,r8
820	isync
821
822	/*
823	* Set the decrementer to the guest decrementer.
824	*/
825	ld r8,VCPU_DEC_EXPIRES(r4)
826	mftb r7
827	subf r3,r7,r8
828	mtspr SPRN_DEC,r3
829
830	/* Check if HDEC expires soon */
831	mfspr r3, SPRN_HDEC
832	extsw r3, r3
833	cmpdi r3, 512 /* 1 microsecond */
834	blt hdec_soon
835
836	/* Clear out and reload the SLB */
837	li r6, 0
838	slbmte r6, r6
839	PPC_SLBIA(6)
840	ptesync
841
842	/* Load up guest SLB entries (N.B. slb_max will be 0 for radix) */
843	lwz r5,VCPU_SLB_MAX(r4)
844	cmpwi r5,0
845	beq 9f
846	mtctr r5
847	addi r6,r4,VCPU_SLB
848	1: ld r8,VCPU_SLB_E(r6)
849	ld r9,VCPU_SLB_V(r6)
850	slbmte r9,r8
851	addi r6,r6,VCPU_SLB_SIZE
852	bdnz 1b
853	9:
854
855	deliver_guest_interrupt: /* r4 = vcpu, r13 = paca */
856	/* Check if we can deliver an external or decrementer interrupt now */
857	ld r0, VCPU_PENDING_EXC(r4)
858	cmpdi r0, 0
859	beq 71f
860	mr r3, r4
861	bl CFUNC(kvmppc_guest_entry_inject_int)
862	ld r4, HSTATE_KVM_VCPU(r13)
863	71:
864	ld r6, VCPU_SRR0(r4)
865	ld r7, VCPU_SRR1(r4)
866	mtspr SPRN_SRR0, r6
867	mtspr SPRN_SRR1, r7
868
869	ld r10, VCPU_PC(r4)
870	ld r11, VCPU_MSR(r4)
871	/* r11 = vcpu->arch.msr & ~MSR_HV */
872	rldicl r11, r11, 63 - MSR_HV_LG, 1
873	rotldi r11, r11, 1 + MSR_HV_LG
874	ori r11, r11, MSR_ME
875
876	ld r6, VCPU_CTR(r4)
877	ld r7, VCPU_XER(r4)
878	mtctr r6
879	mtxer r7
880
881	/*
882	* Required state:
883	* R4 = vcpu
884	* R10: value for HSRR0
885	* R11: value for HSRR1
886	* R13 = PACA
887	*/
888	fast_guest_return:
889	li r0,0
890	stb r0,VCPU_CEDED(r4) /* cancel cede */
891	mtspr SPRN_HSRR0,r10
892	mtspr SPRN_HSRR1,r11
893
894	/* Activate guest mode, so faults get handled by KVM */
895	li r9, KVM_GUEST_MODE_GUEST_HV
896	stb r9, HSTATE_IN_GUEST(r13)
897
898	#ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING
899	/* Accumulate timing */
900	addi r3, r4, VCPU_TB_GUEST
901	bl kvmhv_accumulate_time
902	#endif
903
904	/* Enter guest */
905
906	BEGIN_FTR_SECTION
907	ld r5, VCPU_CFAR(r4)
908	mtspr SPRN_CFAR, r5
909	END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
910	BEGIN_FTR_SECTION
911	ld r0, VCPU_PPR(r4)
912	END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
913
914	ld r5, VCPU_LR(r4)
915	mtlr r5
916
917	ld r1, VCPU_GPR(R1)(r4)
918	ld r5, VCPU_GPR(R5)(r4)
919	ld r8, VCPU_GPR(R8)(r4)
920	ld r9, VCPU_GPR(R9)(r4)
921	ld r10, VCPU_GPR(R10)(r4)
922	ld r11, VCPU_GPR(R11)(r4)
923	ld r12, VCPU_GPR(R12)(r4)
924	ld r13, VCPU_GPR(R13)(r4)
925
926	BEGIN_FTR_SECTION
927	mtspr SPRN_PPR, r0
928	END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
929
930	ld r6, VCPU_GPR(R6)(r4)
931	ld r7, VCPU_GPR(R7)(r4)
932
933	ld r0, VCPU_CR(r4)
934	mtcr r0
935
936	ld r0, VCPU_GPR(R0)(r4)
937	ld r2, VCPU_GPR(R2)(r4)
938	ld r3, VCPU_GPR(R3)(r4)
939	ld r4, VCPU_GPR(R4)(r4)
940	HRFI_TO_GUEST
941	b .
942	SYM_CODE_END(kvmppc_hv_entry)
943
944	secondary_too_late:
945	li r12, 0
946	stw r12, STACK_SLOT_TRAP(r1)
947	cmpdi r4, 0
948	beq 11f
949	stw r12, VCPU_TRAP(r4)
950	#ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING
951	addi r3, r4, VCPU_TB_RMEXIT
952	bl kvmhv_accumulate_time
953	#endif
954	11: b kvmhv_switch_to_host
955
956	no_switch_exit:
957	HMT_MEDIUM
958	li r12, 0
959	b 12f
960	hdec_soon:
961	li r12, BOOK3S_INTERRUPT_HV_DECREMENTER
962	12: stw r12, VCPU_TRAP(r4)
963	mr r9, r4
964	#ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING
965	addi r3, r4, VCPU_TB_RMEXIT
966	bl kvmhv_accumulate_time
967	#endif
968	b guest_bypass
969
970	/******************************************************************************
971	* *
972	* Exit code *
973	* *
974	*****************************************************************************/
975
976	/*
977	* We come here from the first-level interrupt handlers.
978	*/
979	.globl kvmppc_interrupt_hv
980	kvmppc_interrupt_hv:
981	/*
982	* Register contents:
983	* R9 = HSTATE_IN_GUEST
984	* R12 = (guest CR << 32) | interrupt vector
985	* R13 = PACA
986	* guest R12 saved in shadow VCPU SCRATCH0
987	* guest R13 saved in SPRN_SCRATCH0
988	* guest R9 saved in HSTATE_SCRATCH2
989	*/
990	/* We're now back in the host but in guest MMU context */
991	cmpwi r9,KVM_GUEST_MODE_HOST_HV
992	beq kvmppc_bad_host_intr
993	li r9, KVM_GUEST_MODE_HOST_HV
994	stb r9, HSTATE_IN_GUEST(r13)
995
996	ld r9, HSTATE_KVM_VCPU(r13)
997
998	/* Save registers */
999
1000	std r0, VCPU_GPR(R0)(r9)
1001	std r1, VCPU_GPR(R1)(r9)
1002	std r2, VCPU_GPR(R2)(r9)
1003	std r3, VCPU_GPR(R3)(r9)
1004	std r4, VCPU_GPR(R4)(r9)
1005	std r5, VCPU_GPR(R5)(r9)
1006	std r6, VCPU_GPR(R6)(r9)
1007	std r7, VCPU_GPR(R7)(r9)
1008	std r8, VCPU_GPR(R8)(r9)
1009	ld r0, HSTATE_SCRATCH2(r13)
1010	std r0, VCPU_GPR(R9)(r9)
1011	std r10, VCPU_GPR(R10)(r9)
1012	std r11, VCPU_GPR(R11)(r9)
1013	ld r3, HSTATE_SCRATCH0(r13)
1014	std r3, VCPU_GPR(R12)(r9)
1015	/* CR is in the high half of r12 */
1016	srdi r4, r12, 32
1017	std r4, VCPU_CR(r9)
1018	BEGIN_FTR_SECTION
1019	ld r3, HSTATE_CFAR(r13)
1020	std r3, VCPU_CFAR(r9)
1021	END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
1022	BEGIN_FTR_SECTION
1023	ld r4, HSTATE_PPR(r13)
1024	std r4, VCPU_PPR(r9)
1025	END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
1026
1027	/* Restore R1/R2 so we can handle faults */
1028	ld r1, HSTATE_HOST_R1(r13)
1029	LOAD_PACA_TOC()
1030
1031	mfspr r10, SPRN_SRR0
1032	mfspr r11, SPRN_SRR1
1033	std r10, VCPU_SRR0(r9)
1034	std r11, VCPU_SRR1(r9)
1035	/* trap is in the low half of r12, clear CR from the high half */
1036	clrldi r12, r12, 32
1037	andi. r0, r12, 2 /* need to read HSRR0/1? */
1038	beq 1f
1039	mfspr r10, SPRN_HSRR0
1040	mfspr r11, SPRN_HSRR1
1041	clrrdi r12, r12, 2
1042	1: std r10, VCPU_PC(r9)
1043	std r11, VCPU_MSR(r9)
1044
1045	GET_SCRATCH0(r3)
1046	mflr r4
1047	std r3, VCPU_GPR(R13)(r9)
1048	std r4, VCPU_LR(r9)
1049
1050	stw r12,VCPU_TRAP(r9)
1051
1052	/*
1053	* Now that we have saved away SRR0/1 and HSRR0/1,
1054	* interrupts are recoverable in principle, so set MSR_RI.
1055	* This becomes important for relocation-on interrupts from
1056	* the guest, which we can get in radix mode on POWER9.
1057	*/
1058	li r0, MSR_RI
1059	mtmsrd r0, 1
1060
1061	#ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING
1062	addi r3, r9, VCPU_TB_RMINTR
1063	mr r4, r9
1064	bl kvmhv_accumulate_time
1065	ld r5, VCPU_GPR(R5)(r9)
1066	ld r6, VCPU_GPR(R6)(r9)
1067	ld r7, VCPU_GPR(R7)(r9)
1068	ld r8, VCPU_GPR(R8)(r9)
1069	#endif
1070
1071	/* Save HEIR (HV emulation assist reg) in emul_inst
1072	if this is an HEI (HV emulation interrupt, e40) */
1073	li r3,KVM_INST_FETCH_FAILED
1074	std r3,VCPU_LAST_INST(r9)
1075	cmpwi r12,BOOK3S_INTERRUPT_H_EMUL_ASSIST
1076	bne 11f
1077	mfspr r3,SPRN_HEIR
1078	11: std r3,VCPU_HEIR(r9)
1079
1080	/* these are volatile across C function calls */
1081	mfctr r3
1082	mfxer r4
1083	std r3, VCPU_CTR(r9)
1084	std r4, VCPU_XER(r9)
1085
1086	/* Save more register state */
1087	mfdar r3
1088	mfdsisr r4
1089	std r3, VCPU_DAR(r9)
1090	stw r4, VCPU_DSISR(r9)
1091
1092	/* If this is a page table miss then see if it's theirs or ours */
1093	cmpwi r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
1094	beq kvmppc_hdsi
1095	std r3, VCPU_FAULT_DAR(r9)
1096	stw r4, VCPU_FAULT_DSISR(r9)
1097	cmpwi r12, BOOK3S_INTERRUPT_H_INST_STORAGE
1098	beq kvmppc_hisi
1099
1100	/* See if this is a leftover HDEC interrupt */
1101	cmpwi r12,BOOK3S_INTERRUPT_HV_DECREMENTER
1102	bne 2f
1103	mfspr r3,SPRN_HDEC
1104	extsw r3, r3
1105	cmpdi r3,0
1106	mr r4,r9
1107	bge fast_guest_return
1108	2:
1109	/* See if this is an hcall we can handle in real mode */
1110	cmpwi r12,BOOK3S_INTERRUPT_SYSCALL
1111	beq hcall_try_real_mode
1112
1113	/* Hypervisor doorbell - exit only if host IPI flag set */
1114	cmpwi r12, BOOK3S_INTERRUPT_H_DOORBELL
1115	bne 3f
1116	lbz r0, HSTATE_HOST_IPI(r13)
1117	cmpwi r0, 0
1118	beq maybe_reenter_guest
1119	b guest_exit_cont
1120	3:
1121	/* If it's a hypervisor facility unavailable interrupt, save HFSCR */
1122	cmpwi r12, BOOK3S_INTERRUPT_H_FAC_UNAVAIL
1123	bne 14f
1124	mfspr r3, SPRN_HFSCR
1125	std r3, VCPU_HFSCR(r9)
1126	b guest_exit_cont
1127	14:
1128	/* External interrupt ? */
1129	cmpwi r12, BOOK3S_INTERRUPT_EXTERNAL
1130	beq kvmppc_guest_external
1131	/* See if it is a machine check */
1132	cmpwi r12, BOOK3S_INTERRUPT_MACHINE_CHECK
1133	beq machine_check_realmode
1134	/* Or a hypervisor maintenance interrupt */
1135	cmpwi r12, BOOK3S_INTERRUPT_HMI
1136	beq hmi_realmode
1137
1138	guest_exit_cont: /* r9 = vcpu, r12 = trap, r13 = paca */
1139
1140	#ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING
1141	addi r3, r9, VCPU_TB_RMEXIT
1142	mr r4, r9
1143	bl kvmhv_accumulate_time
1144	#endif
1145
1146	/*
1147	* Possibly flush the link stack here, before we do a blr in
1148	* kvmhv_switch_to_host.
1149	*/
1150	1: nop
1151	patch_site 1b patch__call_kvm_flush_link_stack
1152
1153	/* For hash guest, read the guest SLB and save it away */
1154	li r5, 0
1155	lwz r0,VCPU_SLB_NR(r9) /* number of entries in SLB */
1156	mtctr r0
1157	li r6,0
1158	addi r7,r9,VCPU_SLB
1159	1: slbmfee r8,r6
1160	andis. r0,r8,SLB_ESID_V@h
1161	beq 2f
1162	add r8,r8,r6 /* put index in */
1163	slbmfev r3,r6
1164	std r8,VCPU_SLB_E(r7)
1165	std r3,VCPU_SLB_V(r7)
1166	addi r7,r7,VCPU_SLB_SIZE
1167	addi r5,r5,1
1168	2: addi r6,r6,1
1169	bdnz 1b
1170	/* Finally clear out the SLB */
1171	li r0,0
1172	slbmte r0,r0
1173	PPC_SLBIA(6)
1174	ptesync
1175	stw r5,VCPU_SLB_MAX(r9)
1176
1177	/* load host SLB entries */
1178	ld r8,PACA_SLBSHADOWPTR(r13)
1179
1180	.rept SLB_NUM_BOLTED
1181	li r3, SLBSHADOW_SAVEAREA
1182	LDX_BE r5, r8, r3
1183	addi r3, r3, 8
1184	LDX_BE r6, r8, r3
1185	andis. r7,r5,SLB_ESID_V@h
1186	beq 1f
1187	slbmte r6,r5
1188	1: addi r8,r8,16
1189	.endr
1190
1191	guest_bypass:
1192	stw r12, STACK_SLOT_TRAP(r1)
1193
1194	/* Save DEC */
1195	/* Do this before kvmhv_commence_exit so we know TB is guest TB */
1196	ld r3, HSTATE_KVM_VCORE(r13)
1197	mfspr r5,SPRN_DEC
1198	mftb r6
1199	extsw r5,r5
1200	16: add r5,r5,r6
1201	std r5,VCPU_DEC_EXPIRES(r9)
1202
1203	/* Increment exit count, poke other threads to exit */
1204	mr r3, r12
1205	bl kvmhv_commence_exit
1206	nop
1207	ld r9, HSTATE_KVM_VCPU(r13)
1208
1209	/* Stop others sending VCPU interrupts to this physical CPU */
1210	li r0, -1
1211	stw r0, VCPU_CPU(r9)
1212	stw r0, VCPU_THREAD_CPU(r9)
1213
1214	/* Save guest CTRL register, set runlatch to 1 if it was clear */
1215	mfspr r6,SPRN_CTRLF
1216	stw r6,VCPU_CTRL(r9)
1217	andi. r0,r6,1
1218	bne 4f
1219	li r6,1
1220	mtspr SPRN_CTRLT,r6
1221	4:
1222	/*
1223	* Save the guest PURR/SPURR
1224	*/
1225	mfspr r5,SPRN_PURR
1226	mfspr r6,SPRN_SPURR
1227	ld r7,VCPU_PURR(r9)
1228	ld r8,VCPU_SPURR(r9)
1229	std r5,VCPU_PURR(r9)
1230	std r6,VCPU_SPURR(r9)
1231	subf r5,r7,r5
1232	subf r6,r8,r6
1233
1234	/*
1235	* Restore host PURR/SPURR and add guest times
1236	* so that the time in the guest gets accounted.
1237	*/
1238	ld r3,HSTATE_PURR(r13)
1239	ld r4,HSTATE_SPURR(r13)
1240	add r3,r3,r5
1241	add r4,r4,r6
1242	mtspr SPRN_PURR,r3
1243	mtspr SPRN_SPURR,r4
1244
1245	BEGIN_FTR_SECTION
1246	b 8f
1247	END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
1248	/* Save POWER8-specific registers */
1249	mfspr r5, SPRN_IAMR
1250	mfspr r6, SPRN_PSPB
1251	mfspr r7, SPRN_FSCR
1252	std r5, VCPU_IAMR(r9)
1253	stw r6, VCPU_PSPB(r9)
1254	std r7, VCPU_FSCR(r9)
1255	mfspr r5, SPRN_IC
1256	mfspr r7, SPRN_TAR
1257	std r5, VCPU_IC(r9)
1258	std r7, VCPU_TAR(r9)
1259	mfspr r8, SPRN_EBBHR
1260	std r8, VCPU_EBBHR(r9)
1261	mfspr r5, SPRN_EBBRR
1262	mfspr r6, SPRN_BESCR
1263	mfspr r7, SPRN_PID
1264	mfspr r8, SPRN_WORT
1265	std r5, VCPU_EBBRR(r9)
1266	std r6, VCPU_BESCR(r9)
1267	stw r7, VCPU_GUEST_PID(r9)
1268	std r8, VCPU_WORT(r9)
1269	mfspr r5, SPRN_TCSCR
1270	mfspr r6, SPRN_ACOP
1271	mfspr r7, SPRN_CSIGR
1272	mfspr r8, SPRN_TACR
1273	std r5, VCPU_TCSCR(r9)
1274	std r6, VCPU_ACOP(r9)
1275	std r7, VCPU_CSIGR(r9)
1276	std r8, VCPU_TACR(r9)
1277	BEGIN_FTR_SECTION
1278	ld r5, STACK_SLOT_FSCR(r1)
1279	mtspr SPRN_FSCR, r5
1280	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1281	/*
1282	* Restore various registers to 0, where non-zero values
1283	* set by the guest could disrupt the host.
1284	*/
1285	li r0, 0
1286	mtspr SPRN_PSPB, r0
1287	mtspr SPRN_WORT, r0
1288	mtspr SPRN_TCSCR, r0
1289	/* Set MMCRS to 1<<31 to freeze and disable the SPMC counters */
1290	li r0, 1
1291	sldi r0, r0, 31
1292	mtspr SPRN_MMCRS, r0
1293
1294	/* Save and restore AMR, IAMR and UAMOR before turning on the MMU */
1295	ld r8, STACK_SLOT_IAMR(r1)
1296	mtspr SPRN_IAMR, r8
1297
1298	8: /* Power7 jumps back in here */
1299	mfspr r5,SPRN_AMR
1300	mfspr r6,SPRN_UAMOR
1301	std r5,VCPU_AMR(r9)
1302	std r6,VCPU_UAMOR(r9)
1303	ld r5,STACK_SLOT_AMR(r1)
1304	ld r6,STACK_SLOT_UAMOR(r1)
1305	mtspr SPRN_AMR, r5
1306	mtspr SPRN_UAMOR, r6
1307
1308	/* Switch DSCR back to host value */
1309	mfspr r8, SPRN_DSCR
1310	ld r7, HSTATE_DSCR(r13)
1311	std r8, VCPU_DSCR(r9)
1312	mtspr SPRN_DSCR, r7
1313
1314	/* Save non-volatile GPRs */
1315	std r14, VCPU_GPR(R14)(r9)
1316	std r15, VCPU_GPR(R15)(r9)
1317	std r16, VCPU_GPR(R16)(r9)
1318	std r17, VCPU_GPR(R17)(r9)
1319	std r18, VCPU_GPR(R18)(r9)
1320	std r19, VCPU_GPR(R19)(r9)
1321	std r20, VCPU_GPR(R20)(r9)
1322	std r21, VCPU_GPR(R21)(r9)
1323	std r22, VCPU_GPR(R22)(r9)
1324	std r23, VCPU_GPR(R23)(r9)
1325	std r24, VCPU_GPR(R24)(r9)
1326	std r25, VCPU_GPR(R25)(r9)
1327	std r26, VCPU_GPR(R26)(r9)
1328	std r27, VCPU_GPR(R27)(r9)
1329	std r28, VCPU_GPR(R28)(r9)
1330	std r29, VCPU_GPR(R29)(r9)
1331	std r30, VCPU_GPR(R30)(r9)
1332	std r31, VCPU_GPR(R31)(r9)
1333
1334	/* Save SPRGs */
1335	mfspr r3, SPRN_SPRG0
1336	mfspr r4, SPRN_SPRG1
1337	mfspr r5, SPRN_SPRG2
1338	mfspr r6, SPRN_SPRG3
1339	std r3, VCPU_SPRG0(r9)
1340	std r4, VCPU_SPRG1(r9)
1341	std r5, VCPU_SPRG2(r9)
1342	std r6, VCPU_SPRG3(r9)
1343
1344	/* save FP state */
1345	mr r3, r9
1346	bl kvmppc_save_fp
1347
1348	#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1349	BEGIN_FTR_SECTION
1350	b 91f
1351	END_FTR_SECTION_IFCLR(CPU_FTR_TM)
1352	/*
1353	* NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR)
1354	*/
1355	mr r3, r9
1356	ld r4, VCPU_MSR(r3)
1357	li r5, 0 /* don't preserve non-vol regs */
1358	bl kvmppc_save_tm_hv
1359	nop
1360	ld r9, HSTATE_KVM_VCPU(r13)
1361	91:
1362	#endif
1363
1364	/* Increment yield count if they have a VPA */
1365	ld r8, VCPU_VPA(r9) /* do they have a VPA? */
1366	cmpdi r8, 0
1367	beq 25f
1368	li r4, LPPACA_YIELDCOUNT
1369	LWZX_BE r3, r8, r4
1370	addi r3, r3, 1
1371	STWX_BE r3, r8, r4
1372	li r3, 1
1373	stb r3, VCPU_VPA_DIRTY(r9)
1374	25:
1375	/* Save PMU registers if requested */
1376	/* r8 and cr0.eq are live here */
1377	mr r3, r9
1378	li r4, 1
1379	beq 21f /* if no VPA, save PMU stuff anyway */
1380	lbz r4, LPPACA_PMCINUSE(r8)
1381	21: bl kvmhv_save_guest_pmu
1382	ld r9, HSTATE_KVM_VCPU(r13)
1383
1384	/* Restore host values of some registers */
1385	BEGIN_FTR_SECTION
1386	ld r5, STACK_SLOT_CIABR(r1)
1387	ld r6, STACK_SLOT_DAWR0(r1)
1388	ld r7, STACK_SLOT_DAWRX0(r1)
1389	mtspr SPRN_CIABR, r5
1390	/*
1391	* If the DAWR doesn't work, it's ok to write these here as
1392	* this value should always be zero
1393	*/
1394	mtspr SPRN_DAWR0, r6
1395	mtspr SPRN_DAWRX0, r7
1396	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1397
1398	/*
1399	* POWER7/POWER8 guest -> host partition switch code.
1400	* We don't have to lock against tlbies but we do
1401	* have to coordinate the hardware threads.
1402	* Here STACK_SLOT_TRAP(r1) contains the trap number.
1403	*/
1404	kvmhv_switch_to_host:
1405	/* Secondary threads wait for primary to do partition switch */
1406	ld r5,HSTATE_KVM_VCORE(r13)
1407	ld r4,VCORE_KVM(r5) /* pointer to struct kvm */
1408	lbz r3,HSTATE_PTID(r13)
1409	cmpwi r3,0
1410	beq 15f
1411	HMT_LOW
1412	13: lbz r3,VCORE_IN_GUEST(r5)
1413	cmpwi r3,0
1414	bne 13b
1415	HMT_MEDIUM
1416	b 16f
1417
1418	/* Primary thread waits for all the secondaries to exit guest */
1419	15: lwz r3,VCORE_ENTRY_EXIT(r5)
1420	rlwinm r0,r3,32-8,0xff
1421	clrldi r3,r3,56
1422	cmpw r3,r0
1423	bne 15b
1424	isync
1425
1426	/* Did we actually switch to the guest at all? */
1427	lbz r6, VCORE_IN_GUEST(r5)
1428	cmpwi r6, 0
1429	beq 19f
1430
1431	/* Primary thread switches back to host partition */
1432	lwz r7,KVM_HOST_LPID(r4)
1433	ld r6,KVM_HOST_SDR1(r4)
1434	li r8,LPID_RSVD /* switch to reserved LPID */
1435	mtspr SPRN_LPID,r8
1436	ptesync
1437	mtspr SPRN_SDR1,r6 /* switch to host page table */
1438	mtspr SPRN_LPID,r7
1439	isync
1440
1441	BEGIN_FTR_SECTION
1442	/* DPDES and VTB are shared between threads */
1443	mfspr r7, SPRN_DPDES
1444	mfspr r8, SPRN_VTB
1445	std r7, VCORE_DPDES(r5)
1446	std r8, VCORE_VTB(r5)
1447	/* clear DPDES so we don't get guest doorbells in the host */
1448	li r8, 0
1449	mtspr SPRN_DPDES, r8
1450	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1451
1452	/* Subtract timebase offset from timebase */
1453	ld r8, VCORE_TB_OFFSET_APPL(r5)
1454	cmpdi r8,0
1455	beq 17f
1456	li r0, 0
1457	std r0, VCORE_TB_OFFSET_APPL(r5)
1458	mftb r6 /* current guest timebase */
1459	subf r8,r8,r6
1460	mtspr SPRN_TBU40,r8 /* update upper 40 bits */
1461	mftb r7 /* check if lower 24 bits overflowed */
1462	clrldi r6,r6,40
1463	clrldi r7,r7,40
1464	cmpld r7,r6
1465	bge 17f
1466	addis r8,r8,0x100 /* if so, increment upper 40 bits */
1467	mtspr SPRN_TBU40,r8
1468
1469	17:
1470	/*
1471	* If this is an HMI, we called kvmppc_realmode_hmi_handler
1472	* above, which may or may not have already called
1473	* kvmppc_subcore_exit_guest. Fortunately, all that
1474	* kvmppc_subcore_exit_guest does is clear a flag, so calling
1475	* it again here is benign even if kvmppc_realmode_hmi_handler
1476	* has already called it.
1477	*/
1478	bl kvmppc_subcore_exit_guest
1479	nop
1480	30: ld r5,HSTATE_KVM_VCORE(r13)
1481	ld r4,VCORE_KVM(r5) /* pointer to struct kvm */
1482
1483	/* Reset PCR */
1484	ld r0, VCORE_PCR(r5)
1485	LOAD_REG_IMMEDIATE(r6, PCR_MASK)
1486	cmpld r0, r6
1487	beq 18f
1488	mtspr SPRN_PCR, r6
1489	18:
1490	/* Signal secondary CPUs to continue */
1491	li r0, 0
1492	stb r0,VCORE_IN_GUEST(r5)
1493	19: lis r8,0x7fff /* MAX_INT@h */
1494	mtspr SPRN_HDEC,r8
1495
1496	16: ld r8,KVM_HOST_LPCR(r4)
1497	mtspr SPRN_LPCR,r8
1498	isync
1499
1500	#ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING
1501	/* Finish timing, if we have a vcpu */
1502	ld r4, HSTATE_KVM_VCPU(r13)
1503	cmpdi r4, 0
1504	li r3, 0
1505	beq 2f
1506	bl kvmhv_accumulate_time
1507	2:
1508	#endif
1509	/* Unset guest mode */
1510	li r0, KVM_GUEST_MODE_NONE
1511	stb r0, HSTATE_IN_GUEST(r13)
1512
1513	lwz r12, STACK_SLOT_TRAP(r1) /* return trap # in r12 */
1514	ld r0, SFS+PPC_LR_STKOFF(r1)
1515	addi r1, r1, SFS
1516	mtlr r0
1517	blr
1518
1519	.balign 32
1520	.global kvm_flush_link_stack
1521	kvm_flush_link_stack:
1522	/* Save LR into r0 */
1523	mflr r0
1524
1525	/* Flush the link stack. On Power8 it's up to 32 entries in size. */
1526	.rept 32
1527	bl .+4
1528	.endr
1529
1530	/* And on Power9 it's up to 64. */
1531	BEGIN_FTR_SECTION
1532	.rept 32
1533	bl .+4
1534	.endr
1535	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
1536
1537	/* Restore LR */
1538	mtlr r0
1539	blr
1540
1541	kvmppc_guest_external:
1542	/* External interrupt, first check for host_ipi. If this is
1543	* set, we know the host wants us out so let's do it now
1544	*/
1545	bl CFUNC(kvmppc_read_intr)
1546
1547	/*
1548	* Restore the active volatile registers after returning from
1549	* a C function.
1550	*/
1551	ld r9, HSTATE_KVM_VCPU(r13)
1552	li r12, BOOK3S_INTERRUPT_EXTERNAL
1553
1554	/*
1555	* kvmppc_read_intr return codes:
1556	*
1557	* Exit to host (r3 > 0)
1558	* 1 An interrupt is pending that needs to be handled by the host
1559	* Exit guest and return to host by branching to guest_exit_cont
1560	*
1561	* 2 Passthrough that needs completion in the host
1562	* Exit guest and return to host by branching to guest_exit_cont
1563	* However, we also set r12 to BOOK3S_INTERRUPT_HV_RM_HARD
1564	* to indicate to the host to complete handling the interrupt
1565	*
1566	* Before returning to guest, we check if any CPU is heading out
1567	* to the host and if so, we head out also. If no CPUs are heading
1568	* check return values <= 0.
1569	*
1570	* Return to guest (r3 <= 0)
1571	* 0 No external interrupt is pending
1572	* -1 A guest wakeup IPI (which has now been cleared)
1573	* In either case, we return to guest to deliver any pending
1574	* guest interrupts.
1575	*
1576	* -2 A PCI passthrough external interrupt was handled
1577	* (interrupt was delivered directly to guest)
1578	* Return to guest to deliver any pending guest interrupts.
1579	*/
1580
1581	cmpdi r3, 1
1582	ble 1f
1583
1584	/* Return code = 2 */
1585	li r12, BOOK3S_INTERRUPT_HV_RM_HARD
1586	stw r12, VCPU_TRAP(r9)
1587	b guest_exit_cont
1588
1589	1: /* Return code <= 1 */
1590	cmpdi r3, 0
1591	bgt guest_exit_cont
1592
1593	/* Return code <= 0 */
1594	maybe_reenter_guest:
1595	ld r5, HSTATE_KVM_VCORE(r13)
1596	lwz r0, VCORE_ENTRY_EXIT(r5)
1597	cmpwi r0, 0x100
1598	mr r4, r9
1599	blt deliver_guest_interrupt
1600	b guest_exit_cont
1601
1602	/*
1603	* Check whether an HDSI is an HPTE not found fault or something else.
1604	* If it is an HPTE not found fault that is due to the guest accessing
1605	* a page that they have mapped but which we have paged out, then
1606	* we continue on with the guest exit path. In all other cases,
1607	* reflect the HDSI to the guest as a DSI.
1608	*/
1609	kvmppc_hdsi:
1610	mfspr r4, SPRN_HDAR
1611	mfspr r6, SPRN_HDSISR
1612	/* HPTE not found fault or protection fault? */
1613	andis. r0, r6, (DSISR_NOHPTE | DSISR_PROTFAULT)@h
1614	beq 1f /* if not, send it to the guest */
1615	andi. r0, r11, MSR_DR /* data relocation enabled? */
1616	beq 3f
1617	clrrdi r0, r4, 28
1618	PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */
1619	li r0, BOOK3S_INTERRUPT_DATA_SEGMENT
1620	bne 7f /* if no SLB entry found */
1621	4: std r4, VCPU_FAULT_DAR(r9)
1622	stw r6, VCPU_FAULT_DSISR(r9)
1623
1624	/* Search the hash table. */
1625	mr r3, r9 /* vcpu pointer */
1626	li r7, 1 /* data fault */
1627	bl CFUNC(kvmppc_hpte_hv_fault)
1628	ld r9, HSTATE_KVM_VCPU(r13)
1629	ld r10, VCPU_PC(r9)
1630	ld r11, VCPU_MSR(r9)
1631	li r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
1632	cmpdi r3, 0 /* retry the instruction */
1633	beq 6f
1634	cmpdi r3, -1 /* handle in kernel mode */
1635	beq guest_exit_cont
1636	cmpdi r3, -2 /* MMIO emulation; need instr word */
1637	beq 2f
1638
1639	/* Synthesize a DSI (or DSegI) for the guest */
1640	ld r4, VCPU_FAULT_DAR(r9)
1641	mr r6, r3
1642	1: li r0, BOOK3S_INTERRUPT_DATA_STORAGE
1643	mtspr SPRN_DSISR, r6
1644	7: mtspr SPRN_DAR, r4
1645	mtspr SPRN_SRR0, r10
1646	mtspr SPRN_SRR1, r11
1647	mr r10, r0
1648	bl kvmppc_msr_interrupt
1649	fast_interrupt_c_return:
1650	6: ld r7, VCPU_CTR(r9)
1651	ld r8, VCPU_XER(r9)
1652	mtctr r7
1653	mtxer r8
1654	mr r4, r9
1655	b fast_guest_return
1656
1657	3: ld r5, VCPU_KVM(r9) /* not relocated, use VRMA */
1658	ld r5, KVM_VRMA_SLB_V(r5)
1659	b 4b
1660
1661	/* If this is for emulated MMIO, load the instruction word */
1662	2: li r8, KVM_INST_FETCH_FAILED /* In case lwz faults */
1663
1664	/* Set guest mode to 'jump over instruction' so if lwz faults
1665	* we'll just continue at the next IP. */
1666	li r0, KVM_GUEST_MODE_SKIP
1667	stb r0, HSTATE_IN_GUEST(r13)
1668
1669	/* Do the access with MSR:DR enabled */
1670	mfmsr r3
1671	ori r4, r3, MSR_DR /* Enable paging for data */
1672	mtmsrd r4
1673	lwz r8, 0(r10)
1674	mtmsrd r3
1675
1676	/* Store the result */
1677	std r8, VCPU_LAST_INST(r9)
1678
1679	/* Unset guest mode. */
1680	li r0, KVM_GUEST_MODE_HOST_HV
1681	stb r0, HSTATE_IN_GUEST(r13)
1682	b guest_exit_cont
1683
1684	/*
1685	* Similarly for an HISI, reflect it to the guest as an ISI unless
1686	* it is an HPTE not found fault for a page that we have paged out.
1687	*/
1688	kvmppc_hisi:
1689	andis. r0, r11, SRR1_ISI_NOPT@h
1690	beq 1f
1691	andi. r0, r11, MSR_IR /* instruction relocation enabled? */
1692	beq 3f
1693	clrrdi r0, r10, 28
1694	PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */
1695	li r0, BOOK3S_INTERRUPT_INST_SEGMENT
1696	bne 7f /* if no SLB entry found */
1697	4:
1698	/* Search the hash table. */
1699	mr r3, r9 /* vcpu pointer */
1700	mr r4, r10
1701	mr r6, r11
1702	li r7, 0 /* instruction fault */
1703	bl CFUNC(kvmppc_hpte_hv_fault)
1704	ld r9, HSTATE_KVM_VCPU(r13)
1705	ld r10, VCPU_PC(r9)
1706	ld r11, VCPU_MSR(r9)
1707	li r12, BOOK3S_INTERRUPT_H_INST_STORAGE
1708	cmpdi r3, 0 /* retry the instruction */
1709	beq fast_interrupt_c_return
1710	cmpdi r3, -1 /* handle in kernel mode */
1711	beq guest_exit_cont
1712
1713	/* Synthesize an ISI (or ISegI) for the guest */
1714	mr r11, r3
1715	1: li r0, BOOK3S_INTERRUPT_INST_STORAGE
1716	7: mtspr SPRN_SRR0, r10
1717	mtspr SPRN_SRR1, r11
1718	mr r10, r0
1719	bl kvmppc_msr_interrupt
1720	b fast_interrupt_c_return
1721
1722	3: ld r6, VCPU_KVM(r9) /* not relocated, use VRMA */
1723	ld r5, KVM_VRMA_SLB_V(r6)
1724	b 4b
1725
1726	/*
1727	* Try to handle an hcall in real mode.
1728	* Returns to the guest if we handle it, or continues on up to
1729	* the kernel if we can't (i.e. if we don't have a handler for
1730	* it, or if the handler returns H_TOO_HARD).
1731	*
1732	* r5 - r8 contain hcall args,
1733	* r9 = vcpu, r10 = pc, r11 = msr, r12 = trap, r13 = paca
1734	*/
1735	hcall_try_real_mode:
1736	ld r3,VCPU_GPR(R3)(r9)
1737	andi. r0,r11,MSR_PR
1738	/* sc 1 from userspace - reflect to guest syscall */
1739	bne sc_1_fast_return
1740	clrrdi r3,r3,2
1741	cmpldi r3,hcall_real_table_end - hcall_real_table
1742	bge guest_exit_cont
1743	/* See if this hcall is enabled for in-kernel handling */
1744	ld r4, VCPU_KVM(r9)
1745	srdi r0, r3, 8 /* r0 = (r3 / 4) >> 6 */
1746	sldi r0, r0, 3 /* index into kvm->arch.enabled_hcalls[] */
1747	add r4, r4, r0
1748	ld r0, KVM_ENABLED_HCALLS(r4)
1749	rlwinm r4, r3, 32-2, 0x3f /* r4 = (r3 / 4) & 0x3f */
1750	srd r0, r0, r4
1751	andi. r0, r0, 1
1752	beq guest_exit_cont
1753	/* Get pointer to handler, if any, and call it */
1754	LOAD_REG_ADDR(r4, hcall_real_table)
1755	lwax r3,r3,r4
1756	cmpwi r3,0
1757	beq guest_exit_cont
1758	add r12,r3,r4
1759	mtctr r12
1760	mr r3,r9 /* get vcpu pointer */
1761	ld r4,VCPU_GPR(R4)(r9)
1762	bctrl
1763	cmpdi r3,H_TOO_HARD
1764	beq hcall_real_fallback
1765	ld r4,HSTATE_KVM_VCPU(r13)
1766	std r3,VCPU_GPR(R3)(r4)
1767	ld r10,VCPU_PC(r4)
1768	ld r11,VCPU_MSR(r4)
1769	b fast_guest_return
1770
1771	sc_1_fast_return:
1772	mtspr SPRN_SRR0,r10
1773	mtspr SPRN_SRR1,r11
1774	li r10, BOOK3S_INTERRUPT_SYSCALL
1775	bl kvmppc_msr_interrupt
1776	mr r4,r9
1777	b fast_guest_return
1778
1779	/* We've attempted a real mode hcall, but it's punted it back
1780	* to userspace. We need to restore some clobbered volatiles
1781	* before resuming the pass-it-to-qemu path */
1782	hcall_real_fallback:
1783	li r12,BOOK3S_INTERRUPT_SYSCALL
1784	ld r9, HSTATE_KVM_VCPU(r13)
1785
1786	b guest_exit_cont
1787
1788	.globl hcall_real_table
1789	hcall_real_table:
1790	.long 0 /* 0 - unused */
1791	.long DOTSYM(kvmppc_h_remove) - hcall_real_table
1792	.long DOTSYM(kvmppc_h_enter) - hcall_real_table
1793	.long DOTSYM(kvmppc_h_read) - hcall_real_table
1794	.long DOTSYM(kvmppc_h_clear_mod) - hcall_real_table
1795	.long DOTSYM(kvmppc_h_clear_ref) - hcall_real_table
1796	.long DOTSYM(kvmppc_h_protect) - hcall_real_table
1797	.long 0 /* 0x1c */
1798	.long 0 /* 0x20 */
1799	.long 0 /* 0x24 - H_SET_SPRG0 */
1800	.long DOTSYM(kvmppc_h_set_dabr) - hcall_real_table
1801	.long DOTSYM(kvmppc_rm_h_page_init) - hcall_real_table
1802	.long 0 /* 0x30 */
1803	.long 0 /* 0x34 */
1804	.long 0 /* 0x38 */
1805	.long 0 /* 0x3c */
1806	.long 0 /* 0x40 */
1807	.long 0 /* 0x44 */
1808	.long 0 /* 0x48 */
1809	.long 0 /* 0x4c */
1810	.long 0 /* 0x50 */
1811	.long 0 /* 0x54 */
1812	.long 0 /* 0x58 */
1813	.long 0 /* 0x5c */
1814	.long 0 /* 0x60 */
1815	#ifdef CONFIG_KVM_XICS
1816	.long DOTSYM(xics_rm_h_eoi) - hcall_real_table
1817	.long DOTSYM(xics_rm_h_cppr) - hcall_real_table
1818	.long DOTSYM(xics_rm_h_ipi) - hcall_real_table
1819	.long 0 /* 0x70 - H_IPOLL */
1820	.long DOTSYM(xics_rm_h_xirr) - hcall_real_table
1821	#else
1822	.long 0 /* 0x64 - H_EOI */
1823	.long 0 /* 0x68 - H_CPPR */
1824	.long 0 /* 0x6c - H_IPI */
1825	.long 0 /* 0x70 - H_IPOLL */
1826	.long 0 /* 0x74 - H_XIRR */
1827	#endif
1828	.long 0 /* 0x78 */
1829	.long 0 /* 0x7c */
1830	.long 0 /* 0x80 */
1831	.long 0 /* 0x84 */
1832	.long 0 /* 0x88 */
1833	.long 0 /* 0x8c */
1834	.long 0 /* 0x90 */
1835	.long 0 /* 0x94 */
1836	.long 0 /* 0x98 */
1837	.long 0 /* 0x9c */
1838	.long 0 /* 0xa0 */
1839	.long 0 /* 0xa4 */
1840	.long 0 /* 0xa8 */
1841	.long 0 /* 0xac */
1842	.long 0 /* 0xb0 */
1843	.long 0 /* 0xb4 */
1844	.long 0 /* 0xb8 */
1845	.long 0 /* 0xbc */
1846	.long 0 /* 0xc0 */
1847	.long 0 /* 0xc4 */
1848	.long 0 /* 0xc8 */
1849	.long 0 /* 0xcc */
1850	.long 0 /* 0xd0 */
1851	.long 0 /* 0xd4 */
1852	.long 0 /* 0xd8 */
1853	.long 0 /* 0xdc */
1854	.long DOTSYM(kvmppc_h_cede) - hcall_real_table
1855	.long DOTSYM(kvmppc_rm_h_confer) - hcall_real_table
1856	.long 0 /* 0xe8 */
1857	.long 0 /* 0xec */
1858	.long 0 /* 0xf0 */
1859	.long 0 /* 0xf4 */
1860	.long 0 /* 0xf8 */
1861	.long 0 /* 0xfc */
1862	.long 0 /* 0x100 */
1863	.long 0 /* 0x104 */
1864	.long 0 /* 0x108 */
1865	.long 0 /* 0x10c */
1866	.long 0 /* 0x110 */
1867	.long 0 /* 0x114 */
1868	.long 0 /* 0x118 */
1869	.long 0 /* 0x11c */
1870	.long 0 /* 0x120 */
1871	.long DOTSYM(kvmppc_h_bulk_remove) - hcall_real_table
1872	.long 0 /* 0x128 */
1873	.long 0 /* 0x12c */
1874	.long 0 /* 0x130 */
1875	.long DOTSYM(kvmppc_h_set_xdabr) - hcall_real_table
1876	.long 0 /* 0x138 */
1877	.long 0 /* 0x13c */
1878	.long 0 /* 0x140 */
1879	.long 0 /* 0x144 */
1880	.long 0 /* 0x148 */
1881	.long 0 /* 0x14c */
1882	.long 0 /* 0x150 */
1883	.long 0 /* 0x154 */
1884	.long 0 /* 0x158 */
1885	.long 0 /* 0x15c */
1886	.long 0 /* 0x160 */
1887	.long 0 /* 0x164 */
1888	.long 0 /* 0x168 */
1889	.long 0 /* 0x16c */
1890	.long 0 /* 0x170 */
1891	.long 0 /* 0x174 */
1892	.long 0 /* 0x178 */
1893	.long 0 /* 0x17c */
1894	.long 0 /* 0x180 */
1895	.long 0 /* 0x184 */
1896	.long 0 /* 0x188 */
1897	.long 0 /* 0x18c */
1898	.long 0 /* 0x190 */
1899	.long 0 /* 0x194 */
1900	.long 0 /* 0x198 */
1901	.long 0 /* 0x19c */
1902	.long 0 /* 0x1a0 */
1903	.long 0 /* 0x1a4 */
1904	.long 0 /* 0x1a8 */
1905	.long 0 /* 0x1ac */
1906	.long 0 /* 0x1b0 */
1907	.long 0 /* 0x1b4 */
1908	.long 0 /* 0x1b8 */
1909	.long 0 /* 0x1bc */
1910	.long 0 /* 0x1c0 */
1911	.long 0 /* 0x1c4 */
1912	.long 0 /* 0x1c8 */
1913	.long 0 /* 0x1cc */
1914	.long 0 /* 0x1d0 */
1915	.long 0 /* 0x1d4 */
1916	.long 0 /* 0x1d8 */
1917	.long 0 /* 0x1dc */
1918	.long 0 /* 0x1e0 */
1919	.long 0 /* 0x1e4 */
1920	.long 0 /* 0x1e8 */
1921	.long 0 /* 0x1ec */
1922	.long 0 /* 0x1f0 */
1923	.long 0 /* 0x1f4 */
1924	.long 0 /* 0x1f8 */
1925	.long 0 /* 0x1fc */
1926	.long 0 /* 0x200 */
1927	.long 0 /* 0x204 */
1928	.long 0 /* 0x208 */
1929	.long 0 /* 0x20c */
1930	.long 0 /* 0x210 */
1931	.long 0 /* 0x214 */
1932	.long 0 /* 0x218 */
1933	.long 0 /* 0x21c */
1934	.long 0 /* 0x220 */
1935	.long 0 /* 0x224 */
1936	.long 0 /* 0x228 */
1937	.long 0 /* 0x22c */
1938	.long 0 /* 0x230 */
1939	.long 0 /* 0x234 */
1940	.long 0 /* 0x238 */
1941	.long 0 /* 0x23c */
1942	.long 0 /* 0x240 */
1943	.long 0 /* 0x244 */
1944	.long 0 /* 0x248 */
1945	.long 0 /* 0x24c */
1946	.long 0 /* 0x250 */
1947	.long 0 /* 0x254 */
1948	.long 0 /* 0x258 */
1949	.long 0 /* 0x25c */
1950	.long 0 /* 0x260 */
1951	.long 0 /* 0x264 */
1952	.long 0 /* 0x268 */
1953	.long 0 /* 0x26c */
1954	.long 0 /* 0x270 */
1955	.long 0 /* 0x274 */
1956	.long 0 /* 0x278 */
1957	.long 0 /* 0x27c */
1958	.long 0 /* 0x280 */
1959	.long 0 /* 0x284 */
1960	.long 0 /* 0x288 */
1961	.long 0 /* 0x28c */
1962	.long 0 /* 0x290 */
1963	.long 0 /* 0x294 */
1964	.long 0 /* 0x298 */
1965	.long 0 /* 0x29c */
1966	.long 0 /* 0x2a0 */
1967	.long 0 /* 0x2a4 */
1968	.long 0 /* 0x2a8 */
1969	.long 0 /* 0x2ac */
1970	.long 0 /* 0x2b0 */
1971	.long 0 /* 0x2b4 */
1972	.long 0 /* 0x2b8 */
1973	.long 0 /* 0x2bc */
1974	.long 0 /* 0x2c0 */
1975	.long 0 /* 0x2c4 */
1976	.long 0 /* 0x2c8 */
1977	.long 0 /* 0x2cc */
1978	.long 0 /* 0x2d0 */
1979	.long 0 /* 0x2d4 */
1980	.long 0 /* 0x2d8 */
1981	.long 0 /* 0x2dc */
1982	.long 0 /* 0x2e0 */
1983	.long 0 /* 0x2e4 */
1984	.long 0 /* 0x2e8 */
1985	.long 0 /* 0x2ec */
1986	.long 0 /* 0x2f0 */
1987	.long 0 /* 0x2f4 */
1988	.long 0 /* 0x2f8 */
1989	#ifdef CONFIG_KVM_XICS
1990	.long DOTSYM(xics_rm_h_xirr_x) - hcall_real_table
1991	#else
1992	.long 0 /* 0x2fc - H_XIRR_X*/
1993	#endif
1994	.long DOTSYM(kvmppc_rm_h_random) - hcall_real_table
1995	.globl hcall_real_table_end
1996	hcall_real_table_end:
1997
1998	_GLOBAL_TOC(kvmppc_h_set_xdabr)
1999	EXPORT_SYMBOL_GPL(kvmppc_h_set_xdabr)
2000	andi. r0, r5, DABRX_USER | DABRX_KERNEL
2001	beq 6f
2002	li r0, DABRX_USER | DABRX_KERNEL | DABRX_BTI
2003	andc. r0, r5, r0
2004	beq 3f
2005	6: li r3, H_PARAMETER
2006	blr
2007
2008	_GLOBAL_TOC(kvmppc_h_set_dabr)
2009	EXPORT_SYMBOL_GPL(kvmppc_h_set_dabr)
2010	li r5, DABRX_USER | DABRX_KERNEL
2011	3:
2012	BEGIN_FTR_SECTION
2013	b 2f
2014	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2015	std r4,VCPU_DABR(r3)
2016	stw r5, VCPU_DABRX(r3)
2017	mtspr SPRN_DABRX, r5
2018	/* Work around P7 bug where DABR can get corrupted on mtspr */
2019	1: mtspr SPRN_DABR,r4
2020	mfspr r5, SPRN_DABR
2021	cmpd r4, r5
2022	bne 1b
2023	isync
2024	li r3,0
2025	blr
2026
2027	2:
2028	LOAD_REG_ADDR(r11, dawr_force_enable)
2029	lbz r11, 0(r11)
2030	cmpdi r11, 0
2031	bne 3f
2032	li r3, H_HARDWARE
2033	blr
2034	3:
2035	/* Emulate H_SET_DABR/X on P8 for the sake of compat mode guests */
2036	rlwimi r5, r4, 5, DAWRX_DR | DAWRX_DW
2037	rlwimi r5, r4, 2, DAWRX_WT
2038	clrrdi r4, r4, 3
2039	std r4, VCPU_DAWR0(r3)
2040	std r5, VCPU_DAWRX0(r3)
2041	/*
2042	* If came in through the real mode hcall handler then it is necessary
2043	* to write the registers since the return path won't. Otherwise it is
2044	* sufficient to store then in the vcpu struct as they will be loaded
2045	* next time the vcpu is run.
2046	*/
2047	mfmsr r6
2048	andi. r6, r6, MSR_DR /* in real mode? */
2049	bne 4f
2050	mtspr SPRN_DAWR0, r4
2051	mtspr SPRN_DAWRX0, r5
2052	4: li r3, 0
2053	blr
2054
2055	_GLOBAL(kvmppc_h_cede) /* r3 = vcpu pointer, r11 = msr, r13 = paca */
2056	ori r11,r11,MSR_EE
2057	std r11,VCPU_MSR(r3)
2058	li r0,1
2059	stb r0,VCPU_CEDED(r3)
2060	sync /* order setting ceded vs. testing prodded */
2061	lbz r5,VCPU_PRODDED(r3)
2062	cmpwi r5,0
2063	bne kvm_cede_prodded
2064	li r12,0 /* set trap to 0 to say hcall is handled */
2065	stw r12,VCPU_TRAP(r3)
2066	li r0,H_SUCCESS
2067	std r0,VCPU_GPR(R3)(r3)
2068
2069	/*
2070	* Set our bit in the bitmask of napping threads unless all the
2071	* other threads are already napping, in which case we send this
2072	* up to the host.
2073	*/
2074	ld r5,HSTATE_KVM_VCORE(r13)
2075	lbz r6,HSTATE_PTID(r13)
2076	lwz r8,VCORE_ENTRY_EXIT(r5)
2077	clrldi r8,r8,56
2078	li r0,1
2079	sld r0,r0,r6
2080	addi r6,r5,VCORE_NAPPING_THREADS
2081	31: lwarx r4,0,r6
2082	or r4,r4,r0
2083	cmpw r4,r8
2084	beq kvm_cede_exit
2085	stwcx. r4,0,r6
2086	bne 31b
2087	/* order napping_threads update vs testing entry_exit_map */
2088	isync
2089	li r0,NAPPING_CEDE
2090	stb r0,HSTATE_NAPPING(r13)
2091	lwz r7,VCORE_ENTRY_EXIT(r5)
2092	cmpwi r7,0x100
2093	bge 33f /* another thread already exiting */
2094
2095	/*
2096	* Although not specifically required by the architecture, POWER7
2097	* preserves the following registers in nap mode, even if an SMT mode
2098	* switch occurs: SLB entries, PURR, SPURR, AMOR, UAMOR, AMR, SPRG0-3,
2099	* DAR, DSISR, DABR, DABRX, DSCR, PMCx, MMCRx, SIAR, SDAR.
2100	*/
2101	/* Save non-volatile GPRs */
2102	std r14, VCPU_GPR(R14)(r3)
2103	std r15, VCPU_GPR(R15)(r3)
2104	std r16, VCPU_GPR(R16)(r3)
2105	std r17, VCPU_GPR(R17)(r3)
2106	std r18, VCPU_GPR(R18)(r3)
2107	std r19, VCPU_GPR(R19)(r3)
2108	std r20, VCPU_GPR(R20)(r3)
2109	std r21, VCPU_GPR(R21)(r3)
2110	std r22, VCPU_GPR(R22)(r3)
2111	std r23, VCPU_GPR(R23)(r3)
2112	std r24, VCPU_GPR(R24)(r3)
2113	std r25, VCPU_GPR(R25)(r3)
2114	std r26, VCPU_GPR(R26)(r3)
2115	std r27, VCPU_GPR(R27)(r3)
2116	std r28, VCPU_GPR(R28)(r3)
2117	std r29, VCPU_GPR(R29)(r3)
2118	std r30, VCPU_GPR(R30)(r3)
2119	std r31, VCPU_GPR(R31)(r3)
2120
2121	/* save FP state */
2122	bl kvmppc_save_fp
2123
2124	#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2125	BEGIN_FTR_SECTION
2126	b 91f
2127	END_FTR_SECTION_IFCLR(CPU_FTR_TM)
2128	/*
2129	* NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR)
2130	*/
2131	ld r3, HSTATE_KVM_VCPU(r13)
2132	ld r4, VCPU_MSR(r3)
2133	li r5, 0 /* don't preserve non-vol regs */
2134	bl kvmppc_save_tm_hv
2135	nop
2136	91:
2137	#endif
2138
2139	/*
2140	* Set DEC to the smaller of DEC and HDEC, so that we wake
2141	* no later than the end of our timeslice (HDEC interrupts
2142	* don't wake us from nap).
2143	*/
2144	mfspr r3, SPRN_DEC
2145	mfspr r4, SPRN_HDEC
2146	mftb r5
2147	extsw r3, r3
2148	extsw r4, r4
2149	cmpd r3, r4
2150	ble 67f
2151	mtspr SPRN_DEC, r4
2152	67:
2153	/* save expiry time of guest decrementer */
2154	add r3, r3, r5
2155	ld r4, HSTATE_KVM_VCPU(r13)
2156	std r3, VCPU_DEC_EXPIRES(r4)
2157
2158	#ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING
2159	ld r4, HSTATE_KVM_VCPU(r13)
2160	addi r3, r4, VCPU_TB_CEDE
2161	bl kvmhv_accumulate_time
2162	#endif
2163
2164	lis r3, LPCR_PECEDP@h /* Do wake on privileged doorbell */
2165
2166	/* Go back to host stack */
2167	ld r1, HSTATE_HOST_R1(r13)
2168
2169	/*
2170	* Take a nap until a decrementer or external or doobell interrupt
2171	* occurs, with PECE1 and PECE0 set in LPCR.
2172	* On POWER8, set PECEDH, and if we are ceding, also set PECEDP.
2173	* Also clear the runlatch bit before napping.
2174	*/
2175	kvm_do_nap:
2176	li r0,0
2177	mtspr SPRN_CTRLT, r0
2178
2179	li r0,1
2180	stb r0,HSTATE_HWTHREAD_REQ(r13)
2181	mfspr r5,SPRN_LPCR
2182	ori r5,r5,LPCR_PECE0 | LPCR_PECE1
2183	BEGIN_FTR_SECTION
2184	ori r5, r5, LPCR_PECEDH
2185	rlwimi r5, r3, 0, LPCR_PECEDP
2186	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2187
2188	kvm_nap_sequence: /* desired LPCR value in r5 */
2189	li r3, PNV_THREAD_NAP
2190	mtspr SPRN_LPCR,r5
2191	isync
2192
2193	bl isa206_idle_insn_mayloss
2194
2195	li r0,1
2196	mtspr SPRN_CTRLT, r0
2197
2198	mtspr SPRN_SRR1, r3
2199
2200	li r0, 0
2201	stb r0, PACA_FTRACE_ENABLED(r13)
2202
2203	li r0, KVM_HWTHREAD_IN_KVM
2204	stb r0, HSTATE_HWTHREAD_STATE(r13)
2205
2206	lbz r0, HSTATE_NAPPING(r13)
2207	cmpwi r0, NAPPING_CEDE
2208	beq kvm_end_cede
2209	cmpwi r0, NAPPING_NOVCPU
2210	beq kvm_novcpu_wakeup
2211	cmpwi r0, NAPPING_UNSPLIT
2212	beq kvm_unsplit_wakeup
2213	twi 31,0,0 /* Nap state must not be zero */
2214
2215	33: mr r4, r3
2216	li r3, 0
2217	li r12, 0
2218	b 34f
2219
2220	kvm_end_cede:
2221	/* Woken by external or decrementer interrupt */
2222
2223	/* get vcpu pointer */
2224	ld r4, HSTATE_KVM_VCPU(r13)
2225
2226	#ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING
2227	addi r3, r4, VCPU_TB_RMINTR
2228	bl kvmhv_accumulate_time
2229	#endif
2230
2231	#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2232	BEGIN_FTR_SECTION
2233	b 91f
2234	END_FTR_SECTION_IFCLR(CPU_FTR_TM)
2235	/*
2236	* NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR)
2237	*/
2238	mr r3, r4
2239	ld r4, VCPU_MSR(r3)
2240	li r5, 0 /* don't preserve non-vol regs */
2241	bl kvmppc_restore_tm_hv
2242	nop
2243	ld r4, HSTATE_KVM_VCPU(r13)
2244	91:
2245	#endif
2246
2247	/* load up FP state */
2248	bl kvmppc_load_fp
2249
2250	/* Restore guest decrementer */
2251	ld r3, VCPU_DEC_EXPIRES(r4)
2252	mftb r7
2253	subf r3, r7, r3
2254	mtspr SPRN_DEC, r3
2255
2256	/* Load NV GPRS */
2257	ld r14, VCPU_GPR(R14)(r4)
2258	ld r15, VCPU_GPR(R15)(r4)
2259	ld r16, VCPU_GPR(R16)(r4)
2260	ld r17, VCPU_GPR(R17)(r4)
2261	ld r18, VCPU_GPR(R18)(r4)
2262	ld r19, VCPU_GPR(R19)(r4)
2263	ld r20, VCPU_GPR(R20)(r4)
2264	ld r21, VCPU_GPR(R21)(r4)
2265	ld r22, VCPU_GPR(R22)(r4)
2266	ld r23, VCPU_GPR(R23)(r4)
2267	ld r24, VCPU_GPR(R24)(r4)
2268	ld r25, VCPU_GPR(R25)(r4)
2269	ld r26, VCPU_GPR(R26)(r4)
2270	ld r27, VCPU_GPR(R27)(r4)
2271	ld r28, VCPU_GPR(R28)(r4)
2272	ld r29, VCPU_GPR(R29)(r4)
2273	ld r30, VCPU_GPR(R30)(r4)
2274	ld r31, VCPU_GPR(R31)(r4)
2275
2276	/* Check the wake reason in SRR1 to see why we got here */
2277	bl kvmppc_check_wake_reason
2278
2279	/*
2280	* Restore volatile registers since we could have called a
2281	* C routine in kvmppc_check_wake_reason
2282	* r4 = VCPU
2283	* r3 tells us whether we need to return to host or not
2284	* WARNING: it gets checked further down:
2285	* should not modify r3 until this check is done.
2286	*/
2287	ld r4, HSTATE_KVM_VCPU(r13)
2288
2289	/* clear our bit in vcore->napping_threads */
2290	34: ld r5,HSTATE_KVM_VCORE(r13)
2291	lbz r7,HSTATE_PTID(r13)
2292	li r0,1
2293	sld r0,r0,r7
2294	addi r6,r5,VCORE_NAPPING_THREADS
2295	32: lwarx r7,0,r6
2296	andc r7,r7,r0
2297	stwcx. r7,0,r6
2298	bne 32b
2299	li r0,0
2300	stb r0,HSTATE_NAPPING(r13)
2301
2302	/* See if the wake reason saved in r3 means we need to exit */
2303	stw r12, VCPU_TRAP(r4)
2304	mr r9, r4
2305	cmpdi r3, 0
2306	bgt guest_exit_cont
2307	b maybe_reenter_guest
2308
2309	/* cede when already previously prodded case */
2310	kvm_cede_prodded:
2311	li r0,0
2312	stb r0,VCPU_PRODDED(r3)
2313	sync /* order testing prodded vs. clearing ceded */
2314	stb r0,VCPU_CEDED(r3)
2315	li r3,H_SUCCESS
2316	blr
2317
2318	/* we've ceded but we want to give control to the host */
2319	kvm_cede_exit:
2320	ld r9, HSTATE_KVM_VCPU(r13)
2321	b guest_exit_cont
2322
2323	/* Try to do machine check recovery in real mode */
2324	machine_check_realmode:
2325	mr r3, r9 /* get vcpu pointer */
2326	bl kvmppc_realmode_machine_check
2327	nop
2328	/* all machine checks go to virtual mode for further handling */
2329	ld r9, HSTATE_KVM_VCPU(r13)
2330	li r12, BOOK3S_INTERRUPT_MACHINE_CHECK
2331	b guest_exit_cont
2332
2333	/*
2334	* Call C code to handle a HMI in real mode.
2335	* Only the primary thread does the call, secondary threads are handled
2336	* by calling hmi_exception_realmode() after kvmppc_hv_entry returns.
2337	* r9 points to the vcpu on entry
2338	*/
2339	hmi_realmode:
2340	lbz r0, HSTATE_PTID(r13)
2341	cmpwi r0, 0
2342	bne guest_exit_cont
2343	bl CFUNC(kvmppc_realmode_hmi_handler)
2344	ld r9, HSTATE_KVM_VCPU(r13)
2345	li r12, BOOK3S_INTERRUPT_HMI
2346	b guest_exit_cont
2347
2348	/*
2349	* Check the reason we woke from nap, and take appropriate action.
2350	* Returns (in r3):
2351	* 0 if nothing needs to be done
2352	* 1 if something happened that needs to be handled by the host
2353	* -1 if there was a guest wakeup (IPI or msgsnd)
2354	* -2 if we handled a PCI passthrough interrupt (returned by
2355	* kvmppc_read_intr only)
2356	*
2357	* Also sets r12 to the interrupt vector for any interrupt that needs
2358	* to be handled now by the host (0x500 for external interrupt), or zero.
2359	* Modifies all volatile registers (since it may call a C function).
2360	* This routine calls kvmppc_read_intr, a C function, if an external
2361	* interrupt is pending.
2362	*/
2363	SYM_FUNC_START_LOCAL(kvmppc_check_wake_reason)
2364	mfspr r6, SPRN_SRR1
2365	BEGIN_FTR_SECTION
2366	rlwinm r6, r6, 45-31, 0xf /* extract wake reason field (P8) */
2367	FTR_SECTION_ELSE
2368	rlwinm r6, r6, 45-31, 0xe /* P7 wake reason field is 3 bits */
2369	ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S)
2370	cmpwi r6, 8 /* was it an external interrupt? */
2371	beq 7f /* if so, see what it was */
2372	li r3, 0
2373	li r12, 0
2374	cmpwi r6, 6 /* was it the decrementer? */
2375	beq 0f
2376	BEGIN_FTR_SECTION
2377	cmpwi r6, 5 /* privileged doorbell? */
2378	beq 0f
2379	cmpwi r6, 3 /* hypervisor doorbell? */
2380	beq 3f
2381	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2382	cmpwi r6, 0xa /* Hypervisor maintenance ? */
2383	beq 4f
2384	li r3, 1 /* anything else, return 1 */
2385	0: blr
2386
2387	/* hypervisor doorbell */
2388	3: li r12, BOOK3S_INTERRUPT_H_DOORBELL
2389
2390	/*
2391	* Clear the doorbell as we will invoke the handler
2392	* explicitly in the guest exit path.
2393	*/
2394	lis r6, (PPC_DBELL_SERVER << (63-36))@h
2395	PPC_MSGCLR(6)
2396	/* see if it's a host IPI */
2397	li r3, 1
2398	lbz r0, HSTATE_HOST_IPI(r13)
2399	cmpwi r0, 0
2400	bnelr
2401	/* if not, return -1 */
2402	li r3, -1
2403	blr
2404
2405	/* Woken up due to Hypervisor maintenance interrupt */
2406	4: li r12, BOOK3S_INTERRUPT_HMI
2407	li r3, 1
2408	blr
2409
2410	/* external interrupt - create a stack frame so we can call C */
2411	7: mflr r0
2412	std r0, PPC_LR_STKOFF(r1)
2413	stdu r1, -PPC_MIN_STKFRM(r1)
2414	bl CFUNC(kvmppc_read_intr)
2415	nop
2416	li r12, BOOK3S_INTERRUPT_EXTERNAL
2417	cmpdi r3, 1
2418	ble 1f
2419
2420	/*
2421	* Return code of 2 means PCI passthrough interrupt, but
2422	* we need to return back to host to complete handling the
2423	* interrupt. Trap reason is expected in r12 by guest
2424	* exit code.
2425	*/
2426	li r12, BOOK3S_INTERRUPT_HV_RM_HARD
2427	1:
2428	ld r0, PPC_MIN_STKFRM+PPC_LR_STKOFF(r1)
2429	addi r1, r1, PPC_MIN_STKFRM
2430	mtlr r0
2431	blr
2432	SYM_FUNC_END(kvmppc_check_wake_reason)
2433
2434	/*
2435	* Save away FP, VMX and VSX registers.
2436	* r3 = vcpu pointer
2437	* N.B. r30 and r31 are volatile across this function,
2438	* thus it is not callable from C.
2439	*/
2440	SYM_FUNC_START_LOCAL(kvmppc_save_fp)
2441	mflr r30
2442	mr r31,r3
2443	mfmsr r5
2444	ori r8,r5,MSR_FP
2445	#ifdef CONFIG_ALTIVEC
2446	BEGIN_FTR_SECTION
2447	oris r8,r8,MSR_VEC@h
2448	END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2449	#endif
2450	#ifdef CONFIG_VSX
2451	BEGIN_FTR_SECTION
2452	oris r8,r8,MSR_VSX@h
2453	END_FTR_SECTION_IFSET(CPU_FTR_VSX)
2454	#endif
2455	mtmsrd r8
2456	addi r3,r3,VCPU_FPRS
2457	bl store_fp_state
2458	#ifdef CONFIG_ALTIVEC
2459	BEGIN_FTR_SECTION
2460	addi r3,r31,VCPU_VRS
2461	bl store_vr_state
2462	END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2463	#endif
2464	mfspr r6,SPRN_VRSAVE
2465	stw r6,VCPU_VRSAVE(r31)
2466	mtlr r30
2467	blr
2468	SYM_FUNC_END(kvmppc_save_fp)
2469
2470	/*
2471	* Load up FP, VMX and VSX registers
2472	* r4 = vcpu pointer
2473	* N.B. r30 and r31 are volatile across this function,
2474	* thus it is not callable from C.
2475	*/
2476	SYM_FUNC_START_LOCAL(kvmppc_load_fp)
2477	mflr r30
2478	mr r31,r4
2479	mfmsr r9
2480	ori r8,r9,MSR_FP
2481	#ifdef CONFIG_ALTIVEC
2482	BEGIN_FTR_SECTION
2483	oris r8,r8,MSR_VEC@h
2484	END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2485	#endif
2486	#ifdef CONFIG_VSX
2487	BEGIN_FTR_SECTION
2488	oris r8,r8,MSR_VSX@h
2489	END_FTR_SECTION_IFSET(CPU_FTR_VSX)
2490	#endif
2491	mtmsrd r8
2492	addi r3,r4,VCPU_FPRS
2493	bl load_fp_state
2494	#ifdef CONFIG_ALTIVEC
2495	BEGIN_FTR_SECTION
2496	addi r3,r31,VCPU_VRS
2497	bl load_vr_state
2498	END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2499	#endif
2500	lwz r7,VCPU_VRSAVE(r31)
2501	mtspr SPRN_VRSAVE,r7
2502	mtlr r30
2503	mr r4,r31
2504	blr
2505	SYM_FUNC_END(kvmppc_load_fp)
2506
2507	#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2508	/*
2509	* Save transactional state and TM-related registers.
2510	* Called with r3 pointing to the vcpu struct and r4 containing
2511	* the guest MSR value.
2512	* r5 is non-zero iff non-volatile register state needs to be maintained.
2513	* If r5 == 0, this can modify all checkpointed registers, but
2514	* restores r1 and r2 before exit.
2515	*/
2516	_GLOBAL_TOC(kvmppc_save_tm_hv)
2517	EXPORT_SYMBOL_GPL(kvmppc_save_tm_hv)
2518	/* See if we need to handle fake suspend mode */
2519	BEGIN_FTR_SECTION
2520	b __kvmppc_save_tm
2521	END_FTR_SECTION_IFCLR(CPU_FTR_P9_TM_HV_ASSIST)
2522
2523	lbz r0, HSTATE_FAKE_SUSPEND(r13) /* Were we fake suspended? */
2524	cmpwi r0, 0
2525	beq __kvmppc_save_tm
2526
2527	/* The following code handles the fake_suspend = 1 case */
2528	mflr r0
2529	std r0, PPC_LR_STKOFF(r1)
2530	stdu r1, -TM_FRAME_SIZE(r1)
2531
2532	/* Turn on TM. */
2533	mfmsr r8
2534	li r0, 1
2535	rldimi r8, r0, MSR_TM_LG, 63-MSR_TM_LG
2536	mtmsrd r8
2537
2538	rldicl. r8, r8, 64 - MSR_TS_S_LG, 62 /* Did we actually hrfid? */
2539	beq 4f
2540	BEGIN_FTR_SECTION
2541	bl pnv_power9_force_smt4_catch
2542	END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_XER_SO_BUG)
2543	nop
2544
2545	/*
2546	* It's possible that treclaim. may modify registers, if we have lost
2547	* track of fake-suspend state in the guest due to it using rfscv.
2548	* Save and restore registers in case this occurs.
2549	*/
2550	mfspr r3, SPRN_DSCR
2551	mfspr r4, SPRN_XER
2552	mfspr r5, SPRN_AMR
2553	/* SPRN_TAR would need to be saved here if the kernel ever used it */
2554	mfcr r12
2555	SAVE_NVGPRS(r1)
2556	SAVE_GPR(2, r1)
2557	SAVE_GPR(3, r1)
2558	SAVE_GPR(4, r1)
2559	SAVE_GPR(5, r1)
2560	stw r12, 8(r1)
2561	std r1, HSTATE_HOST_R1(r13)
2562
2563	/* We have to treclaim here because that's the only way to do S->N */
2564	li r3, TM_CAUSE_KVM_RESCHED
2565	TRECLAIM(R3)
2566
2567	GET_PACA(r13)
2568	ld r1, HSTATE_HOST_R1(r13)
2569	REST_GPR(2, r1)
2570	REST_GPR(3, r1)
2571	REST_GPR(4, r1)
2572	REST_GPR(5, r1)
2573	lwz r12, 8(r1)
2574	REST_NVGPRS(r1)
2575	mtspr SPRN_DSCR, r3
2576	mtspr SPRN_XER, r4
2577	mtspr SPRN_AMR, r5
2578	mtcr r12
2579	HMT_MEDIUM
2580
2581	/*
2582	* We were in fake suspend, so we are not going to save the
2583	* register state as the guest checkpointed state (since
2584	* we already have it), therefore we can now use any volatile GPR.
2585	* In fact treclaim in fake suspend state doesn't modify
2586	* any registers.
2587	*/
2588
2589	BEGIN_FTR_SECTION
2590	bl pnv_power9_force_smt4_release
2591	END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_XER_SO_BUG)
2592	nop
2593
2594	4:
2595	mfspr r3, SPRN_PSSCR
2596	/* PSSCR_FAKE_SUSPEND is a write-only bit, but clear it anyway */
2597	li r0, PSSCR_FAKE_SUSPEND
2598	andc r3, r3, r0
2599	mtspr SPRN_PSSCR, r3
2600
2601	/* Don't save TEXASR, use value from last exit in real suspend state */
2602	ld r9, HSTATE_KVM_VCPU(r13)
2603	mfspr r5, SPRN_TFHAR
2604	mfspr r6, SPRN_TFIAR
2605	std r5, VCPU_TFHAR(r9)
2606	std r6, VCPU_TFIAR(r9)
2607
2608	addi r1, r1, TM_FRAME_SIZE
2609	ld r0, PPC_LR_STKOFF(r1)
2610	mtlr r0
2611	blr
2612
2613	/*
2614	* Restore transactional state and TM-related registers.
2615	* Called with r3 pointing to the vcpu struct
2616	* and r4 containing the guest MSR value.
2617	* r5 is non-zero iff non-volatile register state needs to be maintained.
2618	* This potentially modifies all checkpointed registers.
2619	* It restores r1 and r2 from the PACA.
2620	*/
2621	_GLOBAL_TOC(kvmppc_restore_tm_hv)
2622	EXPORT_SYMBOL_GPL(kvmppc_restore_tm_hv)
2623	/*
2624	* If we are doing TM emulation for the guest on a POWER9 DD2,
2625	* then we don't actually do a trechkpt -- we either set up
2626	* fake-suspend mode, or emulate a TM rollback.
2627	*/
2628	BEGIN_FTR_SECTION
2629	b __kvmppc_restore_tm
2630	END_FTR_SECTION_IFCLR(CPU_FTR_P9_TM_HV_ASSIST)
2631	mflr r0
2632	std r0, PPC_LR_STKOFF(r1)
2633
2634	li r0, 0
2635	stb r0, HSTATE_FAKE_SUSPEND(r13)
2636
2637	/* Turn on TM so we can restore TM SPRs */
2638	mfmsr r5
2639	li r0, 1
2640	rldimi r5, r0, MSR_TM_LG, 63-MSR_TM_LG
2641	mtmsrd r5
2642
2643	/*
2644	* The user may change these outside of a transaction, so they must
2645	* always be context switched.
2646	*/
2647	ld r5, VCPU_TFHAR(r3)
2648	ld r6, VCPU_TFIAR(r3)
2649	ld r7, VCPU_TEXASR(r3)
2650	mtspr SPRN_TFHAR, r5
2651	mtspr SPRN_TFIAR, r6
2652	mtspr SPRN_TEXASR, r7
2653
2654	rldicl. r5, r4, 64 - MSR_TS_S_LG, 62
2655	beqlr /* TM not active in guest */
2656
2657	/* Make sure the failure summary is set */
2658	oris r7, r7, (TEXASR_FS)@h
2659	mtspr SPRN_TEXASR, r7
2660
2661	cmpwi r5, 1 /* check for suspended state */
2662	bgt 10f
2663	stb r5, HSTATE_FAKE_SUSPEND(r13)
2664	b 9f /* and return */
2665	10: stdu r1, -PPC_MIN_STKFRM(r1)
2666	/* guest is in transactional state, so simulate rollback */
2667	bl kvmhv_emulate_tm_rollback
2668	nop
2669	addi r1, r1, PPC_MIN_STKFRM
2670	9: ld r0, PPC_LR_STKOFF(r1)
2671	mtlr r0
2672	blr
2673	#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
2674
2675	/*
2676	* We come here if we get any exception or interrupt while we are
2677	* executing host real mode code while in guest MMU context.
2678	* r12 is (CR << 32) | vector
2679	* r13 points to our PACA
2680	* r12 is saved in HSTATE_SCRATCH0(r13)
2681	* r9 is saved in HSTATE_SCRATCH2(r13)
2682	* r13 is saved in HSPRG1
2683	* cfar is saved in HSTATE_CFAR(r13)
2684	* ppr is saved in HSTATE_PPR(r13)
2685	*/
2686	kvmppc_bad_host_intr:
2687	/*
2688	* Switch to the emergency stack, but start half-way down in
2689	* case we were already on it.
2690	*/
2691	mr r9, r1
2692	std r1, PACAR1(r13)
2693	ld r1, PACAEMERGSP(r13)
2694	subi r1, r1, THREAD_SIZE/2 + INT_FRAME_SIZE
2695	std r9, 0(r1)
2696	std r0, GPR0(r1)
2697	std r9, GPR1(r1)
2698	std r2, GPR2(r1)
2699	SAVE_GPRS(3, 8, r1)
2700	srdi r0, r12, 32
2701	clrldi r12, r12, 32
2702	std r0, _CCR(r1)
2703	std r12, _TRAP(r1)
2704	andi. r0, r12, 2
2705	beq 1f
2706	mfspr r3, SPRN_HSRR0
2707	mfspr r4, SPRN_HSRR1
2708	mfspr r5, SPRN_HDAR
2709	mfspr r6, SPRN_HDSISR
2710	b 2f
2711	1: mfspr r3, SPRN_SRR0
2712	mfspr r4, SPRN_SRR1
2713	mfspr r5, SPRN_DAR
2714	mfspr r6, SPRN_DSISR
2715	2: std r3, _NIP(r1)
2716	std r4, _MSR(r1)
2717	std r5, _DAR(r1)
2718	std r6, _DSISR(r1)
2719	ld r9, HSTATE_SCRATCH2(r13)
2720	ld r12, HSTATE_SCRATCH0(r13)
2721	GET_SCRATCH0(r0)
2722	SAVE_GPRS(9, 12, r1)
2723	std r0, GPR13(r1)
2724	SAVE_NVGPRS(r1)
2725	ld r5, HSTATE_CFAR(r13)
2726	std r5, ORIG_GPR3(r1)
2727	mflr r3
2728	mfctr r4
2729	mfxer r5
2730	lbz r6, PACAIRQSOFTMASK(r13)
2731	std r3, _LINK(r1)
2732	std r4, _CTR(r1)
2733	std r5, _XER(r1)
2734	std r6, SOFTE(r1)
2735	LOAD_PACA_TOC()
2736	LOAD_REG_IMMEDIATE(3, STACK_FRAME_REGS_MARKER)
2737	std r3, STACK_INT_FRAME_MARKER(r1)
2738
2739	/*
2740	* XXX On POWER7 and POWER8, we just spin here since we don't
2741	* know what the other threads are doing (and we don't want to
2742	* coordinate with them) - but at least we now have register state
2743	* in memory that we might be able to look at from another CPU.
2744	*/
2745	b .
2746
2747	/*
2748	* This mimics the MSR transition on IRQ delivery. The new guest MSR is taken
2749	* from VCPU_INTR_MSR and is modified based on the required TM state changes.
2750	* r11 has the guest MSR value (in/out)
2751	* r9 has a vcpu pointer (in)
2752	* r0 is used as a scratch register
2753	*/
2754	SYM_FUNC_START_LOCAL(kvmppc_msr_interrupt)
2755	rldicl r0, r11, 64 - MSR_TS_S_LG, 62
2756	cmpwi r0, 2 /* Check if we are in transactional state.. */
2757	ld r11, VCPU_INTR_MSR(r9)
2758	bne 1f
2759	/* ... if transactional, change to suspended */
2760	li r0, 1
2761	1: rldimi r11, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG
2762	blr
2763	SYM_FUNC_END(kvmppc_msr_interrupt)
2764
2765	/*
2766	* void kvmhv_load_guest_pmu(struct kvm_vcpu *vcpu)
2767	*
2768	* Load up guest PMU state. R3 points to the vcpu struct.
2769	*/
2770	SYM_FUNC_START_LOCAL(kvmhv_load_guest_pmu)
2771	mr r4, r3
2772	mflr r0
2773	li r3, 1
2774	sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */
2775	mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */
2776	isync
2777	BEGIN_FTR_SECTION
2778	ld r3, VCPU_MMCR(r4)
2779	andi. r5, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO
2780	cmpwi r5, MMCR0_PMAO
2781	beql kvmppc_fix_pmao
2782	END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG)
2783	lwz r3, VCPU_PMC(r4) /* always load up guest PMU registers */
2784	lwz r5, VCPU_PMC + 4(r4) /* to prevent information leak */
2785	lwz r6, VCPU_PMC + 8(r4)
2786	lwz r7, VCPU_PMC + 12(r4)
2787	lwz r8, VCPU_PMC + 16(r4)
2788	lwz r9, VCPU_PMC + 20(r4)
2789	mtspr SPRN_PMC1, r3
2790	mtspr SPRN_PMC2, r5
2791	mtspr SPRN_PMC3, r6
2792	mtspr SPRN_PMC4, r7
2793	mtspr SPRN_PMC5, r8
2794	mtspr SPRN_PMC6, r9
2795	ld r3, VCPU_MMCR(r4)
2796	ld r5, VCPU_MMCR + 8(r4)
2797	ld r6, VCPU_MMCRA(r4)
2798	ld r7, VCPU_SIAR(r4)
2799	ld r8, VCPU_SDAR(r4)
2800	mtspr SPRN_MMCR1, r5
2801	mtspr SPRN_MMCRA, r6
2802	mtspr SPRN_SIAR, r7
2803	mtspr SPRN_SDAR, r8
2804	BEGIN_FTR_SECTION
2805	ld r5, VCPU_MMCR + 16(r4)
2806	ld r6, VCPU_SIER(r4)
2807	mtspr SPRN_MMCR2, r5
2808	mtspr SPRN_SIER, r6
2809	lwz r7, VCPU_PMC + 24(r4)
2810	lwz r8, VCPU_PMC + 28(r4)
2811	ld r9, VCPU_MMCRS(r4)
2812	mtspr SPRN_SPMC1, r7
2813	mtspr SPRN_SPMC2, r8
2814	mtspr SPRN_MMCRS, r9
2815	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2816	mtspr SPRN_MMCR0, r3
2817	isync
2818	mtlr r0
2819	blr
2820	SYM_FUNC_END(kvmhv_load_guest_pmu)
2821
2822	/*
2823	* void kvmhv_load_host_pmu(void)
2824	*
2825	* Reload host PMU state saved in the PACA by kvmhv_save_host_pmu.
2826	*/
2827	SYM_FUNC_START_LOCAL(kvmhv_load_host_pmu)
2828	mflr r0
2829	lbz r4, PACA_PMCINUSE(r13) /* is the host using the PMU? */
2830	cmpwi r4, 0
2831	beq 23f /* skip if not */
2832	BEGIN_FTR_SECTION
2833	ld r3, HSTATE_MMCR0(r13)
2834	andi. r4, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO
2835	cmpwi r4, MMCR0_PMAO
2836	beql kvmppc_fix_pmao
2837	END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG)
2838	lwz r3, HSTATE_PMC1(r13)
2839	lwz r4, HSTATE_PMC2(r13)
2840	lwz r5, HSTATE_PMC3(r13)
2841	lwz r6, HSTATE_PMC4(r13)
2842	lwz r8, HSTATE_PMC5(r13)
2843	lwz r9, HSTATE_PMC6(r13)
2844	mtspr SPRN_PMC1, r3
2845	mtspr SPRN_PMC2, r4
2846	mtspr SPRN_PMC3, r5
2847	mtspr SPRN_PMC4, r6
2848	mtspr SPRN_PMC5, r8
2849	mtspr SPRN_PMC6, r9
2850	ld r3, HSTATE_MMCR0(r13)
2851	ld r4, HSTATE_MMCR1(r13)
2852	ld r5, HSTATE_MMCRA(r13)
2853	ld r6, HSTATE_SIAR(r13)
2854	ld r7, HSTATE_SDAR(r13)
2855	mtspr SPRN_MMCR1, r4
2856	mtspr SPRN_MMCRA, r5
2857	mtspr SPRN_SIAR, r6
2858	mtspr SPRN_SDAR, r7
2859	BEGIN_FTR_SECTION
2860	ld r8, HSTATE_MMCR2(r13)
2861	ld r9, HSTATE_SIER(r13)
2862	mtspr SPRN_MMCR2, r8
2863	mtspr SPRN_SIER, r9
2864	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2865	mtspr SPRN_MMCR0, r3
2866	isync
2867	mtlr r0
2868	23: blr
2869	SYM_FUNC_END(kvmhv_load_host_pmu)
2870
2871	/*
2872	* void kvmhv_save_guest_pmu(struct kvm_vcpu *vcpu, bool pmu_in_use)
2873	*
2874	* Save guest PMU state into the vcpu struct.
2875	* r3 = vcpu, r4 = full save flag (PMU in use flag set in VPA)
2876	*/
2877	SYM_FUNC_START_LOCAL(kvmhv_save_guest_pmu)
2878	mr r9, r3
2879	mr r8, r4
2880	BEGIN_FTR_SECTION
2881	/*
2882	* POWER8 seems to have a hardware bug where setting
2883	* MMCR0[PMAE] along with MMCR0[PMC1CE] and/or MMCR0[PMCjCE]
2884	* when some counters are already negative doesn't seem
2885	* to cause a performance monitor alert (and hence interrupt).
2886	* The effect of this is that when saving the PMU state,
2887	* if there is no PMU alert pending when we read MMCR0
2888	* before freezing the counters, but one becomes pending
2889	* before we read the counters, we lose it.
2890	* To work around this, we need a way to freeze the counters
2891	* before reading MMCR0. Normally, freezing the counters
2892	* is done by writing MMCR0 (to set MMCR0[FC]) which
2893	* unavoidably writes MMCR0[PMA0] as well. On POWER8,
2894	* we can also freeze the counters using MMCR2, by writing
2895	* 1s to all the counter freeze condition bits (there are
2896	* 9 bits each for 6 counters).
2897	*/
2898	li r3, -1 /* set all freeze bits */
2899	clrrdi r3, r3, 10
2900	mfspr r10, SPRN_MMCR2
2901	mtspr SPRN_MMCR2, r3
2902	isync
2903	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2904	li r3, 1
2905	sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */
2906	mfspr r4, SPRN_MMCR0 /* save MMCR0 */
2907	mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */
2908	mfspr r6, SPRN_MMCRA
2909	/* Clear MMCRA in order to disable SDAR updates */
2910	li r7, 0
2911	mtspr SPRN_MMCRA, r7
2912	isync
2913	cmpwi r8, 0 /* did they ask for PMU stuff to be saved? */
2914	bne 21f
2915	std r3, VCPU_MMCR(r9) /* if not, set saved MMCR0 to FC */
2916	b 22f
2917	21: mfspr r5, SPRN_MMCR1
2918	mfspr r7, SPRN_SIAR
2919	mfspr r8, SPRN_SDAR
2920	std r4, VCPU_MMCR(r9)
2921	std r5, VCPU_MMCR + 8(r9)
2922	std r6, VCPU_MMCRA(r9)
2923	BEGIN_FTR_SECTION
2924	std r10, VCPU_MMCR + 16(r9)
2925	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2926	std r7, VCPU_SIAR(r9)
2927	std r8, VCPU_SDAR(r9)
2928	mfspr r3, SPRN_PMC1
2929	mfspr r4, SPRN_PMC2
2930	mfspr r5, SPRN_PMC3
2931	mfspr r6, SPRN_PMC4
2932	mfspr r7, SPRN_PMC5
2933	mfspr r8, SPRN_PMC6
2934	stw r3, VCPU_PMC(r9)
2935	stw r4, VCPU_PMC + 4(r9)
2936	stw r5, VCPU_PMC + 8(r9)
2937	stw r6, VCPU_PMC + 12(r9)
2938	stw r7, VCPU_PMC + 16(r9)
2939	stw r8, VCPU_PMC + 20(r9)
2940	BEGIN_FTR_SECTION
2941	mfspr r5, SPRN_SIER
2942	std r5, VCPU_SIER(r9)
2943	mfspr r6, SPRN_SPMC1
2944	mfspr r7, SPRN_SPMC2
2945	mfspr r8, SPRN_MMCRS
2946	stw r6, VCPU_PMC + 24(r9)
2947	stw r7, VCPU_PMC + 28(r9)
2948	std r8, VCPU_MMCRS(r9)
2949	lis r4, 0x8000
2950	mtspr SPRN_MMCRS, r4
2951	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2952	22: blr
2953	SYM_FUNC_END(kvmhv_save_guest_pmu)
2954
2955	/*
2956	* This works around a hardware bug on POWER8E processors, where
2957	* writing a 1 to the MMCR0[PMAO] bit doesn't generate a
2958	* performance monitor interrupt. Instead, when we need to have
2959	* an interrupt pending, we have to arrange for a counter to overflow.
2960	*/
2961	kvmppc_fix_pmao:
2962	li r3, 0
2963	mtspr SPRN_MMCR2, r3
2964	lis r3, (MMCR0_PMXE | MMCR0_FCECE)@h
2965	ori r3, r3, MMCR0_PMCjCE | MMCR0_C56RUN
2966	mtspr SPRN_MMCR0, r3
2967	lis r3, 0x7fff
2968	ori r3, r3, 0xffff
2969	mtspr SPRN_PMC6, r3
2970	isync
2971	blr
2972
2973	#ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING
2974	/*
2975	* Start timing an activity
2976	* r3 = pointer to time accumulation struct, r4 = vcpu
2977	*/
2978	kvmhv_start_timing:
2979	ld r5, HSTATE_KVM_VCORE(r13)
2980	ld r6, VCORE_TB_OFFSET_APPL(r5)
2981	mftb r5
2982	subf r5, r6, r5 /* subtract current timebase offset */
2983	std r3, VCPU_CUR_ACTIVITY(r4)
2984	std r5, VCPU_ACTIVITY_START(r4)
2985	blr
2986
2987	/*
2988	* Accumulate time to one activity and start another.
2989	* r3 = pointer to new time accumulation struct, r4 = vcpu
2990	*/
2991	kvmhv_accumulate_time:
2992	ld r5, HSTATE_KVM_VCORE(r13)
2993	ld r8, VCORE_TB_OFFSET_APPL(r5)
2994	ld r5, VCPU_CUR_ACTIVITY(r4)
2995	ld r6, VCPU_ACTIVITY_START(r4)
2996	std r3, VCPU_CUR_ACTIVITY(r4)
2997	mftb r7
2998	subf r7, r8, r7 /* subtract current timebase offset */
2999	std r7, VCPU_ACTIVITY_START(r4)
3000	cmpdi r5, 0
3001	beqlr
3002	subf r3, r6, r7
3003	ld r8, TAS_SEQCOUNT(r5)
3004	cmpdi r8, 0
3005	addi r8, r8, 1
3006	std r8, TAS_SEQCOUNT(r5)
3007	lwsync
3008	ld r7, TAS_TOTAL(r5)
3009	add r7, r7, r3
3010	std r7, TAS_TOTAL(r5)
3011	ld r6, TAS_MIN(r5)
3012	ld r7, TAS_MAX(r5)
3013	beq 3f
3014	cmpd r3, r6
3015	bge 1f
3016	3: std r3, TAS_MIN(r5)
3017	1: cmpd r3, r7
3018	ble 2f
3019	std r3, TAS_MAX(r5)
3020	2: lwsync
3021	addi r8, r8, 1
3022	std r8, TAS_SEQCOUNT(r5)
3023	blr
3024	#endif
3025