Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) /* SPDX-License-Identifier: GPL-2.0-only */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  * Derived from book3s_hv_rmhandlers.S, which is:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9) #include <asm/reg.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) #include <asm/ppc_asm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) #include <asm/asm-offsets.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) #include <asm/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) #include <asm/tm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) #include <asm/cputable.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) #define VCPU_GPRS_TM(reg) (((reg) * ULONG_SIZE) + VCPU_GPR_TM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20)  * Save transactional state and TM-related registers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21)  * Called with:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22)  * - r3 pointing to the vcpu struct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23)  * - r4 containing the MSR with current TS bits:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24)  * 	(For HV KVM, it is VCPU_MSR ; For PR KVM, it is host MSR).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25)  * - r5 containing a flag indicating that non-volatile registers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26)  *	must be preserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27)  * If r5 == 0, this can modify all checkpointed registers, but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28)  * restores r1, r2 before exit.  If r5 != 0, this restores the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29)  * MSR TM/FP/VEC/VSX bits to their state on entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) _GLOBAL(__kvmppc_save_tm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) 	mflr	r0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) 	std	r0, PPC_LR_STKOFF(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) 	stdu    r1, -SWITCH_FRAME_SIZE(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) 	mr	r9, r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) 	cmpdi	cr7, r5, 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) 	/* Turn on TM. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) 	mfmsr	r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) 	mr	r10, r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) 	li	r0, 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) 	rldimi	r8, r0, MSR_TM_LG, 63-MSR_TM_LG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) 	ori     r8, r8, MSR_FP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) 	oris    r8, r8, (MSR_VEC | MSR_VSX)@h
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) 	mtmsrd	r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) 	rldicl. r4, r4, 64 - MSR_TS_S_LG, 62
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) 	beq	1f	/* TM not active in guest. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) 	std	r1, HSTATE_SCRATCH2(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) 	std	r3, HSTATE_SCRATCH1(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) 	/* Save CR on the stack - even if r5 == 0 we need to get cr7 back. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) 	mfcr	r6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 	SAVE_GPR(6, r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) 	/* Save DSCR so we can restore it to avoid running with user value */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 	mfspr	r7, SPRN_DSCR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 	SAVE_GPR(7, r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 	 * We are going to do treclaim., which will modify all checkpointed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) 	 * registers.  Save the non-volatile registers on the stack if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 	 * preservation of non-volatile state has been requested.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) 	beq	cr7, 3f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 	SAVE_NVGPRS(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 	/* MSR[TS] will be 0 (non-transactional) once we do treclaim. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 	li	r0, 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) 	rldimi	r10, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 	SAVE_GPR(10, r1)	/* final MSR value */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) BEGIN_FTR_SECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 	/* Emulation of the treclaim instruction needs TEXASR before treclaim */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 	mfspr	r6, SPRN_TEXASR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 	std	r6, VCPU_ORIG_TEXASR(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_HV_ASSIST)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 	/* Clear the MSR RI since r1, r13 are all going to be foobar. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 	li	r5, 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 	mtmsrd	r5, 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 	li	r3, TM_CAUSE_KVM_RESCHED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 	/* All GPRs are volatile at this point. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 	TRECLAIM(R3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 	/* Temporarily store r13 and r9 so we have some regs to play with */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 	SET_SCRATCH0(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 	GET_PACA(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 	std	r9, PACATMSCRATCH(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 	ld	r9, HSTATE_SCRATCH1(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 	/* Save away PPR soon so we don't run with user value. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 	std	r0, VCPU_GPRS_TM(0)(r9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 	mfspr	r0, SPRN_PPR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 	HMT_MEDIUM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 	/* Reload stack pointer. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 	std	r1, VCPU_GPRS_TM(1)(r9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 	ld	r1, HSTATE_SCRATCH2(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 	/* Set MSR RI now we have r1 and r13 back. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 	std	r2, VCPU_GPRS_TM(2)(r9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 	li	r2, MSR_RI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 	mtmsrd	r2, 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 	/* Reload TOC pointer. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 	ld	r2, PACATOC(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 	/* Save all but r0-r2, r9 & r13 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 	reg = 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 	.rept	29
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 	.if (reg != 9) && (reg != 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 	std	reg, VCPU_GPRS_TM(reg)(r9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 	.endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 	reg = reg + 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 	.endr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 	/* ... now save r13 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 	GET_SCRATCH0(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 	std	r4, VCPU_GPRS_TM(13)(r9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 	/* ... and save r9 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 	ld	r4, PACATMSCRATCH(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 	std	r4, VCPU_GPRS_TM(9)(r9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 	/* Restore host DSCR and CR values, after saving guest values */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 	mfcr	r6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 	mfspr	r7, SPRN_DSCR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 	stw	r6, VCPU_CR_TM(r9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 	std	r7, VCPU_DSCR_TM(r9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 	REST_GPR(6, r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 	REST_GPR(7, r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 	mtcr	r6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 	mtspr	SPRN_DSCR, r7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 	/* Save away checkpointed SPRs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 	std	r0, VCPU_PPR_TM(r9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 	mflr	r5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 	mfctr	r7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 	mfspr	r8, SPRN_AMR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 	mfspr	r10, SPRN_TAR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 	mfxer	r11
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 	std	r5, VCPU_LR_TM(r9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 	std	r7, VCPU_CTR_TM(r9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 	std	r8, VCPU_AMR_TM(r9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 	std	r10, VCPU_TAR_TM(r9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 	std	r11, VCPU_XER_TM(r9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 	/* Save FP/VSX. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 	addi	r3, r9, VCPU_FPRS_TM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 	bl	store_fp_state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 	addi	r3, r9, VCPU_VRS_TM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 	bl	store_vr_state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 	mfspr	r6, SPRN_VRSAVE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 	stw	r6, VCPU_VRSAVE_TM(r9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 	/* Restore non-volatile registers if requested to */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) 	beq	cr7, 1f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) 	REST_NVGPRS(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 	REST_GPR(10, r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 	 * We need to save these SPRs after the treclaim so that the software
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 	 * error code is recorded correctly in the TEXASR.  Also the user may
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) 	 * change these outside of a transaction, so they must always be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 	 * context switched.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 	mfspr	r7, SPRN_TEXASR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 	std	r7, VCPU_TEXASR(r9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 	mfspr	r5, SPRN_TFHAR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 	mfspr	r6, SPRN_TFIAR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 	std	r5, VCPU_TFHAR(r9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 	std	r6, VCPU_TFIAR(r9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 	/* Restore MSR state if requested */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 	beq	cr7, 2f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 	mtmsrd	r10, 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 	addi	r1, r1, SWITCH_FRAME_SIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 	ld	r0, PPC_LR_STKOFF(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 	mtlr	r0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 	blr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189)  * _kvmppc_save_tm_pr() is a wrapper around __kvmppc_save_tm(), so that it can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190)  * be invoked from C function by PR KVM only.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) _GLOBAL(_kvmppc_save_tm_pr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 	mflr	r0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 	std	r0, PPC_LR_STKOFF(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 	stdu    r1, -PPC_MIN_STKFRM(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) 	mfspr   r8, SPRN_TAR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 	std	r8, PPC_MIN_STKFRM-8(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) 	li	r5, 1		/* preserve non-volatile registers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 	bl	__kvmppc_save_tm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 	ld	r8, PPC_MIN_STKFRM-8(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 	mtspr   SPRN_TAR, r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 	addi    r1, r1, PPC_MIN_STKFRM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) 	ld	r0, PPC_LR_STKOFF(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) 	mtlr	r0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 	blr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) EXPORT_SYMBOL_GPL(_kvmppc_save_tm_pr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214)  * Restore transactional state and TM-related registers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215)  * Called with:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216)  *  - r3 pointing to the vcpu struct.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217)  *  - r4 is the guest MSR with desired TS bits:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)  * 	For HV KVM, it is VCPU_MSR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219)  * 	For PR KVM, it is provided by caller
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220)  * - r5 containing a flag indicating that non-volatile registers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)  *	must be preserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222)  * If r5 == 0, this potentially modifies all checkpointed registers, but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223)  * restores r1, r2 from the PACA before exit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224)  * If r5 != 0, this restores the MSR TM/FP/VEC/VSX bits to their state on entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) _GLOBAL(__kvmppc_restore_tm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 	mflr	r0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 	std	r0, PPC_LR_STKOFF(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 	cmpdi	cr7, r5, 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) 	/* Turn on TM/FP/VSX/VMX so we can restore them. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 	mfmsr	r5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 	mr	r10, r5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 	li	r6, MSR_TM >> 32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 	sldi	r6, r6, 32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 	or	r5, r5, r6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 	ori	r5, r5, MSR_FP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 	oris	r5, r5, (MSR_VEC | MSR_VSX)@h
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 	mtmsrd	r5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) 	 * The user may change these outside of a transaction, so they must
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) 	 * always be context switched.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) 	ld	r5, VCPU_TFHAR(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 	ld	r6, VCPU_TFIAR(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) 	ld	r7, VCPU_TEXASR(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) 	mtspr	SPRN_TFHAR, r5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) 	mtspr	SPRN_TFIAR, r6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) 	mtspr	SPRN_TEXASR, r7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) 	mr	r5, r4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) 	rldicl. r5, r5, 64 - MSR_TS_S_LG, 62
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 	beq	9f		/* TM not active in guest */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 	/* Make sure the failure summary is set, otherwise we'll program check
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) 	 * when we trechkpt.  It's possible that this might have been not set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 	 * on a kvmppc_set_one_reg() call but we shouldn't let this crash the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) 	 * host.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) 	oris	r7, r7, (TEXASR_FS)@h
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 	mtspr	SPRN_TEXASR, r7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) 	 * Make a stack frame and save non-volatile registers if requested.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 	stdu	r1, -SWITCH_FRAME_SIZE(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) 	std	r1, HSTATE_SCRATCH2(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 	mfcr	r6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 	mfspr	r7, SPRN_DSCR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) 	SAVE_GPR(2, r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 	SAVE_GPR(6, r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 	SAVE_GPR(7, r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) 	beq	cr7, 4f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) 	SAVE_NVGPRS(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) 	/* MSR[TS] will be 1 (suspended) once we do trechkpt */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) 	li	r0, 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) 	rldimi	r10, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) 	SAVE_GPR(10, r1)	/* final MSR value */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) 4:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) 	 * We need to load up the checkpointed state for the guest.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) 	 * We need to do this early as it will blow away any GPRs, VSRs and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) 	 * some SPRs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) 	mr	r31, r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) 	addi	r3, r31, VCPU_FPRS_TM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) 	bl	load_fp_state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) 	addi	r3, r31, VCPU_VRS_TM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) 	bl	load_vr_state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) 	mr	r3, r31
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) 	lwz	r7, VCPU_VRSAVE_TM(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) 	mtspr	SPRN_VRSAVE, r7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) 	ld	r5, VCPU_LR_TM(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) 	lwz	r6, VCPU_CR_TM(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) 	ld	r7, VCPU_CTR_TM(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) 	ld	r8, VCPU_AMR_TM(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) 	ld	r9, VCPU_TAR_TM(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) 	ld	r10, VCPU_XER_TM(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) 	mtlr	r5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) 	mtcr	r6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) 	mtctr	r7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) 	mtspr	SPRN_AMR, r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) 	mtspr	SPRN_TAR, r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) 	mtxer	r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) 	 * Load up PPR and DSCR values but don't put them in the actual SPRs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) 	 * till the last moment to avoid running with userspace PPR and DSCR for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) 	 * too long.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) 	ld	r29, VCPU_DSCR_TM(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) 	ld	r30, VCPU_PPR_TM(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) 	/* Clear the MSR RI since r1, r13 are all going to be foobar. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) 	li	r5, 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) 	mtmsrd	r5, 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) 	/* Load GPRs r0-r28 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) 	reg = 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) 	.rept	29
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) 	ld	reg, VCPU_GPRS_TM(reg)(r31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) 	reg = reg + 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) 	.endr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) 	mtspr	SPRN_DSCR, r29
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) 	mtspr	SPRN_PPR, r30
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) 	/* Load final GPRs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) 	ld	29, VCPU_GPRS_TM(29)(r31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) 	ld	30, VCPU_GPRS_TM(30)(r31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) 	ld	31, VCPU_GPRS_TM(31)(r31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) 	/* TM checkpointed state is now setup.  All GPRs are now volatile. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) 	TRECHKPT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) 	/* Now let's get back the state we need. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) 	HMT_MEDIUM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) 	GET_PACA(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) 	ld	r1, HSTATE_SCRATCH2(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) 	REST_GPR(7, r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) 	mtspr	SPRN_DSCR, r7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) 	/* Set the MSR RI since we have our registers back. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) 	li	r5, MSR_RI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) 	mtmsrd	r5, 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) 	/* Restore TOC pointer and CR */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) 	REST_GPR(2, r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) 	REST_GPR(6, r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) 	mtcr	r6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) 	/* Restore non-volatile registers if requested to. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) 	beq	cr7, 5f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) 	REST_GPR(10, r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) 	REST_NVGPRS(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) 5:	addi	r1, r1, SWITCH_FRAME_SIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) 	ld	r0, PPC_LR_STKOFF(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) 	mtlr	r0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) 9:	/* Restore MSR bits if requested */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) 	beqlr	cr7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) 	mtmsrd	r10, 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) 	blr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374)  * _kvmppc_restore_tm_pr() is a wrapper around __kvmppc_restore_tm(), so that it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375)  * can be invoked from C function by PR KVM only.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) _GLOBAL(_kvmppc_restore_tm_pr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) 	mflr	r0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) 	std	r0, PPC_LR_STKOFF(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) 	stdu    r1, -PPC_MIN_STKFRM(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) 	/* save TAR so that it can be recovered later */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) 	mfspr   r8, SPRN_TAR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) 	std	r8, PPC_MIN_STKFRM-8(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) 	li	r5, 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) 	bl	__kvmppc_restore_tm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) 	ld	r8, PPC_MIN_STKFRM-8(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) 	mtspr   SPRN_TAR, r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) 	addi    r1, r1, PPC_MIN_STKFRM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) 	ld	r0, PPC_LR_STKOFF(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) 	mtlr	r0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) 	blr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) EXPORT_SYMBOL_GPL(_kvmppc_restore_tm_pr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) #endif /* CONFIG_PPC_TRANSACTIONAL_MEM */